src/vendorcode/amd/pi/00670F00/gcc-intrin.h

   1 /* SPDX-License-Identifier: BSD-3-Clause */
   2
   3 /*
   4  * Copyright (c) 2011, Advanced Micro Devices, Inc. All rights reserved.
   5  * Copyright (c) 2014, Edward O'Callaghan <eocallaghan@alterapraxis.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *     * Redistributions of source code must retain the above copyright
  10  *       notice, this list of conditions and the following disclaimer.
  11  *     * Redistributions in binary form must reproduce the above copyright
  12  *       notice, this list of conditions and the following disclaimer in the
  13  *       documentation and/or other materials provided with the distribution.
  14  *     * Neither the name of Advanced Micro Devices, Inc. nor the names of
  15  *       its contributors may be used to endorse or promote products derived
  16  *       from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21  * DISCLAIMED. IN NO EVENT SHALL ADVANCED MICRO DEVICES, INC. BE LIABLE FOR ANY
  22  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  */
  30
  31 #include <check_for_wrapper.h>
  32
  33 #if defined (__GNUC__)
  34 #include <stdint.h>
  35 /* I/O intrin functions.  */
  36 static __inline__ __attribute__((always_inline)) uint8_t __inbyte(uint16_t Port)
  37 {
  38   uint8_t value;
  39
  40   __asm__ __volatile__ (
  41     "in  %1, %0"
  42     : "=a" (value)
  43     : "Nd" (Port)
  44     );
  45
  46   return value;
  47 }
  48
  49 static __inline__ __attribute__((always_inline)) uint16_t __inword(uint16_t Port)
  50 {
  51   uint16_t value;
  52
  53   __asm__ __volatile__ (
  54     "in  %1, %0"
  55     : "=a" (value)
  56     : "Nd" (Port)
  57     );
  58
  59   return value;
  60 }
  61
  62 static __inline__ __attribute__((always_inline)) uint32_t __indword(uint16_t Port)
  63 {
  64   uint32_t value;
  65
  66   __asm__ __volatile__ (
  67     "in  %1, %0"
  68     : "=a" (value)
  69     : "Nd" (Port)
  70     );
  71   return value;
  72
  73 }
  74
  75 static __inline__ __attribute__((always_inline)) void __outbyte(uint16_t Port,uint8_t Data)
  76 {
  77   __asm__ __volatile__ (
  78     "out  %0, %1"
  79     :
  80     : "a" (Data), "Nd" (Port)
  81     );
  82 }
  83
  84 static __inline__ __attribute__((always_inline)) void __outword(uint16_t Port,uint16_t Data)
  85 {
  86   __asm__ __volatile__ (
  87     "out  %0, %1"
  88     :
  89     : "a" (Data), "Nd" (Port)
  90     );
  91 }
  92
  93 static __inline__ __attribute__((always_inline)) void __outdword(uint16_t Port,uint32_t Data)
  94 {
  95   __asm__ __volatile__ (
  96     "out  %0, %1"
  97     :
  98     : "a" (Data), "Nd" (Port)
  99     );
 100 }
 101
 102 static __inline__ __attribute__((always_inline)) void __inbytestring(uint16_t Port,uint8_t *Buffer,unsigned long Count)
 103 {
 104   __asm__ __volatile__ (
 105     "rep ; insb"
 106     : "+D" (Buffer), "+c" (Count)
 107     : "d"(Port)
 108     );
 109 }
 110
 111 static __inline__ __attribute__((always_inline)) void __inwordstring(uint16_t Port,uint16_t *Buffer,unsigned long Count)
 112 {
 113   __asm__ __volatile__ (
 114     "rep ; insw"
 115     : "+D" (Buffer), "+c" (Count)
 116     : "d"(Port)
 117     );
 118 }
 119
 120 static __inline__ __attribute__((always_inline)) void __indwordstring(uint16_t Port,unsigned long *Buffer,unsigned long Count)
 121 {
 122   __asm__ __volatile__ (
 123     "rep ; insl"
 124     : "+D" (Buffer), "+c" (Count)
 125     : "d"(Port)
 126     );
 127 }
 128
 129 static __inline__ __attribute__((always_inline)) void __outbytestring(uint16_t Port,uint8_t *Buffer,unsigned long Count)
 130 {
 131   __asm__ __volatile__ (
 132     "rep ; outsb"
 133     : "+S" (Buffer), "+c" (Count)
 134     : "d"(Port)
 135     );
 136 }
 137
 138 static __inline__ __attribute__((always_inline)) void __outwordstring(uint16_t Port,uint16_t *Buffer,unsigned long Count)
 139 {
 140   __asm__ __volatile__ (
 141     "rep ; outsw"
 142     : "+S" (Buffer), "+c" (Count)
 143     : "d"(Port)
 144   );
 145 }
 146
 147 static __inline__ __attribute__((always_inline)) void __outdwordstring(uint16_t Port,unsigned long *Buffer,unsigned long Count)
 148 {
 149   __asm__ __volatile__ (
 150    "rep ; outsl"
 151    : "+S" (Buffer), "+c" (Count)
 152    : "d"(Port)
 153    );
 154 }
 155
 156 static __inline__ __attribute__((always_inline)) unsigned long __readdr0(void)
 157 {
 158   unsigned long value;
 159   __asm__ __volatile__ (
 160     "mov %%dr0, %[value]"
 161     : [value] "=r" (value)
 162     );
 163   return value;
 164 }
 165
 166 static __inline__ __attribute__((always_inline)) unsigned long __readdr1(void)
 167 {
 168   unsigned long value;
 169   __asm__ __volatile__ (
 170     "mov %%dr1, %[value]"
 171     : [value] "=r" (value)
 172     );
 173   return value;
 174 }
 175
 176 static __inline__ __attribute__((always_inline)) unsigned long __readdr2(void)
 177 {
 178   unsigned long value;
 179   __asm__ __volatile__ (
 180     "mov %%dr2, %[value]"
 181     : [value] "=r" (value)
 182     );
 183   return value;
 184 }
 185
 186 static __inline__ __attribute__((always_inline)) unsigned long __readdr3(void)
 187 {
 188   unsigned long value;
 189   __asm__ __volatile__ (
 190     "mov %%dr3, %[value]"
 191     : [value] "=r" (value)
 192     );
 193   return value;
 194 }
 195
 196 static __inline__ __attribute__((always_inline)) unsigned long __readdr7(void)
 197 {
 198   unsigned long value;
 199   __asm__ __volatile__ (
 200     "mov %%dr7, %[value]"
 201     : [value] "=r" (value)
 202     );
 203   return value;
 204 }
 205
 206 static __inline__ __attribute__((always_inline)) unsigned long __readdr(unsigned long reg)
 207 {
 208   switch (reg){
 209     case 0:
 210       return __readdr0 ();
 211       break;
 212
 213     case 1:
 214       return __readdr1 ();
 215       break;
 216
 217     case 2:
 218       return __readdr2 ();
 219       break;
 220
 221     case 3:
 222       return __readdr3 ();
 223       break;
 224
 225     case 7:
 226       return __readdr7 ();
 227       break;
 228
 229     default:
 230       return -1;
 231   }
 232 }
 233
 234 static __inline__ __attribute__((always_inline)) void __writedr0(unsigned long Data)
 235 {
 236   __asm__ __volatile__ (
 237     "mov %0, %%dr0"
 238     :
 239     : "r" (Data)
 240     );
 241 }
 242
 243 static __inline__ __attribute__((always_inline)) void __writedr1(unsigned long Data)
 244 {
 245   __asm__ __volatile__ (
 246     "mov %0, %%dr1"
 247     :
 248     : "r" (Data)
 249     );
 250 }
 251
 252 static __inline__ __attribute__((always_inline)) void __writedr2(unsigned long Data)
 253 {
 254   __asm__ __volatile__ (
 255     "mov %0, %%dr2"
 256     :
 257     : "r" (Data)
 258     );
 259 }
 260
 261 static __inline__ __attribute__((always_inline)) void __writedr3(unsigned long Data)
 262 {
 263   __asm__ __volatile__ (
 264     "mov %0, %%dr3"
 265     :
 266     : "r" (Data)
 267     );
 268 }
 269
 270 static __inline__ __attribute__((always_inline)) void __writedr7(unsigned long Data)
 271 {
 272   __asm__ __volatile__ (
 273     "mov %0, %%dr7"
 274     :
 275     : "r" (Data)
 276     );
 277 }
 278
 279 static __inline__ __attribute__((always_inline)) void __writedr(unsigned long reg, unsigned long Data)
 280 {
 281   switch (reg){
 282     case 0:
 283       __writedr0 (Data);
 284       break;
 285
 286     case 1:
 287       __writedr1 (Data);
 288       break;
 289
 290     case 2:
 291       __writedr2 (Data);
 292       break;
 293
 294     case 3:
 295       __writedr3 (Data);
 296       break;
 297
 298     case 7:
 299       __writedr7 (Data);
 300       break;
 301
 302     default:
 303       ;
 304   }
 305 }
 306
 307 static __inline__ __attribute__((always_inline)) unsigned long __readcr0(void)
 308 {
 309   unsigned long value;
 310   __asm__ __volatile__ (
 311     "mov %%cr0, %[value]"
 312     : [value] "=r" (value));
 313   return value;
 314 }
 315
 316 static __inline__ __attribute__((always_inline)) unsigned long __readcr2(void)
 317 {
 318   unsigned long value;
 319   __asm__ __volatile__ (
 320     "mov %%cr2, %[value]"
 321     : [value] "=r" (value));
 322   return value;
 323 }
 324
 325 static __inline__ __attribute__((always_inline)) unsigned long __readcr3(void)
 326 {
 327   unsigned long value;
 328   __asm__ __volatile__ (
 329     "mov %%cr3, %[value]"
 330     : [value] "=r" (value));
 331   return value;
 332 }
 333
 334 static __inline__ __attribute__((always_inline)) unsigned long __readcr4(void)
 335 {
 336   unsigned long value;
 337   __asm__ __volatile__ (
 338     "mov %%cr4, %[value]"
 339     : [value] "=r" (value));
 340   return value;
 341 }
 342
 343 static __inline__ __attribute__((always_inline)) unsigned long __readcr8(void)
 344 {
 345   unsigned long value;
 346   __asm__ __volatile__ (
 347     "mov %%cr8, %[value]"
 348     : [value] "=r" (value));
 349   return value;
 350 }
 351
 352 static __inline__ __attribute__((always_inline)) unsigned long __readcr(unsigned long reg)
 353 {
 354   switch (reg){
 355     case 0:
 356       return __readcr0 ();
 357       break;
 358
 359     case 2:
 360       return __readcr2 ();
 361       break;
 362
 363     case 3:
 364       return __readcr3 ();
 365       break;
 366
 367     case 4:
 368       return __readcr4 ();
 369       break;
 370
 371     case 8:
 372       return __readcr8 ();
 373       break;
 374
 375     default:
 376       return -1;
 377   }
 378 }
 379
 380 static __inline__ __attribute__((always_inline)) void __writecr0(unsigned long Data)
 381 {
 382   __asm__ __volatile__ (
 383     "mov %0, %%cr0"
 384     :
 385     : "r" (Data)
 386     : "memory"
 387     );
 388 }
 389
 390 static __inline__ __attribute__((always_inline)) void __writecr2(unsigned long Data)
 391 {
 392   __asm__ __volatile__ (
 393     "mov %0, %%cr2"
 394     :
 395     : "r" (Data)
 396     );
 397 }
 398
 399 static __inline__ __attribute__((always_inline)) void __writecr3(unsigned long Data)
 400 {
 401   __asm__ __volatile__ (
 402     "mov %0, %%cr3"
 403     :
 404     : "r" (Data)
 405     );
 406 }
 407
 408 static __inline__ __attribute__((always_inline)) void __writecr4(unsigned long Data)
 409 {
 410   __asm__ __volatile__ (
 411     "mov %0, %%cr4"
 412     :
 413     : "r" (Data)
 414     );
 415 }
 416
 417 static __inline__ __attribute__((always_inline)) void __writecr8(unsigned long Data)
 418 {
 419   __asm__ __volatile__ (
 420     "mov %0, %%cr8"
 421     :
 422     : "r" (Data)
 423     );
 424 }
 425
 426 static __inline__ __attribute__((always_inline)) void __writecr(unsigned long reg, unsigned long Data)
 427 {
 428   switch (reg){
 429     case 0:
 430       __writecr0 (Data);
 431       break;
 432
 433     case 2:
 434       __writecr2 (Data);
 435       break;
 436
 437     case 3:
 438       __writecr3 (Data);
 439       break;
 440
 441     case 4:
 442       __writecr4 (Data);
 443       break;
 444
 445     case 8:
 446       __writecr8 (Data);
 447       break;
 448
 449     default:
 450       ;
 451   }
 452 }
 453
 454 static __inline__ __attribute__((always_inline)) UINT64 __readmsr(UINT32 msr)
 455 {
 456   UINT64 retval;
 457   __asm__ __volatile__(
 458        "rdmsr"
 459        : "=A" (retval)
 460        : "c" (msr)
 461        );
 462    return retval;
 463 }
 464
 465 static __inline__ __attribute__((always_inline)) void __writemsr (UINT32 msr, UINT64 Value)
 466 {
 467   __asm__ __volatile__ (
 468      "wrmsr"
 469      :
 470      : "c" (msr), "A" (Value)
 471      );
 472 }
 473
 474 #if !defined(__clang__)
 475 static __inline__ __attribute__((always_inline)) UINT64 __rdtsc(void)
 476 {
 477   UINT64 retval;
 478   __asm__ __volatile__ (
 479      "rdtsc"
 480      : "=A" (retval));
 481   return retval;
 482 }
 483 #endif
 484
 485 static __inline__ __attribute__((always_inline)) void __cpuid(int CPUInfo[], const int InfoType)
 486 {
 487    __asm__ __volatile__(
 488      "cpuid"
 489      :"=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3])
 490      : "a" (InfoType)
 491      );
 492 }
 493
 494
 495 static __inline__ __attribute__((always_inline)) void _disable(void)
 496 {
 497   __asm__ __volatile__ ("cli");
 498 }
 499
 500
 501 static __inline__ __attribute__((always_inline)) void _enable(void)
 502 {
 503   __asm__ __volatile__ ("sti");
 504 }
 505
 506
 507 static __inline__ __attribute__((always_inline)) void __halt(void)
 508 {
 509   __asm__ __volatile__ ("hlt");
 510 }
 511
 512
 513 static __inline__ __attribute__((always_inline)) void __debugbreak(void)
 514 {
 515   __asm__ __volatile__ ("int3");
 516 }
 517
 518 static __inline__ __attribute__((always_inline)) void __invd(void)
 519 {
 520   __asm__ __volatile__ ("invd");
 521 }
 522
 523 static __inline__ __attribute__((always_inline)) void __wbinvd(void)
 524 {
 525   __asm__ __volatile__ ("wbinvd");
 526 }
 527
 528 static __inline__ __attribute__((always_inline)) void __lidt(void *Source)
 529 {
 530   __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
 531 }
 532
 533 static __inline__ __attribute__((always_inline)) void
 534 __writefsbyte(const unsigned long Offset, const uint8_t Data)
 535 {
 536   __asm__ ("movb %[Data], %%fs:%a[Offset]"
 537           :
 538           : [Offset] "ir" (Offset), [Data] "iq" (Data));
 539 }
 540
 541 static __inline__ __attribute__((always_inline)) void
 542 __writefsword(const unsigned long Offset, const uint16_t Data)
 543 {
 544   __asm__ ("movw %[Data], %%fs:%a[Offset]"
 545           :
 546           : [Offset] "ir" (Offset), [Data] "ir" (Data));
 547 }
 548
 549 static __inline__ __attribute__((always_inline)) void
 550 __writefsdword(const unsigned long Offset, const uint32_t Data)
 551 {
 552   __asm__ ("movl %[Data], %%fs:%a[Offset]"
 553            :
 554            : [Offset] "ir" (Offset), [Data] "ir" (Data));
 555 }
 556
 557 static __inline__ __attribute__((always_inline)) uint8_t
 558 __readfsbyte(const unsigned long Offset)
 559 {
 560   unsigned char value;
 561   __asm__ ("movb %%fs:%a[Offset], %[value]"
 562           : [value] "=q" (value)
 563           : [Offset] "ir" (Offset));
 564   return value;
 565 }
 566
 567 static __inline__ __attribute__((always_inline)) uint16_t
 568 __readfsword(const unsigned long Offset)
 569 {
 570   unsigned short value;
 571   __asm__ ("movw %%fs:%a[Offset], %[value]"
 572            : [value] "=q" (value)
 573            : [Offset] "ir" (Offset));
 574   return value;
 575 }
 576
 577 static __inline__ __attribute__((always_inline)) uint32_t
 578 __readfsdword(unsigned long Offset)
 579 {
 580   unsigned long value;
 581   __asm__ ("mov %%fs:%a[Offset], %[value]"
 582            : [value] "=r" (value)
 583            : [Offset] "ir" (Offset));
 584   return value;
 585 }
 586
 587 #ifdef __SSE3__
 588 typedef long long __v2di __attribute__((__vector_size__ (16)));
 589 typedef long long __m128i __attribute__((__vector_size__ (16), __may_alias__));
 590
 591 static __inline__ __attribute__((always_inline)) void _mm_stream_si128_fs2 (void *__A, __m128i __B)
 592 {
 593   __asm__(".byte 0x64"); // fs prefix
 594 #if defined(__clang__)
 595   __builtin_nontemporal_store((__v2di)__B, (__v2di *)__A);
 596 #else
 597   __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B);
 598 #endif
 599 }
 600
 601 static __inline__ __attribute__((always_inline)) void _mm_stream_si128_fs (void *__A, void *__B)
 602 {
 603   __m128i data;
 604   data = (__m128i) __builtin_ia32_lddqu ((char const *)__B);
 605   _mm_stream_si128_fs2 (__A, data);
 606 }
 607
 608 static __inline__ __attribute__((always_inline)) void _mm_clflush_fs (void *__A)
 609 {
 610   __asm__(".byte 0x64"); // fs prefix
 611   __builtin_ia32_clflush (__A);
 612 }
 613
 614 #if !defined(__clang__)
 615 static __inline __attribute__(( __always_inline__)) void _mm_mfence (void)
 616 {
 617   __builtin_ia32_mfence ();
 618 }
 619 #else
 620 void _mm_mfence(void);
 621 #endif
 622
 623 #if !defined(__clang__)
 624 static __inline __attribute__(( __always_inline__)) void _mm_sfence (void)
 625 {
 626   __builtin_ia32_sfence ();
 627 }
 628 #else
 629 void _mm_sfence(void);
 630 #endif
 631 #endif /* __SSE3__ */
 632
 633 #endif /* defined (__GNUC__) */