include/asm-alpha/system.h

   1 #ifndef __ALPHA_SYSTEM_H
   2 #define __ALPHA_SYSTEM_H
   3
   4 #include <asm/pal.h>
   5 #include <asm/page.h>
   6 #include <asm/barrier.h>
   7
   8 /*
   9  * System defines.. Note that this is included both from .c and .S
  10  * files, so it does only defines, not any C code.
  11  */
  12
  13 /*
  14  * We leave one page for the initial stack page, and one page for
  15  * the initial process structure. Also, the console eats 3 MB for
  16  * the initial bootloader (one of which we can reclaim later).
  17  */
  18 #define BOOT_PCB        0x20000000
  19 #define BOOT_ADDR       0x20000000
  20 /* Remove when official MILO sources have ELF support: */
  21 #define BOOT_SIZE       (16*1024)
  22
  23 #ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS
  24 #define KERNEL_START_PHYS       0x300000 /* Old bootloaders hardcoded this.  */
  25 #else
  26 #define KERNEL_START_PHYS       0x1000000 /* required: Wildfire/Titan/Marvel */
  27 #endif
  28
  29 #define KERNEL_START    (PAGE_OFFSET+KERNEL_START_PHYS)
  30 #define SWAPPER_PGD     KERNEL_START
  31 #define INIT_STACK      (PAGE_OFFSET+KERNEL_START_PHYS+0x02000)
  32 #define EMPTY_PGT       (PAGE_OFFSET+KERNEL_START_PHYS+0x04000)
  33 #define EMPTY_PGE       (PAGE_OFFSET+KERNEL_START_PHYS+0x08000)
  34 #define ZERO_PGE        (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000)
  35
  36 #define START_ADDR      (PAGE_OFFSET+KERNEL_START_PHYS+0x10000)
  37
  38 /*
  39  * This is setup by the secondary bootstrap loader.  Because
  40  * the zero page is zeroed out as soon as the vm system is
  41  * initialized, we need to copy things out into a more permanent
  42  * place.
  43  */
  44 #define PARAM                   ZERO_PGE
  45 #define COMMAND_LINE            ((char*)(PARAM + 0x0000))
  46 #define INITRD_START            (*(unsigned long *) (PARAM+0x100))
  47 #define INITRD_SIZE             (*(unsigned long *) (PARAM+0x108))
  48
  49 #ifndef __ASSEMBLY__
  50 #include <linux/kernel.h>
  51 #define AT_VECTOR_SIZE_ARCH 4 /* entries in ARCH_DLINFO */
  52
  53 /*
  54  * This is the logout header that should be common to all platforms
  55  * (assuming they are running OSF/1 PALcode, I guess).
  56  */
  57 struct el_common {
  58         unsigned int    size;           /* size in bytes of logout area */
  59         unsigned int    sbz1    : 30;   /* should be zero */
  60         unsigned int    err2    :  1;   /* second error */
  61         unsigned int    retry   :  1;   /* retry flag */
  62         unsigned int    proc_offset;    /* processor-specific offset */
  63         unsigned int    sys_offset;     /* system-specific offset */
  64         unsigned int    code;           /* machine check code */
  65         unsigned int    frame_rev;      /* frame revision */
  66 };
  67
  68 /* Machine Check Frame for uncorrectable errors (Large format)
  69  *      --- This is used to log uncorrectable errors such as
  70  *          double bit ECC errors.
  71  *      --- These errors are detected by both processor and systems.
  72  */
  73 struct el_common_EV5_uncorrectable_mcheck {
  74         unsigned long   shadow[8];        /* Shadow reg. 8-14, 25           */
  75         unsigned long   paltemp[24];      /* PAL TEMP REGS.                 */
  76         unsigned long   exc_addr;         /* Address of excepting instruction*/
  77         unsigned long   exc_sum;          /* Summary of arithmetic traps.   */
  78         unsigned long   exc_mask;         /* Exception mask (from exc_sum). */
  79         unsigned long   pal_base;         /* Base address for PALcode.      */
  80         unsigned long   isr;              /* Interrupt Status Reg.          */
  81         unsigned long   icsr;             /* CURRENT SETUP OF EV5 IBOX      */
  82         unsigned long   ic_perr_stat;     /* I-CACHE Reg. <11> set Data parity
  83                                                          <12> set TAG parity*/
  84         unsigned long   dc_perr_stat;     /* D-CACHE error Reg. Bits set to 1:
  85                                                      <2> Data error in bank 0
  86                                                      <3> Data error in bank 1
  87                                                      <4> Tag error in bank 0
  88                                                      <5> Tag error in bank 1 */
  89         unsigned long   va;               /* Effective VA of fault or miss. */
  90         unsigned long   mm_stat;          /* Holds the reason for D-stream
  91                                              fault or D-cache parity errors */
  92         unsigned long   sc_addr;          /* Address that was being accessed
  93                                              when EV5 detected Secondary cache
  94                                              failure.                 */
  95         unsigned long   sc_stat;          /* Helps determine if the error was
  96                                              TAG/Data parity(Secondary Cache)*/
  97         unsigned long   bc_tag_addr;      /* Contents of EV5 BC_TAG_ADDR    */
  98         unsigned long   ei_addr;          /* Physical address of any transfer
  99                                              that is logged in EV5 EI_STAT */
 100         unsigned long   fill_syndrome;    /* For correcting ECC errors.     */
 101         unsigned long   ei_stat;          /* Helps identify reason of any
 102                                              processor uncorrectable error
 103                                              at its external interface.     */
 104         unsigned long   ld_lock;          /* Contents of EV5 LD_LOCK register*/
 105 };
 106
 107 struct el_common_EV6_mcheck {
 108         unsigned int FrameSize;         /* Bytes, including this field */
 109         unsigned int FrameFlags;        /* <31> = Retry, <30> = Second Error */
 110         unsigned int CpuOffset;         /* Offset to CPU-specific info */
 111         unsigned int SystemOffset;      /* Offset to system-specific info */
 112         unsigned int MCHK_Code;
 113         unsigned int MCHK_Frame_Rev;
 114         unsigned long I_STAT;           /* EV6 Internal Processor Registers */
 115         unsigned long DC_STAT;          /* (See the 21264 Spec) */
 116         unsigned long C_ADDR;
 117         unsigned long DC1_SYNDROME;
 118         unsigned long DC0_SYNDROME;
 119         unsigned long C_STAT;
 120         unsigned long C_STS;
 121         unsigned long MM_STAT;
 122         unsigned long EXC_ADDR;
 123         unsigned long IER_CM;
 124         unsigned long ISUM;
 125         unsigned long RESERVED0;
 126         unsigned long PAL_BASE;
 127         unsigned long I_CTL;
 128         unsigned long PCTX;
 129 };
 130
 131 extern void halt(void) __attribute__((noreturn));
 132 #define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt))
 133
 134 #define switch_to(P,N,L)                                                 \
 135   do {                                                                   \
 136     (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \
 137     check_mmu_context();                                                 \
 138   } while (0)
 139
 140 struct task_struct;
 141 extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*);
 142
 143 #define imb() \
 144 __asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory")
 145
 146 #define draina() \
 147 __asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory")
 148
 149 enum implver_enum {
 150         IMPLVER_EV4,
 151         IMPLVER_EV5,
 152         IMPLVER_EV6
 153 };
 154
 155 #ifdef CONFIG_ALPHA_GENERIC
 156 #define implver()                               \
 157 ({ unsigned long __implver;                     \
 158    __asm__ ("implver %0" : "=r"(__implver));    \
 159    (enum implver_enum) __implver; })
 160 #else
 161 /* Try to eliminate some dead code.  */
 162 #ifdef CONFIG_ALPHA_EV4
 163 #define implver() IMPLVER_EV4
 164 #endif
 165 #ifdef CONFIG_ALPHA_EV5
 166 #define implver() IMPLVER_EV5
 167 #endif
 168 #if defined(CONFIG_ALPHA_EV6)
 169 #define implver() IMPLVER_EV6
 170 #endif
 171 #endif
 172
 173 enum amask_enum {
 174         AMASK_BWX = (1UL << 0),
 175         AMASK_FIX = (1UL << 1),
 176         AMASK_CIX = (1UL << 2),
 177         AMASK_MAX = (1UL << 8),
 178         AMASK_PRECISE_TRAP = (1UL << 9),
 179 };
 180
 181 #define amask(mask)                                             \
 182 ({ unsigned long __amask, __input = (mask);                     \
 183    __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input));     \
 184    __amask; })
 185
 186 #define __CALL_PAL_R0(NAME, TYPE)                               \
 187 static inline TYPE NAME(void)                                   \
 188 {                                                               \
 189         register TYPE __r0 __asm__("$0");                       \
 190         __asm__ __volatile__(                                   \
 191                 "call_pal %1 # " #NAME                          \
 192                 :"=r" (__r0)                                    \
 193                 :"i" (PAL_ ## NAME)                             \
 194                 :"$1", "$16", "$22", "$23", "$24", "$25");      \
 195         return __r0;                                            \
 196 }
 197
 198 #define __CALL_PAL_W1(NAME, TYPE0)                              \
 199 static inline void NAME(TYPE0 arg0)                             \
 200 {                                                               \
 201         register TYPE0 __r16 __asm__("$16") = arg0;             \
 202         __asm__ __volatile__(                                   \
 203                 "call_pal %1 # "#NAME                           \
 204                 : "=r"(__r16)                                   \
 205                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
 206                 : "$1", "$22", "$23", "$24", "$25");            \
 207 }
 208
 209 #define __CALL_PAL_W2(NAME, TYPE0, TYPE1)                       \
 210 static inline void NAME(TYPE0 arg0, TYPE1 arg1)                 \
 211 {                                                               \
 212         register TYPE0 __r16 __asm__("$16") = arg0;             \
 213         register TYPE1 __r17 __asm__("$17") = arg1;             \
 214         __asm__ __volatile__(                                   \
 215                 "call_pal %2 # "#NAME                           \
 216                 : "=r"(__r16), "=r"(__r17)                      \
 217                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
 218                 : "$1", "$22", "$23", "$24", "$25");            \
 219 }
 220
 221 #define __CALL_PAL_RW1(NAME, RTYPE, TYPE0)                      \
 222 static inline RTYPE NAME(TYPE0 arg0)                            \
 223 {                                                               \
 224         register RTYPE __r0 __asm__("$0");                      \
 225         register TYPE0 __r16 __asm__("$16") = arg0;             \
 226         __asm__ __volatile__(                                   \
 227                 "call_pal %2 # "#NAME                           \
 228                 : "=r"(__r16), "=r"(__r0)                       \
 229                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
 230                 : "$1", "$22", "$23", "$24", "$25");            \
 231         return __r0;                                            \
 232 }
 233
 234 #define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1)               \
 235 static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1)                \
 236 {                                                               \
 237         register RTYPE __r0 __asm__("$0");                      \
 238         register TYPE0 __r16 __asm__("$16") = arg0;             \
 239         register TYPE1 __r17 __asm__("$17") = arg1;             \
 240         __asm__ __volatile__(                                   \
 241                 "call_pal %3 # "#NAME                           \
 242                 : "=r"(__r16), "=r"(__r17), "=r"(__r0)          \
 243                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
 244                 : "$1", "$22", "$23", "$24", "$25");            \
 245         return __r0;                                            \
 246 }
 247
 248 __CALL_PAL_W1(cflush, unsigned long);
 249 __CALL_PAL_R0(rdmces, unsigned long);
 250 __CALL_PAL_R0(rdps, unsigned long);
 251 __CALL_PAL_R0(rdusp, unsigned long);
 252 __CALL_PAL_RW1(swpipl, unsigned long, unsigned long);
 253 __CALL_PAL_R0(whami, unsigned long);
 254 __CALL_PAL_W2(wrent, void*, unsigned long);
 255 __CALL_PAL_W1(wripir, unsigned long);
 256 __CALL_PAL_W1(wrkgp, unsigned long);
 257 __CALL_PAL_W1(wrmces, unsigned long);
 258 __CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long);
 259 __CALL_PAL_W1(wrusp, unsigned long);
 260 __CALL_PAL_W1(wrvptptr, unsigned long);
 261
 262 #define IPL_MIN         0
 263 #define IPL_SW0         1
 264 #define IPL_SW1         2
 265 #define IPL_DEV0        3
 266 #define IPL_DEV1        4
 267 #define IPL_TIMER       5
 268 #define IPL_PERF        6
 269 #define IPL_POWERFAIL   6
 270 #define IPL_MCHECK      7
 271 #define IPL_MAX         7
 272
 273 #ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK
 274 #undef IPL_MIN
 275 #define IPL_MIN         __min_ipl
 276 extern int __min_ipl;
 277 #endif
 278
 279 #define getipl()                (rdps() & 7)
 280 #define setipl(ipl)             ((void) swpipl(ipl))
 281
 282 #define local_irq_disable()                     do { setipl(IPL_MAX); barrier(); } while(0)
 283 #define local_irq_enable()                      do { barrier(); setipl(IPL_MIN); } while(0)
 284 #define local_save_flags(flags) ((flags) = rdps())
 285 #define local_irq_save(flags)   do { (flags) = swpipl(IPL_MAX); barrier(); } while(0)
 286 #define local_irq_restore(flags)        do { barrier(); setipl(flags); barrier(); } while(0)
 287
 288 #define irqs_disabled() (getipl() == IPL_MAX)
 289
 290 /*
 291  * TB routines..
 292  */
 293 #define __tbi(nr,arg,arg1...)                                   \
 294 ({                                                              \
 295         register unsigned long __r16 __asm__("$16") = (nr);     \
 296         register unsigned long __r17 __asm__("$17"); arg;       \
 297         __asm__ __volatile__(                                   \
 298                 "call_pal %3 #__tbi"                            \
 299                 :"=r" (__r16),"=r" (__r17)                      \
 300                 :"0" (__r16),"i" (PAL_tbi) ,##arg1              \
 301                 :"$0", "$1", "$22", "$23", "$24", "$25");       \
 302 })
 303
 304 #define tbi(x,y)        __tbi(x,__r17=(y),"1" (__r17))
 305 #define tbisi(x)        __tbi(1,__r17=(x),"1" (__r17))
 306 #define tbisd(x)        __tbi(2,__r17=(x),"1" (__r17))
 307 #define tbis(x)         __tbi(3,__r17=(x),"1" (__r17))
 308 #define tbiap()         __tbi(-1, /* no second argument */)
 309 #define tbia()          __tbi(-2, /* no second argument */)
 310
 311 /*
 312  * Atomic exchange.
 313  * Since it can be used to implement critical sections
 314  * it must clobber "memory" (also for interrupts in UP).
 315  */
 316
 317 static inline unsigned long
 318 __xchg_u8(volatile char *m, unsigned long val)
 319 {
 320         unsigned long ret, tmp, addr64;
 321
 322         __asm__ __volatile__(
 323         "       andnot  %4,7,%3\n"
 324         "       insbl   %1,%4,%1\n"
 325         "1:     ldq_l   %2,0(%3)\n"
 326         "       extbl   %2,%4,%0\n"
 327         "       mskbl   %2,%4,%2\n"
 328         "       or      %1,%2,%2\n"
 329         "       stq_c   %2,0(%3)\n"
 330         "       beq     %2,2f\n"
 331 #ifdef CONFIG_SMP
 332         "       mb\n"
 333 #endif
 334         ".subsection 2\n"
 335         "2:     br      1b\n"
 336         ".previous"
 337         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 338         : "r" ((long)m), "1" (val) : "memory");
 339
 340         return ret;
 341 }
 342
 343 static inline unsigned long
 344 __xchg_u16(volatile short *m, unsigned long val)
 345 {
 346         unsigned long ret, tmp, addr64;
 347
 348         __asm__ __volatile__(
 349         "       andnot  %4,7,%3\n"
 350         "       inswl   %1,%4,%1\n"
 351         "1:     ldq_l   %2,0(%3)\n"
 352         "       extwl   %2,%4,%0\n"
 353         "       mskwl   %2,%4,%2\n"
 354         "       or      %1,%2,%2\n"
 355         "       stq_c   %2,0(%3)\n"
 356         "       beq     %2,2f\n"
 357 #ifdef CONFIG_SMP
 358         "       mb\n"
 359 #endif
 360         ".subsection 2\n"
 361         "2:     br      1b\n"
 362         ".previous"
 363         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 364         : "r" ((long)m), "1" (val) : "memory");
 365
 366         return ret;
 367 }
 368
 369 static inline unsigned long
 370 __xchg_u32(volatile int *m, unsigned long val)
 371 {
 372         unsigned long dummy;
 373
 374         __asm__ __volatile__(
 375         "1:     ldl_l %0,%4\n"
 376         "       bis $31,%3,%1\n"
 377         "       stl_c %1,%2\n"
 378         "       beq %1,2f\n"
 379 #ifdef CONFIG_SMP
 380         "       mb\n"
 381 #endif
 382         ".subsection 2\n"
 383         "2:     br 1b\n"
 384         ".previous"
 385         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 386         : "rI" (val), "m" (*m) : "memory");
 387
 388         return val;
 389 }
 390
 391 static inline unsigned long
 392 __xchg_u64(volatile long *m, unsigned long val)
 393 {
 394         unsigned long dummy;
 395
 396         __asm__ __volatile__(
 397         "1:     ldq_l %0,%4\n"
 398         "       bis $31,%3,%1\n"
 399         "       stq_c %1,%2\n"
 400         "       beq %1,2f\n"
 401 #ifdef CONFIG_SMP
 402         "       mb\n"
 403 #endif
 404         ".subsection 2\n"
 405         "2:     br 1b\n"
 406         ".previous"
 407         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 408         : "rI" (val), "m" (*m) : "memory");
 409
 410         return val;
 411 }
 412
 413 /* This function doesn't exist, so you'll get a linker error
 414    if something tries to do an invalid xchg().  */
 415 extern void __xchg_called_with_bad_pointer(void);
 416
 417 #define __xchg(ptr, x, size) \
 418 ({ \
 419         unsigned long __xchg__res; \
 420         volatile void *__xchg__ptr = (ptr); \
 421         switch (size) { \
 422                 case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \
 423                 case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \
 424                 case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \
 425                 case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \
 426                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
 427         } \
 428         __xchg__res; \
 429 })
 430
 431 #define xchg(ptr,x)                                                          \
 432   ({                                                                         \
 433      __typeof__(*(ptr)) _x_ = (x);                                           \
 434      (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
 435   })
 436
 437 static inline unsigned long
 438 __xchg_u8_local(volatile char *m, unsigned long val)
 439 {
 440         unsigned long ret, tmp, addr64;
 441
 442         __asm__ __volatile__(
 443         "       andnot  %4,7,%3\n"
 444         "       insbl   %1,%4,%1\n"
 445         "1:     ldq_l   %2,0(%3)\n"
 446         "       extbl   %2,%4,%0\n"
 447         "       mskbl   %2,%4,%2\n"
 448         "       or      %1,%2,%2\n"
 449         "       stq_c   %2,0(%3)\n"
 450         "       beq     %2,2f\n"
 451         ".subsection 2\n"
 452         "2:     br      1b\n"
 453         ".previous"
 454         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 455         : "r" ((long)m), "1" (val) : "memory");
 456
 457         return ret;
 458 }
 459
 460 static inline unsigned long
 461 __xchg_u16_local(volatile short *m, unsigned long val)
 462 {
 463         unsigned long ret, tmp, addr64;
 464
 465         __asm__ __volatile__(
 466         "       andnot  %4,7,%3\n"
 467         "       inswl   %1,%4,%1\n"
 468         "1:     ldq_l   %2,0(%3)\n"
 469         "       extwl   %2,%4,%0\n"
 470         "       mskwl   %2,%4,%2\n"
 471         "       or      %1,%2,%2\n"
 472         "       stq_c   %2,0(%3)\n"
 473         "       beq     %2,2f\n"
 474         ".subsection 2\n"
 475         "2:     br      1b\n"
 476         ".previous"
 477         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 478         : "r" ((long)m), "1" (val) : "memory");
 479
 480         return ret;
 481 }
 482
 483 static inline unsigned long
 484 __xchg_u32_local(volatile int *m, unsigned long val)
 485 {
 486         unsigned long dummy;
 487
 488         __asm__ __volatile__(
 489         "1:     ldl_l %0,%4\n"
 490         "       bis $31,%3,%1\n"
 491         "       stl_c %1,%2\n"
 492         "       beq %1,2f\n"
 493         ".subsection 2\n"
 494         "2:     br 1b\n"
 495         ".previous"
 496         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 497         : "rI" (val), "m" (*m) : "memory");
 498
 499         return val;
 500 }
 501
 502 static inline unsigned long
 503 __xchg_u64_local(volatile long *m, unsigned long val)
 504 {
 505         unsigned long dummy;
 506
 507         __asm__ __volatile__(
 508         "1:     ldq_l %0,%4\n"
 509         "       bis $31,%3,%1\n"
 510         "       stq_c %1,%2\n"
 511         "       beq %1,2f\n"
 512         ".subsection 2\n"
 513         "2:     br 1b\n"
 514         ".previous"
 515         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 516         : "rI" (val), "m" (*m) : "memory");
 517
 518         return val;
 519 }
 520
 521 #define __xchg_local(ptr, x, size) \
 522 ({ \
 523         unsigned long __xchg__res; \
 524         volatile void *__xchg__ptr = (ptr); \
 525         switch (size) { \
 526                 case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \
 527                 case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \
 528                 case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \
 529                 case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \
 530                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
 531         } \
 532         __xchg__res; \
 533 })
 534
 535 #define xchg_local(ptr,x)                                                    \
 536   ({                                                                         \
 537      __typeof__(*(ptr)) _x_ = (x);                                           \
 538      (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_,            \
 539                 sizeof(*(ptr))); \
 540   })
 541
 542 /*
 543  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 544  * store NEW in MEM.  Return the initial value in MEM.  Success is
 545  * indicated by comparing RETURN with OLD.
 546  *
 547  * The memory barrier should be placed in SMP only when we actually
 548  * make the change. If we don't change anything (so if the returned
 549  * prev is equal to old) then we aren't acquiring anything new and
 550  * we don't need any memory barrier as far I can tell.
 551  */
 552
 553 #define __HAVE_ARCH_CMPXCHG 1
 554
 555 static inline unsigned long
 556 __cmpxchg_u8(volatile char *m, long old, long new)
 557 {
 558         unsigned long prev, tmp, cmp, addr64;
 559
 560         __asm__ __volatile__(
 561         "       andnot  %5,7,%4\n"
 562         "       insbl   %1,%5,%1\n"
 563         "1:     ldq_l   %2,0(%4)\n"
 564         "       extbl   %2,%5,%0\n"
 565         "       cmpeq   %0,%6,%3\n"
 566         "       beq     %3,2f\n"
 567         "       mskbl   %2,%5,%2\n"
 568         "       or      %1,%2,%2\n"
 569         "       stq_c   %2,0(%4)\n"
 570         "       beq     %2,3f\n"
 571 #ifdef CONFIG_SMP
 572         "       mb\n"
 573 #endif
 574         "2:\n"
 575         ".subsection 2\n"
 576         "3:     br      1b\n"
 577         ".previous"
 578         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 579         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 580
 581         return prev;
 582 }
 583
 584 static inline unsigned long
 585 __cmpxchg_u16(volatile short *m, long old, long new)
 586 {
 587         unsigned long prev, tmp, cmp, addr64;
 588
 589         __asm__ __volatile__(
 590         "       andnot  %5,7,%4\n"
 591         "       inswl   %1,%5,%1\n"
 592         "1:     ldq_l   %2,0(%4)\n"
 593         "       extwl   %2,%5,%0\n"
 594         "       cmpeq   %0,%6,%3\n"
 595         "       beq     %3,2f\n"
 596         "       mskwl   %2,%5,%2\n"
 597         "       or      %1,%2,%2\n"
 598         "       stq_c   %2,0(%4)\n"
 599         "       beq     %2,3f\n"
 600 #ifdef CONFIG_SMP
 601         "       mb\n"
 602 #endif
 603         "2:\n"
 604         ".subsection 2\n"
 605         "3:     br      1b\n"
 606         ".previous"
 607         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 608         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 609
 610         return prev;
 611 }
 612
 613 static inline unsigned long
 614 __cmpxchg_u32(volatile int *m, int old, int new)
 615 {
 616         unsigned long prev, cmp;
 617
 618         __asm__ __volatile__(
 619         "1:     ldl_l %0,%5\n"
 620         "       cmpeq %0,%3,%1\n"
 621         "       beq %1,2f\n"
 622         "       mov %4,%1\n"
 623         "       stl_c %1,%2\n"
 624         "       beq %1,3f\n"
 625 #ifdef CONFIG_SMP
 626         "       mb\n"
 627 #endif
 628         "2:\n"
 629         ".subsection 2\n"
 630         "3:     br 1b\n"
 631         ".previous"
 632         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 633         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 634
 635         return prev;
 636 }
 637
 638 static inline unsigned long
 639 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
 640 {
 641         unsigned long prev, cmp;
 642
 643         __asm__ __volatile__(
 644         "1:     ldq_l %0,%5\n"
 645         "       cmpeq %0,%3,%1\n"
 646         "       beq %1,2f\n"
 647         "       mov %4,%1\n"
 648         "       stq_c %1,%2\n"
 649         "       beq %1,3f\n"
 650 #ifdef CONFIG_SMP
 651         "       mb\n"
 652 #endif
 653         "2:\n"
 654         ".subsection 2\n"
 655         "3:     br 1b\n"
 656         ".previous"
 657         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 658         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 659
 660         return prev;
 661 }
 662
 663 /* This function doesn't exist, so you'll get a linker error
 664    if something tries to do an invalid cmpxchg().  */
 665 extern void __cmpxchg_called_with_bad_pointer(void);
 666
 667 static __always_inline unsigned long
 668 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
 669 {
 670         switch (size) {
 671                 case 1:
 672                         return __cmpxchg_u8(ptr, old, new);
 673                 case 2:
 674                         return __cmpxchg_u16(ptr, old, new);
 675                 case 4:
 676                         return __cmpxchg_u32(ptr, old, new);
 677                 case 8:
 678                         return __cmpxchg_u64(ptr, old, new);
 679         }
 680         __cmpxchg_called_with_bad_pointer();
 681         return old;
 682 }
 683
 684 #define cmpxchg(ptr, o, n)                                               \
 685   ({                                                                     \
 686      __typeof__(*(ptr)) _o_ = (o);                                       \
 687      __typeof__(*(ptr)) _n_ = (n);                                       \
 688      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 689                                     (unsigned long)_n_, sizeof(*(ptr))); \
 690   })
 691 #define cmpxchg64(ptr, o, n)                                             \
 692   ({                                                                     \
 693         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                               \
 694         cmpxchg((ptr), (o), (n));                                        \
 695   })
 696
 697 static inline unsigned long
 698 __cmpxchg_u8_local(volatile char *m, long old, long new)
 699 {
 700         unsigned long prev, tmp, cmp, addr64;
 701
 702         __asm__ __volatile__(
 703         "       andnot  %5,7,%4\n"
 704         "       insbl   %1,%5,%1\n"
 705         "1:     ldq_l   %2,0(%4)\n"
 706         "       extbl   %2,%5,%0\n"
 707         "       cmpeq   %0,%6,%3\n"
 708         "       beq     %3,2f\n"
 709         "       mskbl   %2,%5,%2\n"
 710         "       or      %1,%2,%2\n"
 711         "       stq_c   %2,0(%4)\n"
 712         "       beq     %2,3f\n"
 713         "2:\n"
 714         ".subsection 2\n"
 715         "3:     br      1b\n"
 716         ".previous"
 717         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 718         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 719
 720         return prev;
 721 }
 722
 723 static inline unsigned long
 724 __cmpxchg_u16_local(volatile short *m, long old, long new)
 725 {
 726         unsigned long prev, tmp, cmp, addr64;
 727
 728         __asm__ __volatile__(
 729         "       andnot  %5,7,%4\n"
 730         "       inswl   %1,%5,%1\n"
 731         "1:     ldq_l   %2,0(%4)\n"
 732         "       extwl   %2,%5,%0\n"
 733         "       cmpeq   %0,%6,%3\n"
 734         "       beq     %3,2f\n"
 735         "       mskwl   %2,%5,%2\n"
 736         "       or      %1,%2,%2\n"
 737         "       stq_c   %2,0(%4)\n"
 738         "       beq     %2,3f\n"
 739         "2:\n"
 740         ".subsection 2\n"
 741         "3:     br      1b\n"
 742         ".previous"
 743         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 744         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 745
 746         return prev;
 747 }
 748
 749 static inline unsigned long
 750 __cmpxchg_u32_local(volatile int *m, int old, int new)
 751 {
 752         unsigned long prev, cmp;
 753
 754         __asm__ __volatile__(
 755         "1:     ldl_l %0,%5\n"
 756         "       cmpeq %0,%3,%1\n"
 757         "       beq %1,2f\n"
 758         "       mov %4,%1\n"
 759         "       stl_c %1,%2\n"
 760         "       beq %1,3f\n"
 761         "2:\n"
 762         ".subsection 2\n"
 763         "3:     br 1b\n"
 764         ".previous"
 765         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 766         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 767
 768         return prev;
 769 }
 770
 771 static inline unsigned long
 772 __cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new)
 773 {
 774         unsigned long prev, cmp;
 775
 776         __asm__ __volatile__(
 777         "1:     ldq_l %0,%5\n"
 778         "       cmpeq %0,%3,%1\n"
 779         "       beq %1,2f\n"
 780         "       mov %4,%1\n"
 781         "       stq_c %1,%2\n"
 782         "       beq %1,3f\n"
 783         "2:\n"
 784         ".subsection 2\n"
 785         "3:     br 1b\n"
 786         ".previous"
 787         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 788         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 789
 790         return prev;
 791 }
 792
 793 static __always_inline unsigned long
 794 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
 795                 int size)
 796 {
 797         switch (size) {
 798                 case 1:
 799                         return __cmpxchg_u8_local(ptr, old, new);
 800                 case 2:
 801                         return __cmpxchg_u16_local(ptr, old, new);
 802                 case 4:
 803                         return __cmpxchg_u32_local(ptr, old, new);
 804                 case 8:
 805                         return __cmpxchg_u64_local(ptr, old, new);
 806         }
 807         __cmpxchg_called_with_bad_pointer();
 808         return old;
 809 }
 810
 811 #define cmpxchg_local(ptr, o, n)                                         \
 812   ({                                                                     \
 813      __typeof__(*(ptr)) _o_ = (o);                                       \
 814      __typeof__(*(ptr)) _n_ = (n);                                       \
 815      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 816                                     (unsigned long)_n_, sizeof(*(ptr))); \
 817   })
 818 #define cmpxchg64_local(ptr, o, n)                                       \
 819   ({                                                                     \
 820         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                               \
 821         cmpxchg_local((ptr), (o), (n));                                  \
 822   })
 823
 824
 825 #endif /* __ASSEMBLY__ */
 826
 827 #define arch_align_stack(x) (x)
 828
 829 #endif