include/asm-alpha/system.h

   1 #ifndef __ALPHA_SYSTEM_H
   2 #define __ALPHA_SYSTEM_H
   3
   4 #include <asm/pal.h>
   5 #include <asm/page.h>
   6 #include <asm/barrier.h>
   7
   8 /*
   9  * System defines.. Note that this is included both from .c and .S
  10  * files, so it does only defines, not any C code.
  11  */
  12
  13 /*
  14  * We leave one page for the initial stack page, and one page for
  15  * the initial process structure. Also, the console eats 3 MB for
  16  * the initial bootloader (one of which we can reclaim later).
  17  */
  18 #define BOOT_PCB        0x20000000
  19 #define BOOT_ADDR       0x20000000
  20 /* Remove when official MILO sources have ELF support: */
  21 #define BOOT_SIZE       (16*1024)
  22
  23 #ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS
  24 #define KERNEL_START_PHYS       0x300000 /* Old bootloaders hardcoded this.  */
  25 #else
  26 #define KERNEL_START_PHYS       0x1000000 /* required: Wildfire/Titan/Marvel */
  27 #endif
  28
  29 #define KERNEL_START    (PAGE_OFFSET+KERNEL_START_PHYS)
  30 #define SWAPPER_PGD     KERNEL_START
  31 #define INIT_STACK      (PAGE_OFFSET+KERNEL_START_PHYS+0x02000)
  32 #define EMPTY_PGT       (PAGE_OFFSET+KERNEL_START_PHYS+0x04000)
  33 #define EMPTY_PGE       (PAGE_OFFSET+KERNEL_START_PHYS+0x08000)
  34 #define ZERO_PGE        (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000)
  35
  36 #define START_ADDR      (PAGE_OFFSET+KERNEL_START_PHYS+0x10000)
  37
  38 /*
  39  * This is setup by the secondary bootstrap loader.  Because
  40  * the zero page is zeroed out as soon as the vm system is
  41  * initialized, we need to copy things out into a more permanent
  42  * place.
  43  */
  44 #define PARAM                   ZERO_PGE
  45 #define COMMAND_LINE            ((char*)(PARAM + 0x0000))
  46 #define INITRD_START            (*(unsigned long *) (PARAM+0x100))
  47 #define INITRD_SIZE             (*(unsigned long *) (PARAM+0x108))
  48
  49 #ifndef __ASSEMBLY__
  50 #include <linux/kernel.h>
  51
  52 /*
  53  * This is the logout header that should be common to all platforms
  54  * (assuming they are running OSF/1 PALcode, I guess).
  55  */
  56 struct el_common {
  57         unsigned int    size;           /* size in bytes of logout area */
  58         unsigned int    sbz1    : 30;   /* should be zero */
  59         unsigned int    err2    :  1;   /* second error */
  60         unsigned int    retry   :  1;   /* retry flag */
  61         unsigned int    proc_offset;    /* processor-specific offset */
  62         unsigned int    sys_offset;     /* system-specific offset */
  63         unsigned int    code;           /* machine check code */
  64         unsigned int    frame_rev;      /* frame revision */
  65 };
  66
  67 /* Machine Check Frame for uncorrectable errors (Large format)
  68  *      --- This is used to log uncorrectable errors such as
  69  *          double bit ECC errors.
  70  *      --- These errors are detected by both processor and systems.
  71  */
  72 struct el_common_EV5_uncorrectable_mcheck {
  73         unsigned long   shadow[8];        /* Shadow reg. 8-14, 25           */
  74         unsigned long   paltemp[24];      /* PAL TEMP REGS.                 */
  75         unsigned long   exc_addr;         /* Address of excepting instruction*/
  76         unsigned long   exc_sum;          /* Summary of arithmetic traps.   */
  77         unsigned long   exc_mask;         /* Exception mask (from exc_sum). */
  78         unsigned long   pal_base;         /* Base address for PALcode.      */
  79         unsigned long   isr;              /* Interrupt Status Reg.          */
  80         unsigned long   icsr;             /* CURRENT SETUP OF EV5 IBOX      */
  81         unsigned long   ic_perr_stat;     /* I-CACHE Reg. <11> set Data parity
  82                                                          <12> set TAG parity*/
  83         unsigned long   dc_perr_stat;     /* D-CACHE error Reg. Bits set to 1:
  84                                                      <2> Data error in bank 0
  85                                                      <3> Data error in bank 1
  86                                                      <4> Tag error in bank 0
  87                                                      <5> Tag error in bank 1 */
  88         unsigned long   va;               /* Effective VA of fault or miss. */
  89         unsigned long   mm_stat;          /* Holds the reason for D-stream
  90                                              fault or D-cache parity errors */
  91         unsigned long   sc_addr;          /* Address that was being accessed
  92                                              when EV5 detected Secondary cache
  93                                              failure.                 */
  94         unsigned long   sc_stat;          /* Helps determine if the error was
  95                                              TAG/Data parity(Secondary Cache)*/
  96         unsigned long   bc_tag_addr;      /* Contents of EV5 BC_TAG_ADDR    */
  97         unsigned long   ei_addr;          /* Physical address of any transfer
  98                                              that is logged in EV5 EI_STAT */
  99         unsigned long   fill_syndrome;    /* For correcting ECC errors.     */
 100         unsigned long   ei_stat;          /* Helps identify reason of any
 101                                              processor uncorrectable error
 102                                              at its external interface.     */
 103         unsigned long   ld_lock;          /* Contents of EV5 LD_LOCK register*/
 104 };
 105
 106 struct el_common_EV6_mcheck {
 107         unsigned int FrameSize;         /* Bytes, including this field */
 108         unsigned int FrameFlags;        /* <31> = Retry, <30> = Second Error */
 109         unsigned int CpuOffset;         /* Offset to CPU-specific info */
 110         unsigned int SystemOffset;      /* Offset to system-specific info */
 111         unsigned int MCHK_Code;
 112         unsigned int MCHK_Frame_Rev;
 113         unsigned long I_STAT;           /* EV6 Internal Processor Registers */
 114         unsigned long DC_STAT;          /* (See the 21264 Spec) */
 115         unsigned long C_ADDR;
 116         unsigned long DC1_SYNDROME;
 117         unsigned long DC0_SYNDROME;
 118         unsigned long C_STAT;
 119         unsigned long C_STS;
 120         unsigned long MM_STAT;
 121         unsigned long EXC_ADDR;
 122         unsigned long IER_CM;
 123         unsigned long ISUM;
 124         unsigned long RESERVED0;
 125         unsigned long PAL_BASE;
 126         unsigned long I_CTL;
 127         unsigned long PCTX;
 128 };
 129
 130 extern void halt(void) __attribute__((noreturn));
 131 #define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt))
 132
 133 #define switch_to(P,N,L)                                                 \
 134   do {                                                                   \
 135     (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \
 136     check_mmu_context();                                                 \
 137   } while (0)
 138
 139 struct task_struct;
 140 extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*);
 141
 142 /*
 143  * On SMP systems, when the scheduler does migration-cost autodetection,
 144  * it needs a way to flush as much of the CPU's caches as possible.
 145  *
 146  * TODO: fill this in!
 147  */
 148 static inline void sched_cacheflush(void)
 149 {
 150 }
 151
 152 #define imb() \
 153 __asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory")
 154
 155 #define draina() \
 156 __asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory")
 157
 158 enum implver_enum {
 159         IMPLVER_EV4,
 160         IMPLVER_EV5,
 161         IMPLVER_EV6
 162 };
 163
 164 #ifdef CONFIG_ALPHA_GENERIC
 165 #define implver()                               \
 166 ({ unsigned long __implver;                     \
 167    __asm__ ("implver %0" : "=r"(__implver));    \
 168    (enum implver_enum) __implver; })
 169 #else
 170 /* Try to eliminate some dead code.  */
 171 #ifdef CONFIG_ALPHA_EV4
 172 #define implver() IMPLVER_EV4
 173 #endif
 174 #ifdef CONFIG_ALPHA_EV5
 175 #define implver() IMPLVER_EV5
 176 #endif
 177 #if defined(CONFIG_ALPHA_EV6)
 178 #define implver() IMPLVER_EV6
 179 #endif
 180 #endif
 181
 182 enum amask_enum {
 183         AMASK_BWX = (1UL << 0),
 184         AMASK_FIX = (1UL << 1),
 185         AMASK_CIX = (1UL << 2),
 186         AMASK_MAX = (1UL << 8),
 187         AMASK_PRECISE_TRAP = (1UL << 9),
 188 };
 189
 190 #define amask(mask)                                             \
 191 ({ unsigned long __amask, __input = (mask);                     \
 192    __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input));     \
 193    __amask; })
 194
 195 #define __CALL_PAL_R0(NAME, TYPE)                               \
 196 static inline TYPE NAME(void)                                   \
 197 {                                                               \
 198         register TYPE __r0 __asm__("$0");                       \
 199         __asm__ __volatile__(                                   \
 200                 "call_pal %1 # " #NAME                          \
 201                 :"=r" (__r0)                                    \
 202                 :"i" (PAL_ ## NAME)                             \
 203                 :"$1", "$16", "$22", "$23", "$24", "$25");      \
 204         return __r0;                                            \
 205 }
 206
 207 #define __CALL_PAL_W1(NAME, TYPE0)                              \
 208 static inline void NAME(TYPE0 arg0)                             \
 209 {                                                               \
 210         register TYPE0 __r16 __asm__("$16") = arg0;             \
 211         __asm__ __volatile__(                                   \
 212                 "call_pal %1 # "#NAME                           \
 213                 : "=r"(__r16)                                   \
 214                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
 215                 : "$1", "$22", "$23", "$24", "$25");            \
 216 }
 217
 218 #define __CALL_PAL_W2(NAME, TYPE0, TYPE1)                       \
 219 static inline void NAME(TYPE0 arg0, TYPE1 arg1)                 \
 220 {                                                               \
 221         register TYPE0 __r16 __asm__("$16") = arg0;             \
 222         register TYPE1 __r17 __asm__("$17") = arg1;             \
 223         __asm__ __volatile__(                                   \
 224                 "call_pal %2 # "#NAME                           \
 225                 : "=r"(__r16), "=r"(__r17)                      \
 226                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
 227                 : "$1", "$22", "$23", "$24", "$25");            \
 228 }
 229
 230 #define __CALL_PAL_RW1(NAME, RTYPE, TYPE0)                      \
 231 static inline RTYPE NAME(TYPE0 arg0)                            \
 232 {                                                               \
 233         register RTYPE __r0 __asm__("$0");                      \
 234         register TYPE0 __r16 __asm__("$16") = arg0;             \
 235         __asm__ __volatile__(                                   \
 236                 "call_pal %2 # "#NAME                           \
 237                 : "=r"(__r16), "=r"(__r0)                       \
 238                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
 239                 : "$1", "$22", "$23", "$24", "$25");            \
 240         return __r0;                                            \
 241 }
 242
 243 #define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1)               \
 244 static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1)                \
 245 {                                                               \
 246         register RTYPE __r0 __asm__("$0");                      \
 247         register TYPE0 __r16 __asm__("$16") = arg0;             \
 248         register TYPE1 __r17 __asm__("$17") = arg1;             \
 249         __asm__ __volatile__(                                   \
 250                 "call_pal %3 # "#NAME                           \
 251                 : "=r"(__r16), "=r"(__r17), "=r"(__r0)          \
 252                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
 253                 : "$1", "$22", "$23", "$24", "$25");            \
 254         return __r0;                                            \
 255 }
 256
 257 __CALL_PAL_W1(cflush, unsigned long);
 258 __CALL_PAL_R0(rdmces, unsigned long);
 259 __CALL_PAL_R0(rdps, unsigned long);
 260 __CALL_PAL_R0(rdusp, unsigned long);
 261 __CALL_PAL_RW1(swpipl, unsigned long, unsigned long);
 262 __CALL_PAL_R0(whami, unsigned long);
 263 __CALL_PAL_W2(wrent, void*, unsigned long);
 264 __CALL_PAL_W1(wripir, unsigned long);
 265 __CALL_PAL_W1(wrkgp, unsigned long);
 266 __CALL_PAL_W1(wrmces, unsigned long);
 267 __CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long);
 268 __CALL_PAL_W1(wrusp, unsigned long);
 269 __CALL_PAL_W1(wrvptptr, unsigned long);
 270
 271 #define IPL_MIN         0
 272 #define IPL_SW0         1
 273 #define IPL_SW1         2
 274 #define IPL_DEV0        3
 275 #define IPL_DEV1        4
 276 #define IPL_TIMER       5
 277 #define IPL_PERF        6
 278 #define IPL_POWERFAIL   6
 279 #define IPL_MCHECK      7
 280 #define IPL_MAX         7
 281
 282 #ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK
 283 #undef IPL_MIN
 284 #define IPL_MIN         __min_ipl
 285 extern int __min_ipl;
 286 #endif
 287
 288 #define getipl()                (rdps() & 7)
 289 #define setipl(ipl)             ((void) swpipl(ipl))
 290
 291 #define local_irq_disable()                     do { setipl(IPL_MAX); barrier(); } while(0)
 292 #define local_irq_enable()                      do { barrier(); setipl(IPL_MIN); } while(0)
 293 #define local_save_flags(flags) ((flags) = rdps())
 294 #define local_irq_save(flags)   do { (flags) = swpipl(IPL_MAX); barrier(); } while(0)
 295 #define local_irq_restore(flags)        do { barrier(); setipl(flags); barrier(); } while(0)
 296
 297 #define irqs_disabled() (getipl() == IPL_MAX)
 298
 299 /*
 300  * TB routines..
 301  */
 302 #define __tbi(nr,arg,arg1...)                                   \
 303 ({                                                              \
 304         register unsigned long __r16 __asm__("$16") = (nr);     \
 305         register unsigned long __r17 __asm__("$17"); arg;       \
 306         __asm__ __volatile__(                                   \
 307                 "call_pal %3 #__tbi"                            \
 308                 :"=r" (__r16),"=r" (__r17)                      \
 309                 :"0" (__r16),"i" (PAL_tbi) ,##arg1              \
 310                 :"$0", "$1", "$22", "$23", "$24", "$25");       \
 311 })
 312
 313 #define tbi(x,y)        __tbi(x,__r17=(y),"1" (__r17))
 314 #define tbisi(x)        __tbi(1,__r17=(x),"1" (__r17))
 315 #define tbisd(x)        __tbi(2,__r17=(x),"1" (__r17))
 316 #define tbis(x)         __tbi(3,__r17=(x),"1" (__r17))
 317 #define tbiap()         __tbi(-1, /* no second argument */)
 318 #define tbia()          __tbi(-2, /* no second argument */)
 319
 320 /*
 321  * Atomic exchange.
 322  * Since it can be used to implement critical sections
 323  * it must clobber "memory" (also for interrupts in UP).
 324  */
 325
 326 static inline unsigned long
 327 __xchg_u8(volatile char *m, unsigned long val)
 328 {
 329         unsigned long ret, tmp, addr64;
 330
 331         __asm__ __volatile__(
 332         "       andnot  %4,7,%3\n"
 333         "       insbl   %1,%4,%1\n"
 334         "1:     ldq_l   %2,0(%3)\n"
 335         "       extbl   %2,%4,%0\n"
 336         "       mskbl   %2,%4,%2\n"
 337         "       or      %1,%2,%2\n"
 338         "       stq_c   %2,0(%3)\n"
 339         "       beq     %2,2f\n"
 340 #ifdef CONFIG_SMP
 341         "       mb\n"
 342 #endif
 343         ".subsection 2\n"
 344         "2:     br      1b\n"
 345         ".previous"
 346         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 347         : "r" ((long)m), "1" (val) : "memory");
 348
 349         return ret;
 350 }
 351
 352 static inline unsigned long
 353 __xchg_u16(volatile short *m, unsigned long val)
 354 {
 355         unsigned long ret, tmp, addr64;
 356
 357         __asm__ __volatile__(
 358         "       andnot  %4,7,%3\n"
 359         "       inswl   %1,%4,%1\n"
 360         "1:     ldq_l   %2,0(%3)\n"
 361         "       extwl   %2,%4,%0\n"
 362         "       mskwl   %2,%4,%2\n"
 363         "       or      %1,%2,%2\n"
 364         "       stq_c   %2,0(%3)\n"
 365         "       beq     %2,2f\n"
 366 #ifdef CONFIG_SMP
 367         "       mb\n"
 368 #endif
 369         ".subsection 2\n"
 370         "2:     br      1b\n"
 371         ".previous"
 372         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 373         : "r" ((long)m), "1" (val) : "memory");
 374
 375         return ret;
 376 }
 377
 378 static inline unsigned long
 379 __xchg_u32(volatile int *m, unsigned long val)
 380 {
 381         unsigned long dummy;
 382
 383         __asm__ __volatile__(
 384         "1:     ldl_l %0,%4\n"
 385         "       bis $31,%3,%1\n"
 386         "       stl_c %1,%2\n"
 387         "       beq %1,2f\n"
 388 #ifdef CONFIG_SMP
 389         "       mb\n"
 390 #endif
 391         ".subsection 2\n"
 392         "2:     br 1b\n"
 393         ".previous"
 394         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 395         : "rI" (val), "m" (*m) : "memory");
 396
 397         return val;
 398 }
 399
 400 static inline unsigned long
 401 __xchg_u64(volatile long *m, unsigned long val)
 402 {
 403         unsigned long dummy;
 404
 405         __asm__ __volatile__(
 406         "1:     ldq_l %0,%4\n"
 407         "       bis $31,%3,%1\n"
 408         "       stq_c %1,%2\n"
 409         "       beq %1,2f\n"
 410 #ifdef CONFIG_SMP
 411         "       mb\n"
 412 #endif
 413         ".subsection 2\n"
 414         "2:     br 1b\n"
 415         ".previous"
 416         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 417         : "rI" (val), "m" (*m) : "memory");
 418
 419         return val;
 420 }
 421
 422 /* This function doesn't exist, so you'll get a linker error
 423    if something tries to do an invalid xchg().  */
 424 extern void __xchg_called_with_bad_pointer(void);
 425
 426 #define __xchg(ptr, x, size) \
 427 ({ \
 428         unsigned long __xchg__res; \
 429         volatile void *__xchg__ptr = (ptr); \
 430         switch (size) { \
 431                 case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \
 432                 case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \
 433                 case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \
 434                 case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \
 435                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
 436         } \
 437         __xchg__res; \
 438 })
 439
 440 #define xchg(ptr,x)                                                          \
 441   ({                                                                         \
 442      __typeof__(*(ptr)) _x_ = (x);                                           \
 443      (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
 444   })
 445
 446 static inline unsigned long
 447 __xchg_u8_local(volatile char *m, unsigned long val)
 448 {
 449         unsigned long ret, tmp, addr64;
 450
 451         __asm__ __volatile__(
 452         "       andnot  %4,7,%3\n"
 453         "       insbl   %1,%4,%1\n"
 454         "1:     ldq_l   %2,0(%3)\n"
 455         "       extbl   %2,%4,%0\n"
 456         "       mskbl   %2,%4,%2\n"
 457         "       or      %1,%2,%2\n"
 458         "       stq_c   %2,0(%3)\n"
 459         "       beq     %2,2f\n"
 460         ".subsection 2\n"
 461         "2:     br      1b\n"
 462         ".previous"
 463         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 464         : "r" ((long)m), "1" (val) : "memory");
 465
 466         return ret;
 467 }
 468
 469 static inline unsigned long
 470 __xchg_u16_local(volatile short *m, unsigned long val)
 471 {
 472         unsigned long ret, tmp, addr64;
 473
 474         __asm__ __volatile__(
 475         "       andnot  %4,7,%3\n"
 476         "       inswl   %1,%4,%1\n"
 477         "1:     ldq_l   %2,0(%3)\n"
 478         "       extwl   %2,%4,%0\n"
 479         "       mskwl   %2,%4,%2\n"
 480         "       or      %1,%2,%2\n"
 481         "       stq_c   %2,0(%3)\n"
 482         "       beq     %2,2f\n"
 483         ".subsection 2\n"
 484         "2:     br      1b\n"
 485         ".previous"
 486         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 487         : "r" ((long)m), "1" (val) : "memory");
 488
 489         return ret;
 490 }
 491
 492 static inline unsigned long
 493 __xchg_u32_local(volatile int *m, unsigned long val)
 494 {
 495         unsigned long dummy;
 496
 497         __asm__ __volatile__(
 498         "1:     ldl_l %0,%4\n"
 499         "       bis $31,%3,%1\n"
 500         "       stl_c %1,%2\n"
 501         "       beq %1,2f\n"
 502         ".subsection 2\n"
 503         "2:     br 1b\n"
 504         ".previous"
 505         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 506         : "rI" (val), "m" (*m) : "memory");
 507
 508         return val;
 509 }
 510
 511 static inline unsigned long
 512 __xchg_u64_local(volatile long *m, unsigned long val)
 513 {
 514         unsigned long dummy;
 515
 516         __asm__ __volatile__(
 517         "1:     ldq_l %0,%4\n"
 518         "       bis $31,%3,%1\n"
 519         "       stq_c %1,%2\n"
 520         "       beq %1,2f\n"
 521         ".subsection 2\n"
 522         "2:     br 1b\n"
 523         ".previous"
 524         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 525         : "rI" (val), "m" (*m) : "memory");
 526
 527         return val;
 528 }
 529
 530 #define __xchg_local(ptr, x, size) \
 531 ({ \
 532         unsigned long __xchg__res; \
 533         volatile void *__xchg__ptr = (ptr); \
 534         switch (size) { \
 535                 case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \
 536                 case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \
 537                 case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \
 538                 case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \
 539                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
 540         } \
 541         __xchg__res; \
 542 })
 543
 544 #define xchg_local(ptr,x)                                                    \
 545   ({                                                                         \
 546      __typeof__(*(ptr)) _x_ = (x);                                           \
 547      (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_,            \
 548                 sizeof(*(ptr))); \
 549   })
 550
 551 /*
 552  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 553  * store NEW in MEM.  Return the initial value in MEM.  Success is
 554  * indicated by comparing RETURN with OLD.
 555  *
 556  * The memory barrier should be placed in SMP only when we actually
 557  * make the change. If we don't change anything (so if the returned
 558  * prev is equal to old) then we aren't acquiring anything new and
 559  * we don't need any memory barrier as far I can tell.
 560  */
 561
 562 #define __HAVE_ARCH_CMPXCHG 1
 563
 564 static inline unsigned long
 565 __cmpxchg_u8(volatile char *m, long old, long new)
 566 {
 567         unsigned long prev, tmp, cmp, addr64;
 568
 569         __asm__ __volatile__(
 570         "       andnot  %5,7,%4\n"
 571         "       insbl   %1,%5,%1\n"
 572         "1:     ldq_l   %2,0(%4)\n"
 573         "       extbl   %2,%5,%0\n"
 574         "       cmpeq   %0,%6,%3\n"
 575         "       beq     %3,2f\n"
 576         "       mskbl   %2,%5,%2\n"
 577         "       or      %1,%2,%2\n"
 578         "       stq_c   %2,0(%4)\n"
 579         "       beq     %2,3f\n"
 580 #ifdef CONFIG_SMP
 581         "       mb\n"
 582 #endif
 583         "2:\n"
 584         ".subsection 2\n"
 585         "3:     br      1b\n"
 586         ".previous"
 587         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 588         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 589
 590         return prev;
 591 }
 592
 593 static inline unsigned long
 594 __cmpxchg_u16(volatile short *m, long old, long new)
 595 {
 596         unsigned long prev, tmp, cmp, addr64;
 597
 598         __asm__ __volatile__(
 599         "       andnot  %5,7,%4\n"
 600         "       inswl   %1,%5,%1\n"
 601         "1:     ldq_l   %2,0(%4)\n"
 602         "       extwl   %2,%5,%0\n"
 603         "       cmpeq   %0,%6,%3\n"
 604         "       beq     %3,2f\n"
 605         "       mskwl   %2,%5,%2\n"
 606         "       or      %1,%2,%2\n"
 607         "       stq_c   %2,0(%4)\n"
 608         "       beq     %2,3f\n"
 609 #ifdef CONFIG_SMP
 610         "       mb\n"
 611 #endif
 612         "2:\n"
 613         ".subsection 2\n"
 614         "3:     br      1b\n"
 615         ".previous"
 616         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 617         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 618
 619         return prev;
 620 }
 621
 622 static inline unsigned long
 623 __cmpxchg_u32(volatile int *m, int old, int new)
 624 {
 625         unsigned long prev, cmp;
 626
 627         __asm__ __volatile__(
 628         "1:     ldl_l %0,%5\n"
 629         "       cmpeq %0,%3,%1\n"
 630         "       beq %1,2f\n"
 631         "       mov %4,%1\n"
 632         "       stl_c %1,%2\n"
 633         "       beq %1,3f\n"
 634 #ifdef CONFIG_SMP
 635         "       mb\n"
 636 #endif
 637         "2:\n"
 638         ".subsection 2\n"
 639         "3:     br 1b\n"
 640         ".previous"
 641         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 642         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 643
 644         return prev;
 645 }
 646
 647 static inline unsigned long
 648 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
 649 {
 650         unsigned long prev, cmp;
 651
 652         __asm__ __volatile__(
 653         "1:     ldq_l %0,%5\n"
 654         "       cmpeq %0,%3,%1\n"
 655         "       beq %1,2f\n"
 656         "       mov %4,%1\n"
 657         "       stq_c %1,%2\n"
 658         "       beq %1,3f\n"
 659 #ifdef CONFIG_SMP
 660         "       mb\n"
 661 #endif
 662         "2:\n"
 663         ".subsection 2\n"
 664         "3:     br 1b\n"
 665         ".previous"
 666         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 667         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 668
 669         return prev;
 670 }
 671
 672 /* This function doesn't exist, so you'll get a linker error
 673    if something tries to do an invalid cmpxchg().  */
 674 extern void __cmpxchg_called_with_bad_pointer(void);
 675
 676 static __always_inline unsigned long
 677 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
 678 {
 679         switch (size) {
 680                 case 1:
 681                         return __cmpxchg_u8(ptr, old, new);
 682                 case 2:
 683                         return __cmpxchg_u16(ptr, old, new);
 684                 case 4:
 685                         return __cmpxchg_u32(ptr, old, new);
 686                 case 8:
 687                         return __cmpxchg_u64(ptr, old, new);
 688         }
 689         __cmpxchg_called_with_bad_pointer();
 690         return old;
 691 }
 692
 693 #define cmpxchg(ptr,o,n)                                                 \
 694   ({                                                                     \
 695      __typeof__(*(ptr)) _o_ = (o);                                       \
 696      __typeof__(*(ptr)) _n_ = (n);                                       \
 697      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 698                                     (unsigned long)_n_, sizeof(*(ptr))); \
 699   })
 700
 701 static inline unsigned long
 702 __cmpxchg_u8_local(volatile char *m, long old, long new)
 703 {
 704         unsigned long prev, tmp, cmp, addr64;
 705
 706         __asm__ __volatile__(
 707         "       andnot  %5,7,%4\n"
 708         "       insbl   %1,%5,%1\n"
 709         "1:     ldq_l   %2,0(%4)\n"
 710         "       extbl   %2,%5,%0\n"
 711         "       cmpeq   %0,%6,%3\n"
 712         "       beq     %3,2f\n"
 713         "       mskbl   %2,%5,%2\n"
 714         "       or      %1,%2,%2\n"
 715         "       stq_c   %2,0(%4)\n"
 716         "       beq     %2,3f\n"
 717         "2:\n"
 718         ".subsection 2\n"
 719         "3:     br      1b\n"
 720         ".previous"
 721         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 722         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 723
 724         return prev;
 725 }
 726
 727 static inline unsigned long
 728 __cmpxchg_u16_local(volatile short *m, long old, long new)
 729 {
 730         unsigned long prev, tmp, cmp, addr64;
 731
 732         __asm__ __volatile__(
 733         "       andnot  %5,7,%4\n"
 734         "       inswl   %1,%5,%1\n"
 735         "1:     ldq_l   %2,0(%4)\n"
 736         "       extwl   %2,%5,%0\n"
 737         "       cmpeq   %0,%6,%3\n"
 738         "       beq     %3,2f\n"
 739         "       mskwl   %2,%5,%2\n"
 740         "       or      %1,%2,%2\n"
 741         "       stq_c   %2,0(%4)\n"
 742         "       beq     %2,3f\n"
 743         "2:\n"
 744         ".subsection 2\n"
 745         "3:     br      1b\n"
 746         ".previous"
 747         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 748         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 749
 750         return prev;
 751 }
 752
 753 static inline unsigned long
 754 __cmpxchg_u32_local(volatile int *m, int old, int new)
 755 {
 756         unsigned long prev, cmp;
 757
 758         __asm__ __volatile__(
 759         "1:     ldl_l %0,%5\n"
 760         "       cmpeq %0,%3,%1\n"
 761         "       beq %1,2f\n"
 762         "       mov %4,%1\n"
 763         "       stl_c %1,%2\n"
 764         "       beq %1,3f\n"
 765         "2:\n"
 766         ".subsection 2\n"
 767         "3:     br 1b\n"
 768         ".previous"
 769         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 770         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 771
 772         return prev;
 773 }
 774
 775 static inline unsigned long
 776 __cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new)
 777 {
 778         unsigned long prev, cmp;
 779
 780         __asm__ __volatile__(
 781         "1:     ldq_l %0,%5\n"
 782         "       cmpeq %0,%3,%1\n"
 783         "       beq %1,2f\n"
 784         "       mov %4,%1\n"
 785         "       stq_c %1,%2\n"
 786         "       beq %1,3f\n"
 787         "2:\n"
 788         ".subsection 2\n"
 789         "3:     br 1b\n"
 790         ".previous"
 791         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 792         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 793
 794         return prev;
 795 }
 796
 797 static __always_inline unsigned long
 798 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
 799                 int size)
 800 {
 801         switch (size) {
 802                 case 1:
 803                         return __cmpxchg_u8_local(ptr, old, new);
 804                 case 2:
 805                         return __cmpxchg_u16_local(ptr, old, new);
 806                 case 4:
 807                         return __cmpxchg_u32_local(ptr, old, new);
 808                 case 8:
 809                         return __cmpxchg_u64_local(ptr, old, new);
 810         }
 811         __cmpxchg_called_with_bad_pointer();
 812         return old;
 813 }
 814
 815 #define cmpxchg_local(ptr,o,n)                                           \
 816   ({                                                                     \
 817      __typeof__(*(ptr)) _o_ = (o);                                       \
 818      __typeof__(*(ptr)) _n_ = (n);                                       \
 819      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 820                                     (unsigned long)_n_, sizeof(*(ptr))); \
 821   })
 822
 823 #endif /* __ASSEMBLY__ */
 824
 825 #define arch_align_stack(x) (x)
 826
 827 #endif