arch/mips/include/asm/spinlock.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  */
   9 #ifndef _ASM_SPINLOCK_H
  10 #define _ASM_SPINLOCK_H
  11
  12 #include <linux/compiler.h>
  13
  14 #include <asm/barrier.h>
  15 #include <asm/war.h>
  16
  17 /*
  18  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  19  *
  20  * Simple spin lock operations.  There are two variants, one clears IRQ's
  21  * on the local processor, one does not.
  22  *
  23  * These are fair FIFO ticket locks
  24  *
  25  * (the type definitions are in asm/spinlock_types.h)
  26  */
  27
  28
  29 /*
  30  * Ticket locks are conceptually two parts, one indicating the current head of
  31  * the queue, and the other indicating the current tail. The lock is acquired
  32  * by atomically noting the tail and incrementing it by one (thus adding
  33  * ourself to the queue and noting our position), then waiting until the head
  34  * becomes equal to the the initial value of the tail.
  35  */
  36
  37 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  38 {
  39         u32 counters = ACCESS_ONCE(lock->lock);
  40
  41         return ((counters >> 16) ^ counters) & 0xffff;
  42 }
  43
  44 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
  45 #define arch_spin_unlock_wait(x) \
  46         while (arch_spin_is_locked(x)) { cpu_relax(); }
  47
  48 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
  49 {
  50         u32 counters = ACCESS_ONCE(lock->lock);
  51
  52         return (((counters >> 16) - counters) & 0xffff) > 1;
  53 }
  54 #define arch_spin_is_contended  arch_spin_is_contended
  55
  56 static inline void arch_spin_lock(arch_spinlock_t *lock)
  57 {
  58         int my_ticket;
  59         int tmp;
  60         int inc = 0x10000;
  61
  62         if (R10000_LLSC_WAR) {
  63                 __asm__ __volatile__ (
  64                 "       .set push               # arch_spin_lock        \n"
  65                 "       .set noreorder                                  \n"
  66                 "                                                       \n"
  67                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
  68                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
  69                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
  70                 "       beqzl   %[my_ticket], 1b                        \n"
  71                 "        nop                                            \n"
  72                 "       srl     %[my_ticket], %[ticket], 16             \n"
  73                 "       andi    %[ticket], %[ticket], 0xffff            \n"
  74                 "       bne     %[ticket], %[my_ticket], 4f             \n"
  75                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  76                 "2:                                                     \n"
  77                 "       .subsection 2                                   \n"
  78                 "4:     andi    %[ticket], %[ticket], 0xffff            \n"
  79                 "       sll     %[ticket], 5                            \n"
  80                 "                                                       \n"
  81                 "6:     bnez    %[ticket], 6b                           \n"
  82                 "        subu   %[ticket], 1                            \n"
  83                 "                                                       \n"
  84                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
  85                 "       beq     %[ticket], %[my_ticket], 2b             \n"
  86                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  87                 "       b       4b                                      \n"
  88                 "        subu   %[ticket], %[ticket], 1                 \n"
  89                 "       .previous                                       \n"
  90                 "       .set pop                                        \n"
  91                 : [ticket_ptr] "+m" (lock->lock),
  92                   [serving_now_ptr] "+m" (lock->h.serving_now),
  93                   [ticket] "=&r" (tmp),
  94                   [my_ticket] "=&r" (my_ticket)
  95                 : [inc] "r" (inc));
  96         } else {
  97                 __asm__ __volatile__ (
  98                 "       .set push               # arch_spin_lock        \n"
  99                 "       .set noreorder                                  \n"
 100                 "                                                       \n"
 101                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 102                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
 103                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
 104                 "       beqz    %[my_ticket], 1b                        \n"
 105                 "        srl    %[my_ticket], %[ticket], 16             \n"
 106                 "       andi    %[ticket], %[ticket], 0xffff            \n"
 107                 "       bne     %[ticket], %[my_ticket], 4f             \n"
 108                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 109                 "2:                                                     \n"
 110                 "       .subsection 2                                   \n"
 111                 "4:     andi    %[ticket], %[ticket], 0x1fff            \n"
 112                 "       sll     %[ticket], 5                            \n"
 113                 "                                                       \n"
 114                 "6:     bnez    %[ticket], 6b                           \n"
 115                 "        subu   %[ticket], 1                            \n"
 116                 "                                                       \n"
 117                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
 118                 "       beq     %[ticket], %[my_ticket], 2b             \n"
 119                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 120                 "       b       4b                                      \n"
 121                 "        subu   %[ticket], %[ticket], 1                 \n"
 122                 "       .previous                                       \n"
 123                 "       .set pop                                        \n"
 124                 : [ticket_ptr] "+m" (lock->lock),
 125                   [serving_now_ptr] "+m" (lock->h.serving_now),
 126                   [ticket] "=&r" (tmp),
 127                   [my_ticket] "=&r" (my_ticket)
 128                 : [inc] "r" (inc));
 129         }
 130
 131         smp_llsc_mb();
 132 }
 133
 134 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 135 {
 136         unsigned int serving_now = lock->h.serving_now + 1;
 137         wmb();
 138         lock->h.serving_now = (u16)serving_now;
 139         nudge_writes();
 140 }
 141
 142 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
 143 {
 144         int tmp, tmp2, tmp3;
 145         int inc = 0x10000;
 146
 147         if (R10000_LLSC_WAR) {
 148                 __asm__ __volatile__ (
 149                 "       .set push               # arch_spin_trylock     \n"
 150                 "       .set noreorder                                  \n"
 151                 "                                                       \n"
 152                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 153                 "       srl     %[my_ticket], %[ticket], 16             \n"
 154                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 155                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 156                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 157                 "       sc      %[ticket], %[ticket_ptr]                \n"
 158                 "       beqzl   %[ticket], 1b                           \n"
 159                 "        li     %[ticket], 1                            \n"
 160                 "2:                                                     \n"
 161                 "       .subsection 2                                   \n"
 162                 "3:     b       2b                                      \n"
 163                 "        li     %[ticket], 0                            \n"
 164                 "       .previous                                       \n"
 165                 "       .set pop                                        \n"
 166                 : [ticket_ptr] "+m" (lock->lock),
 167                   [ticket] "=&r" (tmp),
 168                   [my_ticket] "=&r" (tmp2),
 169                   [now_serving] "=&r" (tmp3)
 170                 : [inc] "r" (inc));
 171         } else {
 172                 __asm__ __volatile__ (
 173                 "       .set push               # arch_spin_trylock     \n"
 174                 "       .set noreorder                                  \n"
 175                 "                                                       \n"
 176                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 177                 "       srl     %[my_ticket], %[ticket], 16             \n"
 178                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 179                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 180                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 181                 "       sc      %[ticket], %[ticket_ptr]                \n"
 182                 "       beqz    %[ticket], 1b                           \n"
 183                 "        li     %[ticket], 1                            \n"
 184                 "2:                                                     \n"
 185                 "       .subsection 2                                   \n"
 186                 "3:     b       2b                                      \n"
 187                 "        li     %[ticket], 0                            \n"
 188                 "       .previous                                       \n"
 189                 "       .set pop                                        \n"
 190                 : [ticket_ptr] "+m" (lock->lock),
 191                   [ticket] "=&r" (tmp),
 192                   [my_ticket] "=&r" (tmp2),
 193                   [now_serving] "=&r" (tmp3)
 194                 : [inc] "r" (inc));
 195         }
 196
 197         smp_llsc_mb();
 198
 199         return tmp;
 200 }
 201
 202 /*
 203  * Read-write spinlocks, allowing multiple readers but only one writer.
 204  *
 205  * NOTE! it is quite common to have readers in interrupts but no interrupt
 206  * writers. For those circumstances we can "mix" irq-safe locks - any writer
 207  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 208  * read-locks.
 209  */
 210
 211 /*
 212  * read_can_lock - would read_trylock() succeed?
 213  * @lock: the rwlock in question.
 214  */
 215 #define arch_read_can_lock(rw)  ((rw)->lock >= 0)
 216
 217 /*
 218  * write_can_lock - would write_trylock() succeed?
 219  * @lock: the rwlock in question.
 220  */
 221 #define arch_write_can_lock(rw) (!(rw)->lock)
 222
 223 static inline void arch_read_lock(arch_rwlock_t *rw)
 224 {
 225         unsigned int tmp;
 226
 227         if (R10000_LLSC_WAR) {
 228                 __asm__ __volatile__(
 229                 "       .set    noreorder       # arch_read_lock        \n"
 230                 "1:     ll      %1, %2                                  \n"
 231                 "       bltz    %1, 1b                                  \n"
 232                 "        addu   %1, 1                                   \n"
 233                 "       sc      %1, %0                                  \n"
 234                 "       beqzl   %1, 1b                                  \n"
 235                 "        nop                                            \n"
 236                 "       .set    reorder                                 \n"
 237                 : "=m" (rw->lock), "=&r" (tmp)
 238                 : "m" (rw->lock)
 239                 : "memory");
 240         } else {
 241                 do {
 242                         __asm__ __volatile__(
 243                         "1:     ll      %1, %2  # arch_read_lock        \n"
 244                         "       bltz    %1, 1b                          \n"
 245                         "        addu   %1, 1                           \n"
 246                         "2:     sc      %1, %0                          \n"
 247                         : "=m" (rw->lock), "=&r" (tmp)
 248                         : "m" (rw->lock)
 249                         : "memory");
 250                 } while (unlikely(!tmp));
 251         }
 252
 253         smp_llsc_mb();
 254 }
 255
 256 /* Note the use of sub, not subu which will make the kernel die with an
 257    overflow exception if we ever try to unlock an rwlock that is already
 258    unlocked or is being held by a writer.  */
 259 static inline void arch_read_unlock(arch_rwlock_t *rw)
 260 {
 261         unsigned int tmp;
 262
 263         smp_mb__before_llsc();
 264
 265         if (R10000_LLSC_WAR) {
 266                 __asm__ __volatile__(
 267                 "1:     ll      %1, %2          # arch_read_unlock      \n"
 268                 "       sub     %1, 1                                   \n"
 269                 "       sc      %1, %0                                  \n"
 270                 "       beqzl   %1, 1b                                  \n"
 271                 : "=m" (rw->lock), "=&r" (tmp)
 272                 : "m" (rw->lock)
 273                 : "memory");
 274         } else {
 275                 do {
 276                         __asm__ __volatile__(
 277                         "1:     ll      %1, %2  # arch_read_unlock      \n"
 278                         "       sub     %1, 1                           \n"
 279                         "       sc      %1, %0                          \n"
 280                         : "=m" (rw->lock), "=&r" (tmp)
 281                         : "m" (rw->lock)
 282                         : "memory");
 283                 } while (unlikely(!tmp));
 284         }
 285 }
 286
 287 static inline void arch_write_lock(arch_rwlock_t *rw)
 288 {
 289         unsigned int tmp;
 290
 291         if (R10000_LLSC_WAR) {
 292                 __asm__ __volatile__(
 293                 "       .set    noreorder       # arch_write_lock       \n"
 294                 "1:     ll      %1, %2                                  \n"
 295                 "       bnez    %1, 1b                                  \n"
 296                 "        lui    %1, 0x8000                              \n"
 297                 "       sc      %1, %0                                  \n"
 298                 "       beqzl   %1, 1b                                  \n"
 299                 "        nop                                            \n"
 300                 "       .set    reorder                                 \n"
 301                 : "=m" (rw->lock), "=&r" (tmp)
 302                 : "m" (rw->lock)
 303                 : "memory");
 304         } else {
 305                 do {
 306                         __asm__ __volatile__(
 307                         "1:     ll      %1, %2  # arch_write_lock       \n"
 308                         "       bnez    %1, 1b                          \n"
 309                         "        lui    %1, 0x8000                      \n"
 310                         "2:     sc      %1, %0                          \n"
 311                         : "=m" (rw->lock), "=&r" (tmp)
 312                         : "m" (rw->lock)
 313                         : "memory");
 314                 } while (unlikely(!tmp));
 315         }
 316
 317         smp_llsc_mb();
 318 }
 319
 320 static inline void arch_write_unlock(arch_rwlock_t *rw)
 321 {
 322         smp_mb();
 323
 324         __asm__ __volatile__(
 325         "                               # arch_write_unlock     \n"
 326         "       sw      $0, %0                                  \n"
 327         : "=m" (rw->lock)
 328         : "m" (rw->lock)
 329         : "memory");
 330 }
 331
 332 static inline int arch_read_trylock(arch_rwlock_t *rw)
 333 {
 334         unsigned int tmp;
 335         int ret;
 336
 337         if (R10000_LLSC_WAR) {
 338                 __asm__ __volatile__(
 339                 "       .set    noreorder       # arch_read_trylock     \n"
 340                 "       li      %2, 0                                   \n"
 341                 "1:     ll      %1, %3                                  \n"
 342                 "       bltz    %1, 2f                                  \n"
 343                 "        addu   %1, 1                                   \n"
 344                 "       sc      %1, %0                                  \n"
 345                 "       .set    reorder                                 \n"
 346                 "       beqzl   %1, 1b                                  \n"
 347                 "        nop                                            \n"
 348                 __WEAK_LLSC_MB
 349                 "       li      %2, 1                                   \n"
 350                 "2:                                                     \n"
 351                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
 352                 : "m" (rw->lock)
 353                 : "memory");
 354         } else {
 355                 __asm__ __volatile__(
 356                 "       .set    noreorder       # arch_read_trylock     \n"
 357                 "       li      %2, 0                                   \n"
 358                 "1:     ll      %1, %3                                  \n"
 359                 "       bltz    %1, 2f                                  \n"
 360                 "        addu   %1, 1                                   \n"
 361                 "       sc      %1, %0                                  \n"
 362                 "       beqz    %1, 1b                                  \n"
 363                 "        nop                                            \n"
 364                 "       .set    reorder                                 \n"
 365                 __WEAK_LLSC_MB
 366                 "       li      %2, 1                                   \n"
 367                 "2:                                                     \n"
 368                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
 369                 : "m" (rw->lock)
 370                 : "memory");
 371         }
 372
 373         return ret;
 374 }
 375
 376 static inline int arch_write_trylock(arch_rwlock_t *rw)
 377 {
 378         unsigned int tmp;
 379         int ret;
 380
 381         if (R10000_LLSC_WAR) {
 382                 __asm__ __volatile__(
 383                 "       .set    noreorder       # arch_write_trylock    \n"
 384                 "       li      %2, 0                                   \n"
 385                 "1:     ll      %1, %3                                  \n"
 386                 "       bnez    %1, 2f                                  \n"
 387                 "        lui    %1, 0x8000                              \n"
 388                 "       sc      %1, %0                                  \n"
 389                 "       beqzl   %1, 1b                                  \n"
 390                 "        nop                                            \n"
 391                 __WEAK_LLSC_MB
 392                 "       li      %2, 1                                   \n"
 393                 "       .set    reorder                                 \n"
 394                 "2:                                                     \n"
 395                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
 396                 : "m" (rw->lock)
 397                 : "memory");
 398         } else {
 399                 do {
 400                         __asm__ __volatile__(
 401                         "       ll      %1, %3  # arch_write_trylock    \n"
 402                         "       li      %2, 0                           \n"
 403                         "       bnez    %1, 2f                          \n"
 404                         "       lui     %1, 0x8000                      \n"
 405                         "       sc      %1, %0                          \n"
 406                         "       li      %2, 1                           \n"
 407                         "2:                                             \n"
 408                         : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
 409                         : "m" (rw->lock)
 410                         : "memory");
 411                 } while (unlikely(!tmp));
 412
 413                 smp_llsc_mb();
 414         }
 415
 416         return ret;
 417 }
 418
 419 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
 420 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
 421
 422 #define arch_spin_relax(lock)   cpu_relax()
 423 #define arch_read_relax(lock)   cpu_relax()
 424 #define arch_write_relax(lock)  cpu_relax()
 425
 426 #endif /* _ASM_SPINLOCK_H */