arch/arc/include/asm/spinlock.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef __ASM_SPINLOCK_H
  10 #define __ASM_SPINLOCK_H
  11
  12 #include <asm/spinlock_types.h>
  13 #include <asm/processor.h>
  14 #include <asm/barrier.h>
  15
  16 #define arch_spin_is_locked(x)  ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
  17
  18 #ifdef CONFIG_ARC_HAS_LLSC
  19
  20 static inline void arch_spin_lock(arch_spinlock_t *lock)
  21 {
  22         unsigned int val;
  23
  24         smp_mb();
  25
  26         __asm__ __volatile__(
  27         "1:     llock   %[val], [%[slock]]      \n"
  28         "       breq    %[val], %[LOCKED], 1b   \n"     /* spin while LOCKED */
  29         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  30         "       bnz     1b                      \n"
  31         "                                       \n"
  32         : [val]         "=&r"   (val)
  33         : [slock]       "r"     (&(lock->slock)),
  34           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  35         : "memory", "cc");
  36
  37         smp_mb();
  38 }
  39
  40 /* 1 - lock taken successfully */
  41 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  42 {
  43         unsigned int val, got_it = 0;
  44
  45         smp_mb();
  46
  47         __asm__ __volatile__(
  48         "1:     llock   %[val], [%[slock]]      \n"
  49         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
  50         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  51         "       bnz     1b                      \n"
  52         "       mov     %[got_it], 1            \n"
  53         "4:                                     \n"
  54         "                                       \n"
  55         : [val]         "=&r"   (val),
  56           [got_it]      "+&r"   (got_it)
  57         : [slock]       "r"     (&(lock->slock)),
  58           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  59         : "memory", "cc");
  60
  61         smp_mb();
  62
  63         return got_it;
  64 }
  65
  66 static inline void arch_spin_unlock(arch_spinlock_t *lock)
  67 {
  68         smp_mb();
  69
  70         lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__;
  71
  72         smp_mb();
  73 }
  74
  75 /*
  76  * Read-write spinlocks, allowing multiple readers but only one writer.
  77  * Unfair locking as Writers could be starved indefinitely by Reader(s)
  78  */
  79
  80 static inline void arch_read_lock(arch_rwlock_t *rw)
  81 {
  82         unsigned int val;
  83
  84         smp_mb();
  85
  86         /*
  87          * zero means writer holds the lock exclusively, deny Reader.
  88          * Otherwise grant lock to first/subseq reader
  89          *
  90          *      if (rw->counter > 0) {
  91          *              rw->counter--;
  92          *              ret = 1;
  93          *      }
  94          */
  95
  96         __asm__ __volatile__(
  97         "1:     llock   %[val], [%[rwlock]]     \n"
  98         "       brls    %[val], %[WR_LOCKED], 1b\n"     /* <= 0: spin while write locked */
  99         "       sub     %[val], %[val], 1       \n"     /* reader lock */
 100         "       scond   %[val], [%[rwlock]]     \n"
 101         "       bnz     1b                      \n"
 102         "                                       \n"
 103         : [val]         "=&r"   (val)
 104         : [rwlock]      "r"     (&(rw->counter)),
 105           [WR_LOCKED]   "ir"    (0)
 106         : "memory", "cc");
 107
 108         smp_mb();
 109 }
 110
 111 /* 1 - lock taken successfully */
 112 static inline int arch_read_trylock(arch_rwlock_t *rw)
 113 {
 114         unsigned int val, got_it = 0;
 115
 116         smp_mb();
 117
 118         __asm__ __volatile__(
 119         "1:     llock   %[val], [%[rwlock]]     \n"
 120         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
 121         "       sub     %[val], %[val], 1       \n"     /* counter-- */
 122         "       scond   %[val], [%[rwlock]]     \n"
 123         "       bnz     1b                      \n"     /* retry if collided with someone */
 124         "       mov     %[got_it], 1            \n"
 125         "                                       \n"
 126         "4: ; --- done ---                      \n"
 127
 128         : [val]         "=&r"   (val),
 129           [got_it]      "+&r"   (got_it)
 130         : [rwlock]      "r"     (&(rw->counter)),
 131           [WR_LOCKED]   "ir"    (0)
 132         : "memory", "cc");
 133
 134         smp_mb();
 135
 136         return got_it;
 137 }
 138
 139 static inline void arch_write_lock(arch_rwlock_t *rw)
 140 {
 141         unsigned int val;
 142
 143         smp_mb();
 144
 145         /*
 146          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 147          * deny writer. Otherwise if unlocked grant to writer
 148          * Hence the claim that Linux rwlocks are unfair to writers.
 149          * (can be starved for an indefinite time by readers).
 150          *
 151          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 152          *              rw->counter = 0;
 153          *              ret = 1;
 154          *      }
 155          */
 156
 157         __asm__ __volatile__(
 158         "1:     llock   %[val], [%[rwlock]]     \n"
 159         "       brne    %[val], %[UNLOCKED], 1b \n"     /* while !UNLOCKED spin */
 160         "       mov     %[val], %[WR_LOCKED]    \n"
 161         "       scond   %[val], [%[rwlock]]     \n"
 162         "       bnz     1b                      \n"
 163         "                                       \n"
 164         : [val]         "=&r"   (val)
 165         : [rwlock]      "r"     (&(rw->counter)),
 166           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 167           [WR_LOCKED]   "ir"    (0)
 168         : "memory", "cc");
 169
 170         smp_mb();
 171 }
 172
 173 /* 1 - lock taken successfully */
 174 static inline int arch_write_trylock(arch_rwlock_t *rw)
 175 {
 176         unsigned int val, got_it = 0;
 177
 178         smp_mb();
 179
 180         __asm__ __volatile__(
 181         "1:     llock   %[val], [%[rwlock]]     \n"
 182         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
 183         "       mov     %[val], %[WR_LOCKED]    \n"
 184         "       scond   %[val], [%[rwlock]]     \n"
 185         "       bnz     1b                      \n"     /* retry if collided with someone */
 186         "       mov     %[got_it], 1            \n"
 187         "                                       \n"
 188         "4: ; --- done ---                      \n"
 189
 190         : [val]         "=&r"   (val),
 191           [got_it]      "+&r"   (got_it)
 192         : [rwlock]      "r"     (&(rw->counter)),
 193           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 194           [WR_LOCKED]   "ir"    (0)
 195         : "memory", "cc");
 196
 197         smp_mb();
 198
 199         return got_it;
 200 }
 201
 202 static inline void arch_read_unlock(arch_rwlock_t *rw)
 203 {
 204         unsigned int val;
 205
 206         smp_mb();
 207
 208         /*
 209          * rw->counter++;
 210          */
 211         __asm__ __volatile__(
 212         "1:     llock   %[val], [%[rwlock]]     \n"
 213         "       add     %[val], %[val], 1       \n"
 214         "       scond   %[val], [%[rwlock]]     \n"
 215         "       bnz     1b                      \n"
 216         "                                       \n"
 217         : [val]         "=&r"   (val)
 218         : [rwlock]      "r"     (&(rw->counter))
 219         : "memory", "cc");
 220
 221         smp_mb();
 222 }
 223
 224 static inline void arch_write_unlock(arch_rwlock_t *rw)
 225 {
 226         smp_mb();
 227
 228         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 229
 230         smp_mb();
 231 }
 232
 233 #else   /* !CONFIG_ARC_HAS_LLSC */
 234
 235 static inline void arch_spin_lock(arch_spinlock_t *lock)
 236 {
 237         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 238
 239         /*
 240          * This smp_mb() is technically superfluous, we only need the one
 241          * after the lock for providing the ACQUIRE semantics.
 242          * However doing the "right" thing was regressing hackbench
 243          * so keeping this, pending further investigation
 244          */
 245         smp_mb();
 246
 247         __asm__ __volatile__(
 248         "1:     ex  %0, [%1]            \n"
 249 #ifdef CONFIG_EZNPS_MTM_EXT
 250         "       .word %3                \n"
 251 #endif
 252         "       breq  %0, %2, 1b        \n"
 253         : "+&r" (val)
 254         : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
 255 #ifdef CONFIG_EZNPS_MTM_EXT
 256         , "i"(CTOP_INST_SCHD_RW)
 257 #endif
 258         : "memory");
 259
 260         /*
 261          * ACQUIRE barrier to ensure load/store after taking the lock
 262          * don't "bleed-up" out of the critical section (leak-in is allowed)
 263          * http://www.spinics.net/lists/kernel/msg2010409.html
 264          *
 265          * ARCv2 only has load-load, store-store and all-all barrier
 266          * thus need the full all-all barrier
 267          */
 268         smp_mb();
 269 }
 270
 271 /* 1 - lock taken successfully */
 272 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 273 {
 274         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 275
 276         smp_mb();
 277
 278         __asm__ __volatile__(
 279         "1:     ex  %0, [%1]            \n"
 280         : "+r" (val)
 281         : "r"(&(lock->slock))
 282         : "memory");
 283
 284         smp_mb();
 285
 286         return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
 287 }
 288
 289 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 290 {
 291         unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
 292
 293         /*
 294          * RELEASE barrier: given the instructions avail on ARCv2, full barrier
 295          * is the only option
 296          */
 297         smp_mb();
 298
 299         /*
 300          * EX is not really required here, a simple STore of 0 suffices.
 301          * However this causes tasklist livelocks in SystemC based SMP virtual
 302          * platforms where the systemc core scheduler uses EX as a cue for
 303          * moving to next core. Do a git log of this file for details
 304          */
 305         __asm__ __volatile__(
 306         "       ex  %0, [%1]            \n"
 307         : "+r" (val)
 308         : "r"(&(lock->slock))
 309         : "memory");
 310
 311         /*
 312          * superfluous, but keeping for now - see pairing version in
 313          * arch_spin_lock above
 314          */
 315         smp_mb();
 316 }
 317
 318 /*
 319  * Read-write spinlocks, allowing multiple readers but only one writer.
 320  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 321  *
 322  * The spinlock itself is contained in @counter and access to it is
 323  * serialized with @lock_mutex.
 324  */
 325
 326 /* 1 - lock taken successfully */
 327 static inline int arch_read_trylock(arch_rwlock_t *rw)
 328 {
 329         int ret = 0;
 330         unsigned long flags;
 331
 332         local_irq_save(flags);
 333         arch_spin_lock(&(rw->lock_mutex));
 334
 335         /*
 336          * zero means writer holds the lock exclusively, deny Reader.
 337          * Otherwise grant lock to first/subseq reader
 338          */
 339         if (rw->counter > 0) {
 340                 rw->counter--;
 341                 ret = 1;
 342         }
 343
 344         arch_spin_unlock(&(rw->lock_mutex));
 345         local_irq_restore(flags);
 346
 347         smp_mb();
 348         return ret;
 349 }
 350
 351 /* 1 - lock taken successfully */
 352 static inline int arch_write_trylock(arch_rwlock_t *rw)
 353 {
 354         int ret = 0;
 355         unsigned long flags;
 356
 357         local_irq_save(flags);
 358         arch_spin_lock(&(rw->lock_mutex));
 359
 360         /*
 361          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 362          * deny writer. Otherwise if unlocked grant to writer
 363          * Hence the claim that Linux rwlocks are unfair to writers.
 364          * (can be starved for an indefinite time by readers).
 365          */
 366         if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 367                 rw->counter = 0;
 368                 ret = 1;
 369         }
 370         arch_spin_unlock(&(rw->lock_mutex));
 371         local_irq_restore(flags);
 372
 373         return ret;
 374 }
 375
 376 static inline void arch_read_lock(arch_rwlock_t *rw)
 377 {
 378         while (!arch_read_trylock(rw))
 379                 cpu_relax();
 380 }
 381
 382 static inline void arch_write_lock(arch_rwlock_t *rw)
 383 {
 384         while (!arch_write_trylock(rw))
 385                 cpu_relax();
 386 }
 387
 388 static inline void arch_read_unlock(arch_rwlock_t *rw)
 389 {
 390         unsigned long flags;
 391
 392         local_irq_save(flags);
 393         arch_spin_lock(&(rw->lock_mutex));
 394         rw->counter++;
 395         arch_spin_unlock(&(rw->lock_mutex));
 396         local_irq_restore(flags);
 397 }
 398
 399 static inline void arch_write_unlock(arch_rwlock_t *rw)
 400 {
 401         unsigned long flags;
 402
 403         local_irq_save(flags);
 404         arch_spin_lock(&(rw->lock_mutex));
 405         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 406         arch_spin_unlock(&(rw->lock_mutex));
 407         local_irq_restore(flags);
 408 }
 409
 410 #endif
 411
 412 #endif /* __ASM_SPINLOCK_H */