include/linux/spinlock.h

   1 #ifndef __LINUX_SPINLOCK_H
   2 #define __LINUX_SPINLOCK_H
   3
   4 /*
   5  * include/linux/spinlock.h - generic locking declarations
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/preempt.h>
  10 #include <linux/linkage.h>
  11 #include <linux/compiler.h>
  12 #include <linux/thread_info.h>
  13 #include <linux/kernel.h>
  14 #include <linux/stringify.h>
  15
  16 #include <asm/processor.h>      /* for cpu relax */
  17 #include <asm/system.h>
  18
  19 /*
  20  * Must define these before including other files, inline functions need them
  21  */
  22 #define LOCK_SECTION_NAME                       \
  23         ".text.lock." __stringify(KBUILD_BASENAME)
  24
  25 #define LOCK_SECTION_START(extra)               \
  26         ".subsection 1\n\t"                     \
  27         extra                                   \
  28         ".ifndef " LOCK_SECTION_NAME "\n\t"     \
  29         LOCK_SECTION_NAME ":\n\t"               \
  30         ".endif\n\t"
  31
  32 #define LOCK_SECTION_END                        \
  33         ".previous\n\t"
  34
  35 /*
  36  * If CONFIG_SMP is set, pull in the _raw_* definitions
  37  */
  38 #ifdef CONFIG_SMP
  39 #include <asm/spinlock.h>
  40
  41 #define __lockfunc fastcall __attribute__((section(".spinlock.text")))
  42
  43 int __lockfunc _spin_trylock(spinlock_t *lock);
  44 int __lockfunc _write_trylock(rwlock_t *lock);
  45 void __lockfunc _spin_lock(spinlock_t *lock);
  46 void __lockfunc _write_lock(rwlock_t *lock);
  47 void __lockfunc _spin_lock(spinlock_t *lock);
  48 void __lockfunc _read_lock(rwlock_t *lock);
  49 void __lockfunc _spin_unlock(spinlock_t *lock);
  50 void __lockfunc _write_unlock(rwlock_t *lock);
  51 void __lockfunc _read_unlock(rwlock_t *lock);
  52 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock);
  53 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock);
  54 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock);
  55 void __lockfunc _spin_lock_irq(spinlock_t *lock);
  56 void __lockfunc _spin_lock_bh(spinlock_t *lock);
  57 void __lockfunc _read_lock_irq(rwlock_t *lock);
  58 void __lockfunc _read_lock_bh(rwlock_t *lock);
  59 void __lockfunc _write_lock_irq(rwlock_t *lock);
  60 void __lockfunc _write_lock_bh(rwlock_t *lock);
  61 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags);
  62 void __lockfunc _spin_unlock_irq(spinlock_t *lock);
  63 void __lockfunc _spin_unlock_bh(spinlock_t *lock);
  64 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
  65 void __lockfunc _read_unlock_irq(rwlock_t *lock);
  66 void __lockfunc _read_unlock_bh(rwlock_t *lock);
  67 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
  68 void __lockfunc _write_unlock_irq(rwlock_t *lock);
  69 void __lockfunc _write_unlock_bh(rwlock_t *lock);
  70 int __lockfunc _spin_trylock_bh(spinlock_t *lock);
  71 int in_lock_functions(unsigned long addr);
  72 #else
  73
  74 #define in_lock_functions(ADDR) 0
  75
  76 #if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK)
  77 # define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic)
  78 # define ATOMIC_DEC_AND_LOCK
  79 #endif
  80
  81 #ifdef CONFIG_DEBUG_SPINLOCK
  82
  83 #define SPINLOCK_MAGIC  0x1D244B3C
  84 typedef struct {
  85         unsigned long magic;
  86         volatile unsigned long lock;
  87         volatile unsigned int babble;
  88         const char *module;
  89         char *owner;
  90         int oline;
  91 } spinlock_t;
  92 #define SPIN_LOCK_UNLOCKED (spinlock_t) { SPINLOCK_MAGIC, 0, 10, __FILE__ , NULL, 0}
  93
  94 #define spin_lock_init(x) \
  95         do { \
  96                 (x)->magic = SPINLOCK_MAGIC; \
  97                 (x)->lock = 0; \
  98                 (x)->babble = 5; \
  99                 (x)->module = __FILE__; \
 100                 (x)->owner = NULL; \
 101                 (x)->oline = 0; \
 102         } while (0)
 103
 104 #define CHECK_LOCK(x) \
 105         do { \
 106                 if ((x)->magic != SPINLOCK_MAGIC) { \
 107                         printk(KERN_ERR "%s:%d: spin_is_locked on uninitialized spinlock %p.\n", \
 108                                         __FILE__, __LINE__, (x)); \
 109                 } \
 110         } while(0)
 111
 112 #define _raw_spin_lock(x)               \
 113         do { \
 114                 CHECK_LOCK(x); \
 115                 if ((x)->lock&&(x)->babble) { \
 116                         (x)->babble--; \
 117                         printk("%s:%d: spin_lock(%s:%p) already locked by %s/%d\n", \
 118                                         __FILE__,__LINE__, (x)->module, \
 119                                         (x), (x)->owner, (x)->oline); \
 120                 } \
 121                 (x)->lock = 1; \
 122                 (x)->owner = __FILE__; \
 123                 (x)->oline = __LINE__; \
 124         } while (0)
 125
 126 /* without debugging, spin_is_locked on UP always says
 127  * FALSE. --> printk if already locked. */
 128 #define spin_is_locked(x) \
 129         ({ \
 130                 CHECK_LOCK(x); \
 131                 if ((x)->lock&&(x)->babble) { \
 132                         (x)->babble--; \
 133                         printk("%s:%d: spin_is_locked(%s:%p) already locked by %s/%d\n", \
 134                                         __FILE__,__LINE__, (x)->module, \
 135                                         (x), (x)->owner, (x)->oline); \
 136                 } \
 137                 0; \
 138         })
 139
 140 /* without debugging, spin_trylock on UP always says
 141  * TRUE. --> printk if already locked. */
 142 #define _raw_spin_trylock(x) \
 143         ({ \
 144                 CHECK_LOCK(x); \
 145                 if ((x)->lock&&(x)->babble) { \
 146                         (x)->babble--; \
 147                         printk("%s:%d: spin_trylock(%s:%p) already locked by %s/%d\n", \
 148                                         __FILE__,__LINE__, (x)->module, \
 149                                         (x), (x)->owner, (x)->oline); \
 150                 } \
 151                 (x)->lock = 1; \
 152                 (x)->owner = __FILE__; \
 153                 (x)->oline = __LINE__; \
 154                 1; \
 155         })
 156
 157 #define spin_unlock_wait(x)     \
 158         do { \
 159                 CHECK_LOCK(x); \
 160                 if ((x)->lock&&(x)->babble) { \
 161                         (x)->babble--; \
 162                         printk("%s:%d: spin_unlock_wait(%s:%p) owned by %s/%d\n", \
 163                                         __FILE__,__LINE__, (x)->module, (x), \
 164                                         (x)->owner, (x)->oline); \
 165                 }\
 166         } while (0)
 167
 168 #define _raw_spin_unlock(x) \
 169         do { \
 170                 CHECK_LOCK(x); \
 171                 if (!(x)->lock&&(x)->babble) { \
 172                         (x)->babble--; \
 173                         printk("%s:%d: spin_unlock(%s:%p) not locked\n", \
 174                                         __FILE__,__LINE__, (x)->module, (x));\
 175                 } \
 176                 (x)->lock = 0; \
 177         } while (0)
 178 #else
 179 /*
 180  * gcc versions before ~2.95 have a nasty bug with empty initializers.
 181  */
 182 #if (__GNUC__ > 2)
 183   typedef struct { } spinlock_t;
 184   #define SPIN_LOCK_UNLOCKED (spinlock_t) { }
 185 #else
 186   typedef struct { int gcc_is_buggy; } spinlock_t;
 187   #define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
 188 #endif
 189
 190 /*
 191  * If CONFIG_SMP is unset, declare the _raw_* definitions as nops
 192  */
 193 #define spin_lock_init(lock)    do { (void)(lock); } while(0)
 194 #define _raw_spin_lock(lock)    do { (void)(lock); } while(0)
 195 #define spin_is_locked(lock)    ((void)(lock), 0)
 196 #define _raw_spin_trylock(lock) (((void)(lock), 1))
 197 #define spin_unlock_wait(lock)  (void)(lock);
 198 #define _raw_spin_unlock(lock) do { (void)(lock); } while(0)
 199 #endif /* CONFIG_DEBUG_SPINLOCK */
 200
 201 /* RW spinlocks: No debug version */
 202
 203 #if (__GNUC__ > 2)
 204   typedef struct { } rwlock_t;
 205   #define RW_LOCK_UNLOCKED (rwlock_t) { }
 206 #else
 207   typedef struct { int gcc_is_buggy; } rwlock_t;
 208   #define RW_LOCK_UNLOCKED (rwlock_t) { 0 }
 209 #endif
 210
 211 #define rwlock_init(lock)       do { (void)(lock); } while(0)
 212 #define _raw_read_lock(lock)    do { (void)(lock); } while(0)
 213 #define _raw_read_unlock(lock)  do { (void)(lock); } while(0)
 214 #define _raw_write_lock(lock)   do { (void)(lock); } while(0)
 215 #define _raw_write_unlock(lock) do { (void)(lock); } while(0)
 216 #define _raw_write_trylock(lock) ({ (void)(lock); (1); })
 217
 218 #define _spin_trylock(lock)     ({preempt_disable(); _raw_spin_trylock(lock) ? \
 219                                 1 : ({preempt_enable(); 0;});})
 220
 221 #define _write_trylock(lock)    ({preempt_disable(); _raw_write_trylock(lock) ? \
 222                                 1 : ({preempt_enable(); 0;});})
 223
 224 #define _spin_trylock_bh(lock)  ({preempt_disable(); local_bh_disable(); \
 225                                 _raw_spin_trylock(lock) ? \
 226                                 1 : ({preempt_enable(); local_bh_enable(); 0;});})
 227
 228 #define _spin_lock(lock)        \
 229 do { \
 230         preempt_disable(); \
 231         _raw_spin_lock(lock); \
 232 } while(0)
 233
 234 #define _write_lock(lock) \
 235 do { \
 236         preempt_disable(); \
 237         _raw_write_lock(lock); \
 238 } while(0)
 239
 240 #define _read_lock(lock)        \
 241 do { \
 242         preempt_disable(); \
 243         _raw_read_lock(lock); \
 244 } while(0)
 245
 246 #define _spin_unlock(lock) \
 247 do { \
 248         _raw_spin_unlock(lock); \
 249         preempt_enable(); \
 250 } while (0)
 251
 252 #define _write_unlock(lock) \
 253 do { \
 254         _raw_write_unlock(lock); \
 255         preempt_enable(); \
 256 } while(0)
 257
 258 #define _read_unlock(lock) \
 259 do { \
 260         _raw_read_unlock(lock); \
 261         preempt_enable(); \
 262 } while(0)
 263
 264 #define _spin_lock_irqsave(lock, flags) \
 265 do {    \
 266         local_irq_save(flags); \
 267         preempt_disable(); \
 268         _raw_spin_lock(lock); \
 269 } while (0)
 270
 271 #define _spin_lock_irq(lock) \
 272 do { \
 273         local_irq_disable(); \
 274         preempt_disable(); \
 275         _raw_spin_lock(lock); \
 276 } while (0)
 277
 278 #define _spin_lock_bh(lock) \
 279 do { \
 280         local_bh_disable(); \
 281         preempt_disable(); \
 282         _raw_spin_lock(lock); \
 283 } while (0)
 284
 285 #define _read_lock_irqsave(lock, flags) \
 286 do {    \
 287         local_irq_save(flags); \
 288         preempt_disable(); \
 289         _raw_read_lock(lock); \
 290 } while (0)
 291
 292 #define _read_lock_irq(lock) \
 293 do { \
 294         local_irq_disable(); \
 295         preempt_disable(); \
 296         _raw_read_lock(lock); \
 297 } while (0)
 298
 299 #define _read_lock_bh(lock) \
 300 do { \
 301         local_bh_disable(); \
 302         preempt_disable(); \
 303         _raw_read_lock(lock); \
 304 } while (0)
 305
 306 #define _write_lock_irqsave(lock, flags) \
 307 do {    \
 308         local_irq_save(flags); \
 309         preempt_disable(); \
 310         _raw_write_lock(lock); \
 311 } while (0)
 312
 313 #define _write_lock_irq(lock) \
 314 do { \
 315         local_irq_disable(); \
 316         preempt_disable(); \
 317         _raw_write_lock(lock); \
 318 } while (0)
 319
 320 #define _write_lock_bh(lock) \
 321 do { \
 322         local_bh_disable(); \
 323         preempt_disable(); \
 324         _raw_write_lock(lock); \
 325 } while (0)
 326
 327 #define _spin_unlock_irqrestore(lock, flags) \
 328 do { \
 329         _raw_spin_unlock(lock); \
 330         local_irq_restore(flags); \
 331         preempt_enable(); \
 332 } while (0)
 333
 334 #define _raw_spin_unlock_irqrestore(lock, flags) \
 335 do { \
 336         _raw_spin_unlock(lock); \
 337         local_irq_restore(flags); \
 338 } while (0)
 339
 340 #define _spin_unlock_irq(lock) \
 341 do { \
 342         _raw_spin_unlock(lock); \
 343         local_irq_enable(); \
 344         preempt_enable(); \
 345 } while (0)
 346
 347 #define _spin_unlock_bh(lock) \
 348 do { \
 349         _raw_spin_unlock(lock); \
 350         preempt_enable(); \
 351         local_bh_enable(); \
 352 } while (0)
 353
 354 #define _write_unlock_bh(lock) \
 355 do { \
 356         _raw_write_unlock(lock); \
 357         preempt_enable(); \
 358         local_bh_enable(); \
 359 } while (0)
 360
 361 #define _read_unlock_irqrestore(lock, flags) \
 362 do { \
 363         _raw_read_unlock(lock); \
 364         local_irq_restore(flags); \
 365         preempt_enable(); \
 366 } while (0)
 367
 368 #define _write_unlock_irqrestore(lock, flags) \
 369 do { \
 370         _raw_write_unlock(lock); \
 371         local_irq_restore(flags); \
 372         preempt_enable(); \
 373 } while (0)
 374
 375 #define _read_unlock_irq(lock)  \
 376 do { \
 377         _raw_read_unlock(lock); \
 378         local_irq_enable();     \
 379         preempt_enable();       \
 380 } while (0)
 381
 382 #define _read_unlock_bh(lock)   \
 383 do { \
 384         _raw_read_unlock(lock); \
 385         local_bh_enable();      \
 386         preempt_enable();       \
 387 } while (0)
 388
 389 #define _write_unlock_irq(lock) \
 390 do { \
 391         _raw_write_unlock(lock);        \
 392         local_irq_enable();     \
 393         preempt_enable();       \
 394 } while (0)
 395
 396 #endif /* !SMP */
 397
 398 /*
 399  * Define the various spin_lock and rw_lock methods.  Note we define these
 400  * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
 401  * methods are defined as nops in the case they are not required.
 402  */
 403 #define spin_trylock(lock)      _spin_trylock(lock)
 404 #define write_trylock(lock)     _write_trylock(lock)
 405
 406 /* Where's read_trylock? */
 407
 408 #define spin_lock(lock)         _spin_lock(lock)
 409 #define write_lock(lock)        _write_lock(lock)
 410 #define read_lock(lock)         _read_lock(lock)
 411 #define spin_unlock(lock)       _spin_unlock(lock)
 412 #define write_unlock(lock)      _write_unlock(lock)
 413 #define read_unlock(lock)       _read_unlock(lock)
 414
 415 #ifdef CONFIG_SMP
 416 #define spin_lock_irqsave(lock, flags)  flags = _spin_lock_irqsave(lock)
 417 #define read_lock_irqsave(lock, flags)  flags = _read_lock_irqsave(lock)
 418 #define write_lock_irqsave(lock, flags) flags = _write_lock_irqsave(lock)
 419 #else
 420 #define spin_lock_irqsave(lock, flags)  _spin_lock_irqsave(lock, flags)
 421 #define read_lock_irqsave(lock, flags)  _read_lock_irqsave(lock, flags)
 422 #define write_lock_irqsave(lock, flags) _write_lock_irqsave(lock, flags)
 423 #endif
 424
 425 #define spin_lock_irq(lock)             _spin_lock_irq(lock)
 426 #define spin_lock_bh(lock)              _spin_lock_bh(lock)
 427
 428 #define read_lock_irq(lock)             _read_lock_irq(lock)
 429 #define read_lock_bh(lock)              _read_lock_bh(lock)
 430
 431 #define write_lock_irq(lock)            _write_lock_irq(lock)
 432 #define write_lock_bh(lock)             _write_lock_bh(lock)
 433 #define spin_unlock_irqrestore(lock, flags)     _spin_unlock_irqrestore(lock, flags)
 434 #define spin_unlock_irq(lock)           _spin_unlock_irq(lock)
 435 #define spin_unlock_bh(lock)            _spin_unlock_bh(lock)
 436
 437 #define read_unlock_irqrestore(lock, flags)     _read_unlock_irqrestore(lock, flags)
 438 #define read_unlock_irq(lock)                   _read_unlock_irq(lock)
 439 #define read_unlock_bh(lock)                    _read_unlock_bh(lock)
 440
 441 #define write_unlock_irqrestore(lock, flags)    _write_unlock_irqrestore(lock, flags)
 442 #define write_unlock_irq(lock)                  _write_unlock_irq(lock)
 443 #define write_unlock_bh(lock)                   _write_unlock_bh(lock)
 444
 445 #define spin_trylock_bh(lock)                   _spin_trylock_bh(lock)
 446
 447 #ifdef CONFIG_LOCKMETER
 448 extern void _metered_spin_lock   (spinlock_t *lock);
 449 extern void _metered_spin_unlock (spinlock_t *lock);
 450 extern int  _metered_spin_trylock(spinlock_t *lock);
 451 extern void _metered_read_lock    (rwlock_t *lock);
 452 extern void _metered_read_unlock  (rwlock_t *lock);
 453 extern void _metered_write_lock   (rwlock_t *lock);
 454 extern void _metered_write_unlock (rwlock_t *lock);
 455 extern int  _metered_write_trylock(rwlock_t *lock);
 456 #endif
 457
 458 /* "lock on reference count zero" */
 459 #ifndef ATOMIC_DEC_AND_LOCK
 460 #include <asm/atomic.h>
 461 extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
 462 #endif
 463
 464 /*
 465  *  bit-based spin_lock()
 466  *
 467  * Don't use this unless you really need to: spin_lock() and spin_unlock()
 468  * are significantly faster.
 469  */
 470 static inline void bit_spin_lock(int bitnum, unsigned long *addr)
 471 {
 472         /*
 473          * Assuming the lock is uncontended, this never enters
 474          * the body of the outer loop. If it is contended, then
 475          * within the inner loop a non-atomic test is used to
 476          * busywait with less bus contention for a good time to
 477          * attempt to acquire the lock bit.
 478          */
 479         preempt_disable();
 480 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 481         while (test_and_set_bit(bitnum, addr)) {
 482                 while (test_bit(bitnum, addr))
 483                         cpu_relax();
 484         }
 485 #endif
 486 }
 487
 488 /*
 489  * Return true if it was acquired
 490  */
 491 static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
 492 {
 493 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 494         int ret;
 495
 496         preempt_disable();
 497         ret = !test_and_set_bit(bitnum, addr);
 498         if (!ret)
 499                 preempt_enable();
 500         return ret;
 501 #else
 502         preempt_disable();
 503         return 1;
 504 #endif
 505 }
 506
 507 /*
 508  *  bit-based spin_unlock()
 509  */
 510 static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
 511 {
 512 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 513         BUG_ON(!test_bit(bitnum, addr));
 514         smp_mb__before_clear_bit();
 515         clear_bit(bitnum, addr);
 516 #endif
 517         preempt_enable();
 518 }
 519
 520 /*
 521  * Return true if the lock is held.
 522  */
 523 static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
 524 {
 525 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 526         return test_bit(bitnum, addr);
 527 #elif defined CONFIG_PREEMPT
 528         return preempt_count();
 529 #else
 530         return 1;
 531 #endif
 532 }
 533
 534 #endif /* __LINUX_SPINLOCK_H */