kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 int __lockfunc _spin_trylock(spinlock_t *lock)
  25 {
  26         preempt_disable();
  27         if (_raw_spin_trylock(lock)) {
  28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  29                 return 1;
  30         }
  31
  32         preempt_enable();
  33         return 0;
  34 }
  35 EXPORT_SYMBOL(_spin_trylock);
  36
  37 int __lockfunc _read_trylock(rwlock_t *lock)
  38 {
  39         preempt_disable();
  40         if (_raw_read_trylock(lock)) {
  41                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
  42                 return 1;
  43         }
  44
  45         preempt_enable();
  46         return 0;
  47 }
  48 EXPORT_SYMBOL(_read_trylock);
  49
  50 int __lockfunc _write_trylock(rwlock_t *lock)
  51 {
  52         preempt_disable();
  53         if (_raw_write_trylock(lock)) {
  54                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  55                 return 1;
  56         }
  57
  58         preempt_enable();
  59         return 0;
  60 }
  61 EXPORT_SYMBOL(_write_trylock);
  62
  63 /*
  64  * If lockdep is enabled then we use the non-preemption spin-ops
  65  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  66  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  67  */
  68 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
  69
  70 void __lockfunc _read_lock(rwlock_t *lock)
  71 {
  72         preempt_disable();
  73         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
  74         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
  75 }
  76 EXPORT_SYMBOL(_read_lock);
  77
  78 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  79 {
  80         unsigned long flags;
  81
  82         local_irq_save(flags);
  83         preempt_disable();
  84         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  85         /*
  86          * On lockdep we dont want the hand-coded irq-enable of
  87          * _raw_spin_lock_flags() code, because lockdep assumes
  88          * that interrupts are not re-enabled during lock-acquire:
  89          */
  90 #ifdef CONFIG_LOCKDEP
  91         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
  92 #else
  93         _raw_spin_lock_flags(lock, &flags);
  94 #endif
  95         return flags;
  96 }
  97 EXPORT_SYMBOL(_spin_lock_irqsave);
  98
  99 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 100 {
 101         local_irq_disable();
 102         preempt_disable();
 103         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 104         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 105 }
 106 EXPORT_SYMBOL(_spin_lock_irq);
 107
 108 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 109 {
 110         local_bh_disable();
 111         preempt_disable();
 112         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 113         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 114 }
 115 EXPORT_SYMBOL(_spin_lock_bh);
 116
 117 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 118 {
 119         unsigned long flags;
 120
 121         local_irq_save(flags);
 122         preempt_disable();
 123         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 124         LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
 125                              _raw_read_lock_flags, &flags);
 126         return flags;
 127 }
 128 EXPORT_SYMBOL(_read_lock_irqsave);
 129
 130 void __lockfunc _read_lock_irq(rwlock_t *lock)
 131 {
 132         local_irq_disable();
 133         preempt_disable();
 134         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 135         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 136 }
 137 EXPORT_SYMBOL(_read_lock_irq);
 138
 139 void __lockfunc _read_lock_bh(rwlock_t *lock)
 140 {
 141         local_bh_disable();
 142         preempt_disable();
 143         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 144         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 145 }
 146 EXPORT_SYMBOL(_read_lock_bh);
 147
 148 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 149 {
 150         unsigned long flags;
 151
 152         local_irq_save(flags);
 153         preempt_disable();
 154         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 155         LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
 156                              _raw_write_lock_flags, &flags);
 157         return flags;
 158 }
 159 EXPORT_SYMBOL(_write_lock_irqsave);
 160
 161 void __lockfunc _write_lock_irq(rwlock_t *lock)
 162 {
 163         local_irq_disable();
 164         preempt_disable();
 165         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 166         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 167 }
 168 EXPORT_SYMBOL(_write_lock_irq);
 169
 170 void __lockfunc _write_lock_bh(rwlock_t *lock)
 171 {
 172         local_bh_disable();
 173         preempt_disable();
 174         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 175         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 176 }
 177 EXPORT_SYMBOL(_write_lock_bh);
 178
 179 void __lockfunc _spin_lock(spinlock_t *lock)
 180 {
 181         preempt_disable();
 182         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 183         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 184 }
 185
 186 EXPORT_SYMBOL(_spin_lock);
 187
 188 void __lockfunc _write_lock(rwlock_t *lock)
 189 {
 190         preempt_disable();
 191         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 192         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 193 }
 194
 195 EXPORT_SYMBOL(_write_lock);
 196
 197 #else /* CONFIG_PREEMPT: */
 198
 199 /*
 200  * This could be a long-held lock. We both prepare to spin for a long
 201  * time (making _this_ CPU preemptable if possible), and we also signal
 202  * towards that other CPU that it should break the lock ASAP.
 203  *
 204  * (We do this in a function because inlining it would be excessive.)
 205  */
 206
 207 #define BUILD_LOCK_OPS(op, locktype)                                    \
 208 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 209 {                                                                       \
 210         for (;;) {                                                      \
 211                 preempt_disable();                                      \
 212                 if (likely(_raw_##op##_trylock(lock)))                  \
 213                         break;                                          \
 214                 preempt_enable();                                       \
 215                                                                         \
 216                 if (!(lock)->break_lock)                                \
 217                         (lock)->break_lock = 1;                         \
 218                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 219                         _raw_##op##_relax(&lock->raw_lock);             \
 220         }                                                               \
 221         (lock)->break_lock = 0;                                         \
 222 }                                                                       \
 223                                                                         \
 224 EXPORT_SYMBOL(_##op##_lock);                                            \
 225                                                                         \
 226 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 227 {                                                                       \
 228         unsigned long flags;                                            \
 229                                                                         \
 230         for (;;) {                                                      \
 231                 preempt_disable();                                      \
 232                 local_irq_save(flags);                                  \
 233                 if (likely(_raw_##op##_trylock(lock)))                  \
 234                         break;                                          \
 235                 local_irq_restore(flags);                               \
 236                 preempt_enable();                                       \
 237                                                                         \
 238                 if (!(lock)->break_lock)                                \
 239                         (lock)->break_lock = 1;                         \
 240                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 241                         _raw_##op##_relax(&lock->raw_lock);             \
 242         }                                                               \
 243         (lock)->break_lock = 0;                                         \
 244         return flags;                                                   \
 245 }                                                                       \
 246                                                                         \
 247 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 248                                                                         \
 249 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 250 {                                                                       \
 251         _##op##_lock_irqsave(lock);                                     \
 252 }                                                                       \
 253                                                                         \
 254 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 255                                                                         \
 256 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 257 {                                                                       \
 258         unsigned long flags;                                            \
 259                                                                         \
 260         /*                                                      */      \
 261         /* Careful: we must exclude softirqs too, hence the     */      \
 262         /* irq-disabling. We use the generic preemption-aware   */      \
 263         /* function:                                            */      \
 264         /**/                                                            \
 265         flags = _##op##_lock_irqsave(lock);                             \
 266         local_bh_disable();                                             \
 267         local_irq_restore(flags);                                       \
 268 }                                                                       \
 269                                                                         \
 270 EXPORT_SYMBOL(_##op##_lock_bh)
 271
 272 /*
 273  * Build preemption-friendly versions of the following
 274  * lock-spinning functions:
 275  *
 276  *         _[spin|read|write]_lock()
 277  *         _[spin|read|write]_lock_irq()
 278  *         _[spin|read|write]_lock_irqsave()
 279  *         _[spin|read|write]_lock_bh()
 280  */
 281 BUILD_LOCK_OPS(spin, spinlock);
 282 BUILD_LOCK_OPS(read, rwlock);
 283 BUILD_LOCK_OPS(write, rwlock);
 284
 285 #endif /* CONFIG_PREEMPT */
 286
 287 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 288
 289 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 290 {
 291         preempt_disable();
 292         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 293         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 294 }
 295 EXPORT_SYMBOL(_spin_lock_nested);
 296
 297 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
 298 {
 299         unsigned long flags;
 300
 301         local_irq_save(flags);
 302         preempt_disable();
 303         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 304         LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
 305                                 _raw_spin_lock_flags, &flags);
 306         return flags;
 307 }
 308 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
 309
 310 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
 311                                      struct lockdep_map *nest_lock)
 312 {
 313         preempt_disable();
 314         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 315         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 316 }
 317 EXPORT_SYMBOL(_spin_lock_nest_lock);
 318
 319 #endif
 320
 321 void __lockfunc _spin_unlock(spinlock_t *lock)
 322 {
 323         spin_release(&lock->dep_map, 1, _RET_IP_);
 324         _raw_spin_unlock(lock);
 325         preempt_enable();
 326 }
 327 EXPORT_SYMBOL(_spin_unlock);
 328
 329 void __lockfunc _write_unlock(rwlock_t *lock)
 330 {
 331         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 332         _raw_write_unlock(lock);
 333         preempt_enable();
 334 }
 335 EXPORT_SYMBOL(_write_unlock);
 336
 337 void __lockfunc _read_unlock(rwlock_t *lock)
 338 {
 339         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 340         _raw_read_unlock(lock);
 341         preempt_enable();
 342 }
 343 EXPORT_SYMBOL(_read_unlock);
 344
 345 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 346 {
 347         spin_release(&lock->dep_map, 1, _RET_IP_);
 348         _raw_spin_unlock(lock);
 349         local_irq_restore(flags);
 350         preempt_enable();
 351 }
 352 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 353
 354 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 355 {
 356         spin_release(&lock->dep_map, 1, _RET_IP_);
 357         _raw_spin_unlock(lock);
 358         local_irq_enable();
 359         preempt_enable();
 360 }
 361 EXPORT_SYMBOL(_spin_unlock_irq);
 362
 363 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 364 {
 365         spin_release(&lock->dep_map, 1, _RET_IP_);
 366         _raw_spin_unlock(lock);
 367         preempt_enable_no_resched();
 368         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 369 }
 370 EXPORT_SYMBOL(_spin_unlock_bh);
 371
 372 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 373 {
 374         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 375         _raw_read_unlock(lock);
 376         local_irq_restore(flags);
 377         preempt_enable();
 378 }
 379 EXPORT_SYMBOL(_read_unlock_irqrestore);
 380
 381 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 382 {
 383         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 384         _raw_read_unlock(lock);
 385         local_irq_enable();
 386         preempt_enable();
 387 }
 388 EXPORT_SYMBOL(_read_unlock_irq);
 389
 390 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 391 {
 392         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 393         _raw_read_unlock(lock);
 394         preempt_enable_no_resched();
 395         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 396 }
 397 EXPORT_SYMBOL(_read_unlock_bh);
 398
 399 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 400 {
 401         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 402         _raw_write_unlock(lock);
 403         local_irq_restore(flags);
 404         preempt_enable();
 405 }
 406 EXPORT_SYMBOL(_write_unlock_irqrestore);
 407
 408 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 409 {
 410         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 411         _raw_write_unlock(lock);
 412         local_irq_enable();
 413         preempt_enable();
 414 }
 415 EXPORT_SYMBOL(_write_unlock_irq);
 416
 417 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 418 {
 419         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 420         _raw_write_unlock(lock);
 421         preempt_enable_no_resched();
 422         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 423 }
 424 EXPORT_SYMBOL(_write_unlock_bh);
 425
 426 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 427 {
 428         local_bh_disable();
 429         preempt_disable();
 430         if (_raw_spin_trylock(lock)) {
 431                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 432                 return 1;
 433         }
 434
 435         preempt_enable_no_resched();
 436         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 437         return 0;
 438 }
 439 EXPORT_SYMBOL(_spin_trylock_bh);
 440
 441 notrace int in_lock_functions(unsigned long addr)
 442 {
 443         /* Linker adds these: start and end of __lockfunc functions */
 444         extern char __lock_text_start[], __lock_text_end[];
 445
 446         return addr >= (unsigned long)__lock_text_start
 447         && addr < (unsigned long)__lock_text_end;
 448 }
 449 EXPORT_SYMBOL(in_lock_functions);