1 #ifndef _ASM_X86_SPINLOCK_H
2 #define _ASM_X86_SPINLOCK_H
4 #include <linux/jump_label.h>
5 #include <linux/atomic.h>
7 #include <asm/processor.h>
8 #include <linux/compiler.h>
9 #include <asm/paravirt.h>
10 #include <asm/bitops.h>
13 * Your basic SMP spinlocks, allowing only a single CPU anywhere
15 * Simple spin lock operations. There are two variants, one clears IRQ's
16 * on the local processor, one does not.
18 * These are fair FIFO ticket locks, which support up to 2^16 CPUs.
20 * (the type definitions are in asm/spinlock_types.h)
24 # define LOCK_PTR_REG "a"
26 # define LOCK_PTR_REG "D"
29 #if defined(CONFIG_X86_32) && (defined(CONFIG_X86_PPRO_FENCE))
31 * On PPro SMP, we use a locked operation to unlock
32 * (PPro errata 66, 92)
34 # define UNLOCK_LOCK_PREFIX LOCK_PREFIX
36 # define UNLOCK_LOCK_PREFIX
39 /* How long a lock should spin before we consider blocking */
40 #define SPIN_THRESHOLD (1 << 15)
42 extern struct static_key paravirt_ticketlocks_enabled
;
43 static __always_inline
bool static_key_false(struct static_key
*key
);
45 #ifdef CONFIG_QUEUED_SPINLOCKS
46 #include <asm/qspinlock.h>
49 #ifdef CONFIG_PARAVIRT_SPINLOCKS
51 static inline void __ticket_enter_slowpath(arch_spinlock_t
*lock
)
53 set_bit(0, (volatile unsigned long *)&lock
->tickets
.head
);
56 #else /* !CONFIG_PARAVIRT_SPINLOCKS */
57 static __always_inline
void __ticket_lock_spinning(arch_spinlock_t
*lock
,
61 static inline void __ticket_unlock_kick(arch_spinlock_t
*lock
,
66 #endif /* CONFIG_PARAVIRT_SPINLOCKS */
67 static inline int __tickets_equal(__ticket_t one
, __ticket_t two
)
69 return !((one
^ two
) & ~TICKET_SLOWPATH_FLAG
);
72 static inline void __ticket_check_and_clear_slowpath(arch_spinlock_t
*lock
,
75 if (head
& TICKET_SLOWPATH_FLAG
) {
76 arch_spinlock_t old
, new;
78 old
.tickets
.head
= head
;
79 new.tickets
.head
= head
& ~TICKET_SLOWPATH_FLAG
;
80 old
.tickets
.tail
= new.tickets
.head
+ TICKET_LOCK_INC
;
81 new.tickets
.tail
= old
.tickets
.tail
;
83 /* try to clear slowpath flag when there are no contenders */
84 cmpxchg(&lock
->head_tail
, old
.head_tail
, new.head_tail
);
88 static __always_inline
int arch_spin_value_unlocked(arch_spinlock_t lock
)
90 return __tickets_equal(lock
.tickets
.head
, lock
.tickets
.tail
);
94 * Ticket locks are conceptually two parts, one indicating the current head of
95 * the queue, and the other indicating the current tail. The lock is acquired
96 * by atomically noting the tail and incrementing it by one (thus adding
97 * ourself to the queue and noting our position), then waiting until the head
98 * becomes equal to the the initial value of the tail.
100 * We use an xadd covering *both* parts of the lock, to increment the tail and
101 * also load the position of the head, which takes care of memory ordering
102 * issues and should be optimal for the uncontended case. Note the tail must be
103 * in the high part, because a wide xadd increment of the low part would carry
104 * up and contaminate the high part.
106 static __always_inline
void arch_spin_lock(arch_spinlock_t
*lock
)
108 register struct __raw_tickets inc
= { .tail
= TICKET_LOCK_INC
};
110 inc
= xadd(&lock
->tickets
, inc
);
111 if (likely(inc
.head
== inc
.tail
))
115 unsigned count
= SPIN_THRESHOLD
;
118 inc
.head
= READ_ONCE(lock
->tickets
.head
);
119 if (__tickets_equal(inc
.head
, inc
.tail
))
123 __ticket_lock_spinning(lock
, inc
.tail
);
126 __ticket_check_and_clear_slowpath(lock
, inc
.head
);
128 barrier(); /* make sure nothing creeps before the lock is taken */
131 static __always_inline
int arch_spin_trylock(arch_spinlock_t
*lock
)
133 arch_spinlock_t old
, new;
135 old
.tickets
= READ_ONCE(lock
->tickets
);
136 if (!__tickets_equal(old
.tickets
.head
, old
.tickets
.tail
))
139 new.head_tail
= old
.head_tail
+ (TICKET_LOCK_INC
<< TICKET_SHIFT
);
140 new.head_tail
&= ~TICKET_SLOWPATH_FLAG
;
142 /* cmpxchg is a full barrier, so nothing can move before it */
143 return cmpxchg(&lock
->head_tail
, old
.head_tail
, new.head_tail
) == old
.head_tail
;
146 static __always_inline
void arch_spin_unlock(arch_spinlock_t
*lock
)
148 if (TICKET_SLOWPATH_FLAG
&&
149 static_key_false(¶virt_ticketlocks_enabled
)) {
152 BUILD_BUG_ON(((__ticket_t
)NR_CPUS
) != NR_CPUS
);
154 head
= xadd(&lock
->tickets
.head
, TICKET_LOCK_INC
);
156 if (unlikely(head
& TICKET_SLOWPATH_FLAG
)) {
157 head
&= ~TICKET_SLOWPATH_FLAG
;
158 __ticket_unlock_kick(lock
, (head
+ TICKET_LOCK_INC
));
161 __add(&lock
->tickets
.head
, TICKET_LOCK_INC
, UNLOCK_LOCK_PREFIX
);
164 static inline int arch_spin_is_locked(arch_spinlock_t
*lock
)
166 struct __raw_tickets tmp
= READ_ONCE(lock
->tickets
);
168 return !__tickets_equal(tmp
.tail
, tmp
.head
);
171 static inline int arch_spin_is_contended(arch_spinlock_t
*lock
)
173 struct __raw_tickets tmp
= READ_ONCE(lock
->tickets
);
175 tmp
.head
&= ~TICKET_SLOWPATH_FLAG
;
176 return (__ticket_t
)(tmp
.tail
- tmp
.head
) > TICKET_LOCK_INC
;
178 #define arch_spin_is_contended arch_spin_is_contended
180 static __always_inline
void arch_spin_lock_flags(arch_spinlock_t
*lock
,
183 arch_spin_lock(lock
);
186 static inline void arch_spin_unlock_wait(arch_spinlock_t
*lock
)
188 __ticket_t head
= READ_ONCE(lock
->tickets
.head
);
191 struct __raw_tickets tmp
= READ_ONCE(lock
->tickets
);
193 * We need to check "unlocked" in a loop, tmp.head == head
194 * can be false positive because of overflow.
196 if (__tickets_equal(tmp
.head
, tmp
.tail
) ||
197 !__tickets_equal(tmp
.head
, head
))
203 #endif /* CONFIG_QUEUED_SPINLOCKS */
206 * Read-write spinlocks, allowing multiple readers
207 * but only one writer.
209 * NOTE! it is quite common to have readers in interrupts
210 * but no interrupt writers. For those circumstances we
211 * can "mix" irq-safe locks - any writer needs to get a
212 * irq-safe write-lock, but readers can get non-irqsafe
215 * On x86, we implement read-write locks using the generic qrwlock with
216 * x86 specific optimization.
219 #include <asm/qrwlock.h>
221 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
222 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
224 #define arch_spin_relax(lock) cpu_relax()
225 #define arch_read_relax(lock) cpu_relax()
226 #define arch_write_relax(lock) cpu_relax()
228 #endif /* _ASM_X86_SPINLOCK_H */