fs: use kmem_cache_zalloc instead
[pv_ops_mirror.git] / include / asm-x86 / rwsem.h
blob041906f3c6df73375abda597ba38e63922ffb542
1 /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+
3 * Written by David Howells (dhowells@redhat.com).
5 * Derived from asm-i386/semaphore.h
8 * The MSW of the count is the negated number of active writers and waiting
9 * lockers, and the LSW is the total number of active locks
11 * The lock count is initialized to 0 (no active and no waiting lockers).
13 * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an
14 * uncontended lock. This can be determined because XADD returns the old value.
15 * Readers increment by 1 and see a positive value when uncontended, negative
16 * if there are writers (and maybe) readers waiting (in which case it goes to
17 * sleep).
19 * The value of WAITING_BIAS supports up to 32766 waiting processes. This can
20 * be extended to 65534 by manually checking the whole MSW rather than relying
21 * on the S flag.
23 * The value of ACTIVE_BIAS supports up to 65535 active processes.
25 * This should be totally fair - if anything is waiting, a process that wants a
26 * lock will go to the back of the queue. When the currently active lock is
27 * released, if there's a writer at the front of the queue, then that and only
28 * that will be woken up; if there's a bunch of consequtive readers at the
29 * front, then they'll all be woken up, but no other readers will be.
32 #ifndef _I386_RWSEM_H
33 #define _I386_RWSEM_H
35 #ifndef _LINUX_RWSEM_H
36 #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
37 #endif
39 #ifdef __KERNEL__
41 #include <linux/list.h>
42 #include <linux/spinlock.h>
43 #include <linux/lockdep.h>
45 struct rwsem_waiter;
47 extern struct rw_semaphore *FASTCALL(rwsem_down_read_failed(struct rw_semaphore *sem));
48 extern struct rw_semaphore *FASTCALL(rwsem_down_write_failed(struct rw_semaphore *sem));
49 extern struct rw_semaphore *FASTCALL(rwsem_wake(struct rw_semaphore *));
50 extern struct rw_semaphore *FASTCALL(rwsem_downgrade_wake(struct rw_semaphore *sem));
53 * the semaphore definition
55 struct rw_semaphore {
56 signed long count;
57 #define RWSEM_UNLOCKED_VALUE 0x00000000
58 #define RWSEM_ACTIVE_BIAS 0x00000001
59 #define RWSEM_ACTIVE_MASK 0x0000ffff
60 #define RWSEM_WAITING_BIAS (-0x00010000)
61 #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
62 #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
63 spinlock_t wait_lock;
64 struct list_head wait_list;
65 #ifdef CONFIG_DEBUG_LOCK_ALLOC
66 struct lockdep_map dep_map;
67 #endif
70 #ifdef CONFIG_DEBUG_LOCK_ALLOC
71 # define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname }
72 #else
73 # define __RWSEM_DEP_MAP_INIT(lockname)
74 #endif
77 #define __RWSEM_INITIALIZER(name) \
78 { RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \
79 LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) }
81 #define DECLARE_RWSEM(name) \
82 struct rw_semaphore name = __RWSEM_INITIALIZER(name)
84 extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
85 struct lock_class_key *key);
87 #define init_rwsem(sem) \
88 do { \
89 static struct lock_class_key __key; \
91 __init_rwsem((sem), #sem, &__key); \
92 } while (0)
95 * lock for reading
97 static inline void __down_read(struct rw_semaphore *sem)
99 __asm__ __volatile__(
100 "# beginning down_read\n\t"
101 LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
102 " jns 1f\n"
103 " call call_rwsem_down_read_failed\n"
104 "1:\n\t"
105 "# ending down_read\n\t"
106 : "+m" (sem->count)
107 : "a" (sem)
108 : "memory", "cc");
112 * trylock for reading -- returns 1 if successful, 0 if contention
114 static inline int __down_read_trylock(struct rw_semaphore *sem)
116 __s32 result, tmp;
117 __asm__ __volatile__(
118 "# beginning __down_read_trylock\n\t"
119 " movl %0,%1\n\t"
120 "1:\n\t"
121 " movl %1,%2\n\t"
122 " addl %3,%2\n\t"
123 " jle 2f\n\t"
124 LOCK_PREFIX " cmpxchgl %2,%0\n\t"
125 " jnz 1b\n\t"
126 "2:\n\t"
127 "# ending __down_read_trylock\n\t"
128 : "+m" (sem->count), "=&a" (result), "=&r" (tmp)
129 : "i" (RWSEM_ACTIVE_READ_BIAS)
130 : "memory", "cc");
131 return result>=0 ? 1 : 0;
135 * lock for writing
137 static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
139 int tmp;
141 tmp = RWSEM_ACTIVE_WRITE_BIAS;
142 __asm__ __volatile__(
143 "# beginning down_write\n\t"
144 LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
145 " testl %%edx,%%edx\n\t" /* was the count 0 before? */
146 " jz 1f\n"
147 " call call_rwsem_down_write_failed\n"
148 "1:\n"
149 "# ending down_write"
150 : "+m" (sem->count), "=d" (tmp)
151 : "a" (sem), "1" (tmp)
152 : "memory", "cc");
155 static inline void __down_write(struct rw_semaphore *sem)
157 __down_write_nested(sem, 0);
161 * trylock for writing -- returns 1 if successful, 0 if contention
163 static inline int __down_write_trylock(struct rw_semaphore *sem)
165 signed long ret = cmpxchg(&sem->count,
166 RWSEM_UNLOCKED_VALUE,
167 RWSEM_ACTIVE_WRITE_BIAS);
168 if (ret == RWSEM_UNLOCKED_VALUE)
169 return 1;
170 return 0;
174 * unlock after reading
176 static inline void __up_read(struct rw_semaphore *sem)
178 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
179 __asm__ __volatile__(
180 "# beginning __up_read\n\t"
181 LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
182 " jns 1f\n\t"
183 " call call_rwsem_wake\n"
184 "1:\n"
185 "# ending __up_read\n"
186 : "+m" (sem->count), "=d" (tmp)
187 : "a" (sem), "1" (tmp)
188 : "memory", "cc");
192 * unlock after writing
194 static inline void __up_write(struct rw_semaphore *sem)
196 __asm__ __volatile__(
197 "# beginning __up_write\n\t"
198 " movl %2,%%edx\n\t"
199 LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
200 " jz 1f\n"
201 " call call_rwsem_wake\n"
202 "1:\n\t"
203 "# ending __up_write\n"
204 : "+m" (sem->count)
205 : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS)
206 : "memory", "cc", "edx");
210 * downgrade write lock to read lock
212 static inline void __downgrade_write(struct rw_semaphore *sem)
214 __asm__ __volatile__(
215 "# beginning __downgrade_write\n\t"
216 LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
217 " jns 1f\n\t"
218 " call call_rwsem_downgrade_wake\n"
219 "1:\n\t"
220 "# ending __downgrade_write\n"
221 : "+m" (sem->count)
222 : "a" (sem), "i" (-RWSEM_WAITING_BIAS)
223 : "memory", "cc");
227 * implement atomic add functionality
229 static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
231 __asm__ __volatile__(
232 LOCK_PREFIX "addl %1,%0"
233 : "+m" (sem->count)
234 : "ir" (delta));
238 * implement exchange and add functionality
240 static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
242 int tmp = delta;
244 __asm__ __volatile__(
245 LOCK_PREFIX "xadd %0,%1"
246 : "+r" (tmp), "+m" (sem->count)
247 : : "memory");
249 return tmp+delta;
252 static inline int rwsem_is_locked(struct rw_semaphore *sem)
254 return (sem->count != 0);
257 #endif /* __KERNEL__ */
258 #endif /* _I386_RWSEM_H */