sys/arch/x86/include/lock.h

   1 /*      $NetBSD: lock.h,v 1.27 2013/01/22 22:09:44 christos Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Jason R. Thorpe and Andrew Doran.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 /*
  33  * Machine-dependent spin lock operations.
  34  */
  35
  36 #ifndef _X86_LOCK_H_
  37 #define _X86_LOCK_H_
  38
  39 #include <sys/param.h>
  40
  41 static __inline int
  42 __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr)
  43 {
  44         return *__ptr == __SIMPLELOCK_LOCKED;
  45 }
  46
  47 static __inline int
  48 __SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t *__ptr)
  49 {
  50         return *__ptr == __SIMPLELOCK_UNLOCKED;
  51 }
  52
  53 static __inline void
  54 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
  55 {
  56
  57         *__ptr = __SIMPLELOCK_LOCKED;
  58 }
  59
  60 static __inline void
  61 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
  62 {
  63
  64         *__ptr = __SIMPLELOCK_UNLOCKED;
  65 }
  66
  67 #ifdef _HARDKERNEL
  68 # include <machine/cpufunc.h>
  69 # define SPINLOCK_SPIN_HOOK     /* nothing */
  70 # ifdef SPINLOCK_BACKOFF_HOOK
  71 #  undef SPINLOCK_BACKOFF_HOOK
  72 # endif
  73 # define SPINLOCK_BACKOFF_HOOK  x86_pause()
  74 # define SPINLOCK_INLINE
  75 #else /* !_HARDKERNEL */
  76 # define SPINLOCK_BODY
  77 # define SPINLOCK_INLINE static __inline __unused
  78 #endif /* _HARDKERNEL */
  79
  80 SPINLOCK_INLINE void    __cpu_simple_lock_init(__cpu_simple_lock_t *);
  81 SPINLOCK_INLINE void    __cpu_simple_lock(__cpu_simple_lock_t *);
  82 SPINLOCK_INLINE int     __cpu_simple_lock_try(__cpu_simple_lock_t *);
  83 SPINLOCK_INLINE void    __cpu_simple_unlock(__cpu_simple_lock_t *);
  84
  85 #ifdef SPINLOCK_BODY
  86 SPINLOCK_INLINE void
  87 __cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
  88 {
  89
  90         *lockp = __SIMPLELOCK_UNLOCKED;
  91         __insn_barrier();
  92 }
  93
  94 SPINLOCK_INLINE int
  95 __cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
  96 {
  97         uint8_t val;
  98
  99         val = __SIMPLELOCK_LOCKED;
 100         __asm volatile ("xchgb %0,(%2)" :
 101             "=qQ" (val)
 102             :"0" (val), "r" (lockp));
 103         __insn_barrier();
 104         return val == __SIMPLELOCK_UNLOCKED;
 105 }
 106
 107 SPINLOCK_INLINE void
 108 __cpu_simple_lock(__cpu_simple_lock_t *lockp)
 109 {
 110
 111         while (!__cpu_simple_lock_try(lockp))
 112                 /* nothing */;
 113         __insn_barrier();
 114 }
 115
 116 /*
 117  * Note on x86 memory ordering
 118  *
 119  * When releasing a lock we must ensure that no stores or loads from within
 120  * the critical section are re-ordered by the CPU to occur outside of it:
 121  * they must have completed and be visible to other processors once the lock
 122  * has been released.
 123  *
 124  * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
 125  * back) memory region.  In that case, memory ordering on x86 platforms
 126  * looks like this:
 127  *
 128  * i386         All loads/stores occur in instruction sequence.
 129  *
 130  * i486         All loads/stores occur in instruction sequence.  In
 131  * Pentium      exceptional circumstances, loads can be re-ordered around
 132  *              stores, but for the purposes of releasing a lock it does
 133  *              not matter.  Stores may not be immediately visible to other
 134  *              processors as they can be buffered.  However, since the
 135  *              stores are buffered in order the lock release will always be
 136  *              the last operation in the critical section that becomes
 137  *              visible to other CPUs.
 138  *
 139  * Pentium Pro  The "Intel 64 and IA-32 Architectures Software Developer's
 140  * onwards      Manual" volume 3A (order number 248966) says that (1) "Reads
 141  *              can be carried out speculatively and in any order" and (2)
 142  *              "Reads can pass buffered stores, but the processor is
 143  *              self-consistent.".  This would be a problem for the below,
 144  *              and would mandate a locked instruction cycle or load fence
 145  *              before releasing the simple lock.
 146  *
 147  *              The "Intel Pentium 4 Processor Optimization" guide (order
 148  *              number 253668-022US) says: "Loads can be moved before stores
 149  *              that occurred earlier in the program if they are not
 150  *              predicted to load from the same linear address.".  This is
 151  *              not a problem since the only loads that can be re-ordered
 152  *              take place once the lock has been released via a store.
 153  *
 154  *              The above two documents seem to contradict each other,
 155  *              however with the exception of early steppings of the Pentium
 156  *              Pro, the second document is closer to the truth: a store
 157  *              will always act as a load fence for all loads that precede
 158  *              the store in instruction order.
 159  *
 160  *              Again, note that stores can be buffered and will not always
 161  *              become immediately visible to other CPUs: they are however
 162  *              buffered in order.
 163  *
 164  * AMD64        Stores occur in order and are buffered.  Loads can be
 165  *              reordered, however stores act as load fences, meaning that
 166  *              loads can not be reordered around stores.
 167  */
 168 SPINLOCK_INLINE void
 169 __cpu_simple_unlock(__cpu_simple_lock_t *lockp)
 170 {
 171
 172         __insn_barrier();
 173         *lockp = __SIMPLELOCK_UNLOCKED;
 174 }
 175
 176 #endif  /* SPINLOCK_BODY */
 177
 178 #endif /* _X86_LOCK_H_ */