include/asm-x86/mutex_32.h

   1 /*
   2  * Assembly implementation of the mutex fastpath, based on atomic
   3  * decrement/increment.
   4  *
   5  * started by Ingo Molnar:
   6  *
   7  *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
   8  */
   9 #ifndef _ASM_MUTEX_H
  10 #define _ASM_MUTEX_H
  11
  12 #include "asm/alternative.h"
  13
  14 /**
  15  *  __mutex_fastpath_lock - try to take the lock by moving the count
  16  *                          from 1 to a 0 value
  17  *  @count: pointer of type atomic_t
  18  *  @fn: function to call if the original value was not 1
  19  *
  20  * Change the count from 1 to a value lower than 1, and call <fn> if it
  21  * wasn't 1 originally. This function MUST leave the value lower than 1
  22  * even when the "1" assertion wasn't true.
  23  */
  24 #define __mutex_fastpath_lock(count, fail_fn)                           \
  25 do {                                                                    \
  26         unsigned int dummy;                                             \
  27                                                                         \
  28         typecheck(atomic_t *, count);                                   \
  29         typecheck_fn(void (*)(atomic_t *), fail_fn);            \
  30                                                                         \
  31         __asm__ __volatile__(                                           \
  32                 LOCK_PREFIX "   decl (%%eax)    \n"                     \
  33                         "   jns 1f              \n"                     \
  34                         "   call "#fail_fn"     \n"                     \
  35                         "1:                     \n"                     \
  36                                                                         \
  37                 :"=a" (dummy)                                           \
  38                 : "a" (count)                                           \
  39                 : "memory", "ecx", "edx");                              \
  40 } while (0)
  41
  42
  43 /**
  44  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  45  *                                 from 1 to a 0 value
  46  *  @count: pointer of type atomic_t
  47  *  @fail_fn: function to call if the original value was not 1
  48  *
  49  * Change the count from 1 to a value lower than 1, and call <fail_fn> if it
  50  * wasn't 1 originally. This function returns 0 if the fastpath succeeds,
  51  * or anything the slow path function returns
  52  */
  53 static inline int
  54 __mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
  55 {
  56         if (unlikely(atomic_dec_return(count) < 0))
  57                 return fail_fn(count);
  58         else
  59                 return 0;
  60 }
  61
  62 /**
  63  *  __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
  64  *  @count: pointer of type atomic_t
  65  *  @fail_fn: function to call if the original value was not 0
  66  *
  67  * try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>.
  68  * In the failure case, this function is allowed to either set the value
  69  * to 1, or to set it to a value lower than 1.
  70  *
  71  * If the implementation sets it to a value of lower than 1, the
  72  * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
  73  * to return 0 otherwise.
  74  */
  75 #define __mutex_fastpath_unlock(count, fail_fn)                         \
  76 do {                                                                    \
  77         unsigned int dummy;                                             \
  78                                                                         \
  79         typecheck(atomic_t *, count);                                   \
  80         typecheck_fn(void (*)(atomic_t *), fail_fn);            \
  81                                                                         \
  82         __asm__ __volatile__(                                           \
  83                 LOCK_PREFIX "   incl (%%eax)    \n"                     \
  84                         "   jg  1f              \n"                     \
  85                         "   call "#fail_fn"     \n"                     \
  86                         "1:                     \n"                     \
  87                                                                         \
  88                 :"=a" (dummy)                                           \
  89                 : "a" (count)                                           \
  90                 : "memory", "ecx", "edx");                              \
  91 } while (0)
  92
  93 #define __mutex_slowpath_needs_to_unlock()      1
  94
  95 /**
  96  * __mutex_fastpath_trylock - try to acquire the mutex, without waiting
  97  *
  98  *  @count: pointer of type atomic_t
  99  *  @fail_fn: fallback function
 100  *
 101  * Change the count from 1 to a value lower than 1, and return 0 (failure)
 102  * if it wasn't 1 originally, or return 1 (success) otherwise. This function
 103  * MUST leave the value lower than 1 even when the "1" assertion wasn't true.
 104  * Additionally, if the value was < 0 originally, this function must not leave
 105  * it to 0 on failure.
 106  */
 107 static inline int
 108 __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
 109 {
 110         /*
 111          * We have two variants here. The cmpxchg based one is the best one
 112          * because it never induce a false contention state.  It is included
 113          * here because architectures using the inc/dec algorithms over the
 114          * xchg ones are much more likely to support cmpxchg natively.
 115          *
 116          * If not we fall back to the spinlock based variant - that is
 117          * just as efficient (and simpler) as a 'destructive' probing of
 118          * the mutex state would be.
 119          */
 120 #ifdef __HAVE_ARCH_CMPXCHG
 121         if (likely(atomic_cmpxchg(count, 1, 0) == 1))
 122                 return 1;
 123         return 0;
 124 #else
 125         return fail_fn(count);
 126 #endif
 127 }
 128
 129 #endif