lib/asyncsafe-spin.c

   1 /* Spin locks for communication between threads and signal handlers.
   2    Copyright (C) 2020-2024 Free Software Foundation, Inc.
   3
   4    This file is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU Lesser General Public License as
   6    published by the Free Software Foundation; either version 2.1 of the
   7    License, or (at your option) any later version.
   8
   9    This file is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public License
  15    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  16
  17 /* Written by Bruno Haible <bruno@clisp.org>, 2020.  */
  18
  19 #include <config.h>
  20
  21 /* Specification.  */
  22 #include "asyncsafe-spin.h"
  23
  24 #include <stdlib.h>
  25 #if defined _AIX
  26 # include <sys/atomic_op.h>
  27 #endif
  28 #if 0x590 <= __SUNPRO_C && __STDC__
  29 # define asm __asm
  30 #endif
  31
  32 #if defined _WIN32 && ! defined __CYGWIN__
  33 /* Use Windows threads.  */
  34
  35 void
  36 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
  37 {
  38   glwthread_spin_init (lock);
  39 }
  40
  41 static inline void
  42 do_lock (asyncsafe_spinlock_t *lock)
  43 {
  44   glwthread_spin_lock (lock);
  45 }
  46
  47 static inline void
  48 do_unlock (asyncsafe_spinlock_t *lock)
  49 {
  50   if (glwthread_spin_unlock (lock))
  51     abort ();
  52 }
  53
  54 void
  55 asyncsafe_spin_destroy (asyncsafe_spinlock_t *lock)
  56 {
  57   glwthread_spin_destroy (lock);
  58 }
  59
  60 #else
  61
  62 # if HAVE_PTHREAD_H
  63 /* Use POSIX threads.  */
  64
  65 /* We don't use semaphores (although sem_post() is allowed in signal handlers),
  66    because it would require to link with -lrt on HP-UX 11, OSF/1, Solaris 10,
  67    and also because on macOS only named semaphores work.
  68
  69    We don't use the C11 <stdatomic.h> (available in GCC >= 4.9) because it would
  70    require to link with -latomic.  */
  71
  72 #  if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7) \
  73        || __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 1)) \
  74       && !defined __ibmxl__
  75 /* Use GCC built-ins (available in GCC >= 4.7 and clang >= 3.1) that operate on
  76    the first byte of the lock.
  77    Documentation:
  78    <https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/_005f_005fatomic-Builtins.html>
  79  */
  80
  81 #   if 1
  82 /* An implementation that verifies the unlocks.  */
  83
  84 void
  85 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
  86 {
  87   __atomic_store_n (lock, 0, __ATOMIC_SEQ_CST);
  88 }
  89
  90 static inline void
  91 do_lock (asyncsafe_spinlock_t *lock)
  92 {
  93   /* Wait until *lock becomes 0, then replace it with 1.  */
  94   asyncsafe_spinlock_t zero;
  95   while (!(zero = 0,
  96            __atomic_compare_exchange_n (lock, &zero, 1, false,
  97                                         __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)))
  98     ;
  99 }
 100
 101 static inline void
 102 do_unlock (asyncsafe_spinlock_t *lock)
 103 {
 104   /* If *lock is 1, then replace it with 0.  */
 105   asyncsafe_spinlock_t one = 1;
 106   if (!__atomic_compare_exchange_n (lock, &one, 0, false,
 107                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
 108     abort ();
 109 }
 110
 111 #   else
 112 /* An implementation that is a little bit more optimized, but does not verify
 113    the unlocks.  */
 114
 115 void
 116 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 117 {
 118   __atomic_clear (lock, __ATOMIC_SEQ_CST);
 119 }
 120
 121 static inline void
 122 do_lock (asyncsafe_spinlock_t *lock)
 123 {
 124   while (__atomic_test_and_set (lock, __ATOMIC_SEQ_CST))
 125     ;
 126 }
 127
 128 static inline void
 129 do_unlock (asyncsafe_spinlock_t *lock)
 130 {
 131   __atomic_clear (lock, __ATOMIC_SEQ_CST);
 132 }
 133
 134 #   endif
 135
 136 #  elif (((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) \
 137           || __clang_major__ >= 3) \
 138          && HAVE_ATOMIC_COMPARE_AND_SWAP_GCC41)
 139 /* Use GCC built-ins (available on many platforms with GCC >= 4.1 or
 140    clang >= 3.0).
 141    Documentation:
 142    <https://gcc.gnu.org/onlinedocs/gcc-4.1.2/gcc/Atomic-Builtins.html>  */
 143
 144 void
 145 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 146 {
 147   volatile unsigned int *vp = lock;
 148   *vp = 0;
 149   __sync_synchronize ();
 150 }
 151
 152 static inline void
 153 do_lock (asyncsafe_spinlock_t *lock)
 154 {
 155   /* Wait until *lock becomes 0, then replace it with 1.  */
 156   while (__sync_val_compare_and_swap (lock, 0, 1) != 0)
 157     ;
 158 }
 159
 160 static inline void
 161 do_unlock (asyncsafe_spinlock_t *lock)
 162 {
 163   /* If *lock is 1, then replace it with 0.  */
 164   if (__sync_val_compare_and_swap (lock, 1, 0) != 1)
 165     abort ();
 166 }
 167
 168 #  elif defined _AIX
 169 /* AIX */
 170
 171 void
 172 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 173 {
 174   atomic_p vp = (int *) lock;
 175   _clear_lock (vp, 0);
 176 }
 177
 178 static inline void
 179 do_lock (asyncsafe_spinlock_t *lock)
 180 {
 181   atomic_p vp = (int *) lock;
 182   while (_check_lock (vp, 0, 1))
 183     ;
 184 }
 185
 186 static inline void
 187 do_unlock (asyncsafe_spinlock_t *lock)
 188 {
 189   atomic_p vp = (int *) lock;
 190   if (_check_lock (vp, 1, 0))
 191     abort ();
 192 }
 193
 194 #  elif ((defined __GNUC__ || defined __clang__ || defined __SUNPRO_C) && (defined __sparc || defined __i386 || defined __x86_64__)) || (defined __TINYC__ && (defined __i386 || defined __x86_64__))
 195 /* For older versions of GCC or clang, use inline assembly.
 196    GCC, clang, and the Oracle Studio C 12 compiler understand GCC's extended
 197    asm syntax, but the plain Oracle Studio C 11 compiler understands only
 198    simple asm.  */
 199 /* An implementation that verifies the unlocks.  */
 200
 201 static void
 202 memory_barrier (void)
 203 {
 204 #   if defined __GNUC__ || defined __clang__ || __SUNPRO_C >= 0x590 || defined __TINYC__
 205 #    if defined __i386 || defined __x86_64__
 206 #     if defined __TINYC__ && defined __i386
 207   /* Cannot use the SSE instruction "mfence" with this compiler.  */
 208   asm volatile ("lock orl $0,(%esp)");
 209 #     else
 210   asm volatile ("mfence");
 211 #     endif
 212 #    endif
 213 #    if defined __sparc
 214   asm volatile ("membar 2");
 215 #    endif
 216 #   else
 217 #    if defined __i386 || defined __x86_64__
 218   asm ("mfence");
 219 #    endif
 220 #    if defined __sparc
 221   asm ("membar 2");
 222 #    endif
 223 #   endif
 224 }
 225
 226 /* Store NEWVAL in *VP if the old value *VP is == CMP.
 227    Return the old value.  */
 228 static unsigned int
 229 atomic_compare_and_swap (volatile unsigned int *vp, unsigned int cmp,
 230                          unsigned int newval)
 231 {
 232 #   if defined __GNUC__ || defined __clang__ || __SUNPRO_C >= 0x590 || defined __TINYC__
 233   unsigned int oldval;
 234 #    if defined __i386 || defined __x86_64__
 235   asm volatile (" lock\n cmpxchgl %3,(%1)"
 236                 : "=a" (oldval) : "r" (vp), "a" (cmp), "r" (newval) : "memory");
 237 #    endif
 238 #    if defined __sparc
 239   asm volatile (" cas [%1],%2,%3\n"
 240                 " mov %3,%0"
 241                 : "=r" (oldval) : "r" (vp), "r" (cmp), "r" (newval) : "memory");
 242 #    endif
 243   return oldval;
 244 #   else /* __SUNPRO_C */
 245 #    if defined __x86_64__
 246   asm (" movl %esi,%eax\n"
 247        " lock\n cmpxchgl %edx,(%rdi)");
 248 #    elif defined __i386
 249   asm (" movl 16(%ebp),%ecx\n"
 250        " movl 12(%ebp),%eax\n"
 251        " movl 8(%ebp),%edx\n"
 252        " lock\n cmpxchgl %ecx,(%edx)");
 253 #    endif
 254 #    if defined __sparc
 255   asm (" cas [%i0],%i1,%i2\n"
 256        " mov %i2,%i0");
 257 #    endif
 258 #   endif
 259 }
 260
 261 void
 262 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 263 {
 264   volatile unsigned int *vp = lock;
 265   *vp = 0;
 266   memory_barrier ();
 267 }
 268
 269 static inline void
 270 do_lock (asyncsafe_spinlock_t *lock)
 271 {
 272   volatile unsigned int *vp = lock;
 273   while (atomic_compare_and_swap (vp, 0, 1) != 0)
 274     ;
 275 }
 276
 277 static inline void
 278 do_unlock (asyncsafe_spinlock_t *lock)
 279 {
 280   volatile unsigned int *vp = lock;
 281   if (atomic_compare_and_swap (vp, 1, 0) != 1)
 282     abort ();
 283 }
 284
 285 #  else
 286 /* Fallback code.  It has some race conditions.  */
 287
 288 void
 289 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 290 {
 291   volatile unsigned int *vp = lock;
 292   *vp = 0;
 293 }
 294
 295 static inline void
 296 do_lock (asyncsafe_spinlock_t *lock)
 297 {
 298   volatile unsigned int *vp = lock;
 299   while (*vp)
 300     ;
 301   *vp = 1;
 302 }
 303
 304 static inline void
 305 do_unlock (asyncsafe_spinlock_t *lock)
 306 {
 307   volatile unsigned int *vp = lock;
 308   *vp = 0;
 309 }
 310
 311 #  endif
 312
 313 # else
 314 /* Provide a dummy implementation for single-threaded applications.  */
 315
 316 void
 317 asyncsafe_spin_init (asyncsafe_spinlock_t *lock)
 318 {
 319 }
 320
 321 static inline void
 322 do_lock (asyncsafe_spinlock_t *lock)
 323 {
 324 }
 325
 326 static inline void
 327 do_unlock (asyncsafe_spinlock_t *lock)
 328 {
 329 }
 330
 331 # endif
 332
 333 void
 334 asyncsafe_spin_destroy (asyncsafe_spinlock_t *lock)
 335 {
 336 }
 337
 338 #endif
 339
 340 void
 341 asyncsafe_spin_lock (asyncsafe_spinlock_t *lock,
 342                      const sigset_t *mask, sigset_t *saved_mask)
 343 {
 344   sigprocmask (SIG_BLOCK, mask, saved_mask); /* equivalent to pthread_sigmask */
 345   do_lock (lock);
 346 }
 347
 348 void
 349 asyncsafe_spin_unlock (asyncsafe_spinlock_t *lock, const sigset_t *saved_mask)
 350 {
 351   do_unlock (lock);
 352   sigprocmask (SIG_SETMASK, saved_mask, NULL); /* equivalent to pthread_sigmask */
 353 }