headers/private/kernel/smp.h

   1 /*
   2  * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de.
   3  * Distributed under the terms of the MIT License.
   4  *
   5  * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
   6  * Distributed under the terms of the NewOS License.
   7  */
   8 #ifndef KERNEL_SMP_H
   9 #define KERNEL_SMP_H
  10
  11
  12 #include <arch/atomic.h>
  13 #include <boot/kernel_args.h>
  14 #include <kernel.h>
  15
  16 #include <KernelExport.h>
  17
  18 #include <string.h>
  19
  20
  21 struct kernel_args;
  22
  23
  24 // intercpu messages
  25 enum {
  26         SMP_MSG_INVALIDATE_PAGE_RANGE = 0,
  27         SMP_MSG_INVALIDATE_PAGE_LIST,
  28         SMP_MSG_USER_INVALIDATE_PAGES,
  29         SMP_MSG_GLOBAL_INVALIDATE_PAGES,
  30         SMP_MSG_CPU_HALT,
  31         SMP_MSG_CALL_FUNCTION,
  32         SMP_MSG_RESCHEDULE
  33 };
  34
  35 enum {
  36         SMP_MSG_FLAG_ASYNC              = 0x0,
  37         SMP_MSG_FLAG_SYNC               = 0x1,
  38         SMP_MSG_FLAG_FREE_ARG   = 0x2,
  39 };
  40
  41 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
  42
  43 class CPUSet {
  44 public:
  45         inline                          CPUSet();
  46
  47         inline  void            ClearAll();
  48         inline  void            SetAll();
  49
  50         inline  void            SetBit(int32 cpu);
  51         inline  void            ClearBit(int32 cpu);
  52
  53         inline  void            SetBitAtomic(int32 cpu);
  54         inline  void            ClearBitAtomic(int32 cpu);
  55
  56         inline  bool            GetBit(int32 cpu) const;
  57
  58         inline  bool            IsEmpty() const;
  59
  60 private:
  61         static  const int       kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
  62
  63                         uint32          fBitmap[kArraySize];
  64 };
  65
  66
  67 #ifdef __cplusplus
  68 extern "C" {
  69 #endif
  70
  71 bool try_acquire_spinlock(spinlock* lock);
  72
  73 status_t smp_init(struct kernel_args *args);
  74 status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
  75 status_t smp_init_post_generic_syscalls(void);
  76 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
  77 void smp_wake_up_non_boot_cpus(void);
  78 void smp_cpu_rendezvous(uint32* var);
  79 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
  80                 void *data_ptr, uint32 flags);
  81 void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
  82                 addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
  83 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
  84                 void *data_ptr, uint32 flags);
  85 void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
  86                 addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
  87
  88 int32 smp_get_num_cpus(void);
  89 void smp_set_num_cpus(int32 numCPUs);
  90 int32 smp_get_current_cpu(void);
  91
  92 int smp_intercpu_int_handler(int32 cpu);
  93
  94 #ifdef __cplusplus
  95 }
  96 #endif
  97
  98
  99 inline
 100 CPUSet::CPUSet()
 101 {
 102         memset(fBitmap, 0, sizeof(fBitmap));
 103 }
 104
 105
 106 inline void
 107 CPUSet::ClearAll()
 108 {
 109         memset(fBitmap, 0, sizeof(fBitmap));
 110 }
 111
 112
 113 inline void
 114 CPUSet::SetAll()
 115 {
 116         memset(fBitmap, ~uint8(0), sizeof(fBitmap));
 117 }
 118
 119
 120 inline void
 121 CPUSet::SetBit(int32 cpu)
 122 {
 123         int32* element = (int32*)&fBitmap[cpu % kArraySize];
 124         *element |= 1u << (cpu / kArraySize);
 125 }
 126
 127
 128 inline void
 129 CPUSet::ClearBit(int32 cpu)
 130 {
 131         int32* element = (int32*)&fBitmap[cpu % kArraySize];
 132         *element &= ~uint32(1u << (cpu / kArraySize));
 133 }
 134
 135
 136 inline void
 137 CPUSet::SetBitAtomic(int32 cpu)
 138 {
 139         int32* element = (int32*)&fBitmap[cpu % kArraySize];
 140         atomic_or(element, 1u << (cpu / kArraySize));
 141 }
 142
 143
 144 inline void
 145 CPUSet::ClearBitAtomic(int32 cpu)
 146 {
 147         int32* element = (int32*)&fBitmap[cpu % kArraySize];
 148         atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
 149 }
 150
 151
 152 inline bool
 153 CPUSet::GetBit(int32 cpu) const
 154 {
 155         int32* element = (int32*)&fBitmap[cpu % kArraySize];
 156         return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
 157 }
 158
 159
 160 inline bool
 161 CPUSet::IsEmpty() const
 162 {
 163         for (int i = 0; i < kArraySize; i++) {
 164                 if (fBitmap[i] != 0)
 165                         return false;
 166         }
 167
 168         return true;
 169 }
 170
 171
 172 // Unless spinlock debug features are enabled, try to inline
 173 // {acquire,release}_spinlock().
 174 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
 175
 176
 177 static inline bool
 178 try_acquire_spinlock_inline(spinlock* lock)
 179 {
 180         return atomic_get_and_set((int32*)lock, 1) == 0;
 181 }
 182
 183
 184 static inline void
 185 acquire_spinlock_inline(spinlock* lock)
 186 {
 187         if (try_acquire_spinlock_inline(lock))
 188                 return;
 189         acquire_spinlock(lock);
 190 }
 191
 192
 193 static inline void
 194 release_spinlock_inline(spinlock* lock)
 195 {
 196         atomic_set((int32*)lock, 0);
 197 }
 198
 199
 200 #define try_acquire_spinlock(lock)      try_acquire_spinlock_inline(lock)
 201 #define acquire_spinlock(lock)          acquire_spinlock_inline(lock)
 202 #define release_spinlock(lock)          release_spinlock_inline(lock)
 203
 204
 205 static inline bool
 206 try_acquire_write_spinlock_inline(rw_spinlock* lock)
 207 {
 208         return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
 209 }
 210
 211
 212 static inline void
 213 acquire_write_spinlock_inline(rw_spinlock* lock)
 214 {
 215         if (try_acquire_write_spinlock(lock))
 216                 return;
 217         acquire_write_spinlock(lock);
 218 }
 219
 220
 221 static inline void
 222 release_write_spinlock_inline(rw_spinlock* lock)
 223 {
 224         atomic_set(&lock->lock, 0);
 225 }
 226
 227
 228 static inline bool
 229 try_acquire_read_spinlock_inline(rw_spinlock* lock)
 230 {
 231         uint32 previous = atomic_add(&lock->lock, 1);
 232         return (previous & (1u << 31)) == 0;
 233 }
 234
 235
 236 static inline void
 237 acquire_read_spinlock_inline(rw_spinlock* lock)
 238 {
 239         if (try_acquire_read_spinlock(lock))
 240                 return;
 241         acquire_read_spinlock(lock);
 242 }
 243
 244
 245 static inline void
 246 release_read_spinlock_inline(rw_spinlock* lock)
 247 {
 248         atomic_add(&lock->lock, -1);
 249 }
 250
 251
 252 #define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock)
 253 #define acquire_read_spinlock(lock)             acquire_read_spinlock_inline(lock)
 254 #define release_read_spinlock(lock)             release_read_spinlock_inline(lock)
 255 #define try_acquire_write_spinlock(lock) \
 256         try_acquire_write_spinlock(lock)
 257 #define acquire_write_spinlock(lock)    acquire_write_spinlock_inline(lock)
 258 #define release_write_spinlock(lock)    release_write_spinlock_inline(lock)
 259
 260
 261 static inline bool
 262 try_acquire_write_seqlock_inline(seqlock* lock) {
 263         bool succeed = try_acquire_spinlock(&lock->lock);
 264         if (succeed)
 265                 atomic_add((int32*)&lock->count, 1);
 266         return succeed;
 267 }
 268
 269
 270 static inline void
 271 acquire_write_seqlock_inline(seqlock* lock) {
 272         acquire_spinlock(&lock->lock);
 273         atomic_add((int32*)&lock->count, 1);
 274 }
 275
 276
 277 static inline void
 278 release_write_seqlock_inline(seqlock* lock) {
 279         atomic_add((int32*)&lock->count, 1);
 280         release_spinlock(&lock->lock);
 281 }
 282
 283
 284 static inline uint32
 285 acquire_read_seqlock_inline(seqlock* lock) {
 286         return atomic_get((int32*)&lock->count);
 287 }
 288
 289
 290 static inline bool
 291 release_read_seqlock_inline(seqlock* lock, uint32 count) {
 292         uint32 current = atomic_get((int32*)&lock->count);
 293
 294         return count % 2 == 0 && current == count;
 295 }
 296
 297
 298 #define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock)
 299 #define acquire_write_seqlock(lock)             acquire_write_seqlock_inline(lock)
 300 #define release_write_seqlock(lock)             release_write_seqlock_inline(lock)
 301 #define acquire_read_seqlock(lock)              acquire_read_seqlock_inline(lock)
 302 #define release_read_seqlock(lock, count)       \
 303         release_read_seqlock_inline(lock, count)
 304
 305
 306 #endif  // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
 307
 308
 309 #endif  /* KERNEL_SMP_H */