src/system/kernel/smp.cpp

   1 /*
   2  * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
   3  * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de.
   4  * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
   5  * Distributed under the terms of the MIT License.
   6  *
   7  * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
   8  * Distributed under the terms of the NewOS License.
   9  */
  10
  11
  12 /*! Functionality for symetrical multi-processors */
  13
  14
  15 #include <smp.h>
  16
  17 #include <stdlib.h>
  18 #include <string.h>
  19
  20 #include <arch/atomic.h>
  21 #include <arch/cpu.h>
  22 #include <arch/debug.h>
  23 #include <arch/int.h>
  24 #include <arch/smp.h>
  25 #include <boot/kernel_args.h>
  26 #include <cpu.h>
  27 #include <generic_syscall.h>
  28 #include <int.h>
  29 #include <spinlock_contention.h>
  30 #include <thread.h>
  31 #include <util/atomic.h>
  32 #if DEBUG_SPINLOCK_LATENCIES
  33 #       include <safemode.h>
  34 #endif
  35
  36 #include "kernel_debug_config.h"
  37
  38
  39 //#define TRACE_SMP
  40 #ifdef TRACE_SMP
  41 #       define TRACE(...) dprintf_no_syslog(__VA_ARGS__)
  42 #else
  43 #       define TRACE(...) (void)0
  44 #endif
  45
  46
  47 #undef try_acquire_spinlock
  48 #undef acquire_spinlock
  49 #undef release_spinlock
  50
  51 #undef try_acquire_read_spinlock
  52 #undef acquire_read_spinlock
  53 #undef release_read_spinlock
  54 #undef try_acquire_write_spinlock
  55 #undef acquire_write_spinlock
  56 #undef release_write_spinlock
  57
  58 #undef try_acquire_write_seqlock
  59 #undef acquire_write_seqlock
  60 #undef release_write_seqlock
  61 #undef acquire_read_seqlock
  62 #undef release_read_seqlock
  63
  64
  65 #define MSG_POOL_SIZE (SMP_MAX_CPUS * 4)
  66
  67 // These macros define the number of unsuccessful iterations in
  68 // acquire_spinlock() and acquire_spinlock_nocheck() after which the functions
  69 // panic(), assuming a deadlock.
  70 #define SPINLOCK_DEADLOCK_COUNT                         100000000
  71 #define SPINLOCK_DEADLOCK_COUNT_NO_CHECK        2000000000
  72
  73
  74 struct smp_msg {
  75         struct smp_msg  *next;
  76         int32                   message;
  77         addr_t                  data;
  78         addr_t                  data2;
  79         addr_t                  data3;
  80         void                    *data_ptr;
  81         uint32                  flags;
  82         int32                   ref_count;
  83         int32                   done;
  84         CPUSet                  proc_bitmap;
  85 };
  86
  87 enum mailbox_source {
  88         MAILBOX_LOCAL,
  89         MAILBOX_BCAST,
  90 };
  91
  92 static int32 sBootCPUSpin = 0;
  93
  94 static int32 sEarlyCPUCallCount;
  95 static CPUSet sEarlyCPUCallSet;
  96 static void (*sEarlyCPUCallFunction)(void*, int);
  97 void* sEarlyCPUCallCookie;
  98
  99 static struct smp_msg* sFreeMessages = NULL;
 100 static int32 sFreeMessageCount = 0;
 101 static spinlock sFreeMessageSpinlock = B_SPINLOCK_INITIALIZER;
 102
 103 static struct smp_msg* sCPUMessages[SMP_MAX_CPUS] = { NULL, };
 104
 105 static struct smp_msg* sBroadcastMessages = NULL;
 106 static spinlock sBroadcastMessageSpinlock = B_SPINLOCK_INITIALIZER;
 107 static int32 sBroadcastMessageCounter;
 108
 109 static bool sICIEnabled = false;
 110 static int32 sNumCPUs = 1;
 111
 112 static int32 process_pending_ici(int32 currentCPU);
 113
 114
 115 #if DEBUG_SPINLOCKS
 116 #define NUM_LAST_CALLERS        32
 117
 118 static struct {
 119         void            *caller;
 120         spinlock        *lock;
 121 } sLastCaller[NUM_LAST_CALLERS];
 122
 123 static int32 sLastIndex = 0;
 124         // Is incremented atomically. Must be % NUM_LAST_CALLERS before being used
 125         // as index into sLastCaller. Note, that it has to be casted to uint32
 126         // before applying the modulo operation, since otherwise after overflowing
 127         // that would yield negative indices.
 128
 129
 130 static void
 131 push_lock_caller(void* caller, spinlock* lock)
 132 {
 133         int32 index = (uint32)atomic_add(&sLastIndex, 1) % NUM_LAST_CALLERS;
 134
 135         sLastCaller[index].caller = caller;
 136         sLastCaller[index].lock = lock;
 137 }
 138
 139
 140 static void*
 141 find_lock_caller(spinlock* lock)
 142 {
 143         int32 lastIndex = (uint32)atomic_get(&sLastIndex) % NUM_LAST_CALLERS;
 144
 145         for (int32 i = 0; i < NUM_LAST_CALLERS; i++) {
 146                 int32 index = (NUM_LAST_CALLERS + lastIndex - 1 - i) % NUM_LAST_CALLERS;
 147                 if (sLastCaller[index].lock == lock)
 148                         return sLastCaller[index].caller;
 149         }
 150
 151         return NULL;
 152 }
 153
 154
 155 int
 156 dump_spinlock(int argc, char** argv)
 157 {
 158         if (argc != 2) {
 159                 print_debugger_command_usage(argv[0]);
 160                 return 0;
 161         }
 162
 163         uint64 address;
 164         if (!evaluate_debug_expression(argv[1], &address, false))
 165                 return 0;
 166
 167         spinlock* lock = (spinlock*)(addr_t)address;
 168         kprintf("spinlock %p:\n", lock);
 169         bool locked = B_SPINLOCK_IS_LOCKED(lock);
 170         if (locked) {
 171                 kprintf("  locked from %p\n", find_lock_caller(lock));
 172         } else
 173                 kprintf("  not locked\n");
 174
 175         return 0;
 176 }
 177
 178
 179 #endif  // DEBUG_SPINLOCKS
 180
 181
 182 #if DEBUG_SPINLOCK_LATENCIES
 183
 184
 185 #define NUM_LATENCY_LOCKS       4
 186 #define DEBUG_LATENCY           200
 187
 188
 189 static struct {
 190         spinlock        *lock;
 191         bigtime_t       timestamp;
 192 } sLatency[SMP_MAX_CPUS][NUM_LATENCY_LOCKS];
 193
 194 static int32 sLatencyIndex[SMP_MAX_CPUS];
 195 static bool sEnableLatencyCheck;
 196
 197
 198 static void
 199 push_latency(spinlock* lock)
 200 {
 201         if (!sEnableLatencyCheck)
 202                 return;
 203
 204         int32 cpu = smp_get_current_cpu();
 205         int32 index = (++sLatencyIndex[cpu]) % NUM_LATENCY_LOCKS;
 206
 207         sLatency[cpu][index].lock = lock;
 208         sLatency[cpu][index].timestamp = system_time();
 209 }
 210
 211
 212 static void
 213 test_latency(spinlock* lock)
 214 {
 215         if (!sEnableLatencyCheck)
 216                 return;
 217
 218         int32 cpu = smp_get_current_cpu();
 219
 220         for (int32 i = 0; i < NUM_LATENCY_LOCKS; i++) {
 221                 if (sLatency[cpu][i].lock == lock) {
 222                         bigtime_t diff = system_time() - sLatency[cpu][i].timestamp;
 223                         if (diff > DEBUG_LATENCY && diff < 500000) {
 224                                 panic("spinlock %p was held for %lld usecs (%d allowed)\n",
 225                                         lock, diff, DEBUG_LATENCY);
 226                         }
 227
 228                         sLatency[cpu][i].lock = NULL;
 229                 }
 230         }
 231 }
 232
 233
 234 #endif  // DEBUG_SPINLOCK_LATENCIES
 235
 236
 237 int
 238 dump_ici_messages(int argc, char** argv)
 239 {
 240         // count broadcast messages
 241         int32 count = 0;
 242         int32 doneCount = 0;
 243         int32 unreferencedCount = 0;
 244         smp_msg* message = sBroadcastMessages;
 245         while (message != NULL) {
 246                 count++;
 247                 if (message->done == 1)
 248                         doneCount++;
 249                 if (message->ref_count <= 0)
 250                         unreferencedCount++;
 251                 message = message->next;
 252         }
 253
 254         kprintf("ICI broadcast messages: %" B_PRId32 ", first: %p\n", count,
 255                 sBroadcastMessages);
 256         kprintf("  done:         %" B_PRId32 "\n", doneCount);
 257         kprintf("  unreferenced: %" B_PRId32 "\n", unreferencedCount);
 258
 259         // count per-CPU messages
 260         for (int32 i = 0; i < sNumCPUs; i++) {
 261                 count = 0;
 262                 message = sCPUMessages[i];
 263                 while (message != NULL) {
 264                         count++;
 265                         message = message->next;
 266                 }
 267
 268                 kprintf("CPU %" B_PRId32 " messages: %" B_PRId32 ", first: %p\n", i,
 269                         count, sCPUMessages[i]);
 270         }
 271
 272         return 0;
 273 }
 274
 275
 276 int
 277 dump_ici_message(int argc, char** argv)
 278 {
 279         if (argc != 2) {
 280                 print_debugger_command_usage(argv[0]);
 281                 return 0;
 282         }
 283
 284         uint64 address;
 285         if (!evaluate_debug_expression(argv[1], &address, false))
 286                 return 0;
 287
 288         smp_msg* message = (smp_msg*)(addr_t)address;
 289         kprintf("ICI message %p:\n", message);
 290         kprintf("  next:        %p\n", message->next);
 291         kprintf("  message:     %" B_PRId32 "\n", message->message);
 292         kprintf("  data:        0x%lx\n", message->data);
 293         kprintf("  data2:       0x%lx\n", message->data2);
 294         kprintf("  data3:       0x%lx\n", message->data3);
 295         kprintf("  data_ptr:    %p\n", message->data_ptr);
 296         kprintf("  flags:       %" B_PRIx32 "\n", message->flags);
 297         kprintf("  ref_count:   %" B_PRIx32 "\n", message->ref_count);
 298         kprintf("  done:        %s\n", message->done == 1 ? "true" : "false");
 299
 300         kprintf("  proc_bitmap: ");
 301         for (int32 i = 0; i < sNumCPUs; i++) {
 302                 if (message->proc_bitmap.GetBit(i))
 303                         kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i);
 304         }
 305         kprintf("\n");
 306
 307         return 0;
 308 }
 309
 310
 311 static inline void
 312 process_all_pending_ici(int32 currentCPU)
 313 {
 314         while (process_pending_ici(currentCPU) != B_ENTRY_NOT_FOUND)
 315                 ;
 316 }
 317
 318
 319 bool
 320 try_acquire_spinlock(spinlock* lock)
 321 {
 322 #if DEBUG_SPINLOCKS
 323         if (are_interrupts_enabled()) {
 324                 panic("try_acquire_spinlock: attempt to acquire lock %p with "
 325                         "interrupts enabled", lock);
 326         }
 327 #endif
 328
 329 #if B_DEBUG_SPINLOCK_CONTENTION
 330         if (atomic_add(&lock->lock, 1) != 0)
 331                 return false;
 332 #else
 333         if (atomic_get_and_set((int32*)lock, 1) != 0)
 334                 return false;
 335
 336 #       if DEBUG_SPINLOCKS
 337         push_lock_caller(arch_debug_get_caller(), lock);
 338 #       endif
 339 #endif
 340
 341         return true;
 342 }
 343
 344
 345 void
 346 acquire_spinlock(spinlock* lock)
 347 {
 348 #if DEBUG_SPINLOCKS
 349         if (are_interrupts_enabled()) {
 350                 panic("acquire_spinlock: attempt to acquire lock %p with interrupts "
 351                         "enabled", lock);
 352         }
 353 #endif
 354
 355         if (sNumCPUs > 1) {
 356                 int currentCPU = smp_get_current_cpu();
 357 #if B_DEBUG_SPINLOCK_CONTENTION
 358                 while (atomic_add(&lock->lock, 1) != 0)
 359                         process_all_pending_ici(currentCPU);
 360 #else
 361                 while (1) {
 362                         uint32 count = 0;
 363                         while (lock->lock != 0) {
 364                                 if (++count == SPINLOCK_DEADLOCK_COUNT) {
 365 #       if DEBUG_SPINLOCKS
 366                                         panic("acquire_spinlock(): Failed to acquire spinlock %p "
 367                                                 "for a long time (last caller: %p, value: %" B_PRIx32
 368                                                 ")", lock, find_lock_caller(lock), lock->lock);
 369 #       else
 370                                         panic("acquire_spinlock(): Failed to acquire spinlock %p "
 371                                                 "for a long time (value: %" B_PRIx32 ")", lock,
 372                                                 lock->lock);
 373 #       endif
 374                                         count = 0;
 375                                 }
 376
 377                                 process_all_pending_ici(currentCPU);
 378                                 cpu_wait(&lock->lock, 0);
 379                         }
 380                         if (atomic_get_and_set(&lock->lock, 1) == 0)
 381                                 break;
 382                 }
 383
 384 #       if DEBUG_SPINLOCKS
 385                 push_lock_caller(arch_debug_get_caller(), lock);
 386 #       endif
 387 #endif
 388         } else {
 389 #if DEBUG_SPINLOCKS
 390                 int32 oldValue = atomic_get_and_set(&lock->lock, 1);
 391                 if (oldValue != 0) {
 392                         panic("acquire_spinlock: attempt to acquire lock %p twice on "
 393                                 "non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock,
 394                                 find_lock_caller(lock), oldValue);
 395                 }
 396
 397                 push_lock_caller(arch_debug_get_caller(), lock);
 398 #endif
 399         }
 400 #if DEBUG_SPINLOCK_LATENCIES
 401         push_latency(lock);
 402 #endif
 403 }
 404
 405
 406 static void
 407 acquire_spinlock_nocheck(spinlock *lock)
 408 {
 409 #if DEBUG_SPINLOCKS
 410         if (are_interrupts_enabled()) {
 411                 panic("acquire_spinlock_nocheck: attempt to acquire lock %p with "
 412                         "interrupts enabled", lock);
 413         }
 414 #endif
 415
 416         if (sNumCPUs > 1) {
 417 #if B_DEBUG_SPINLOCK_CONTENTION
 418                 while (atomic_add(&lock->lock, 1) != 0) {
 419                 }
 420 #else
 421                 while (1) {
 422                         uint32 count = 0;
 423                         while (lock->lock != 0) {
 424                                 if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) {
 425 #       if DEBUG_SPINLOCKS
 426                                         panic("acquire_spinlock_nocheck(): Failed to acquire "
 427                                                 "spinlock %p for a long time (last caller: %p, value: %"
 428                                                 B_PRIx32 ")", lock, find_lock_caller(lock), lock->lock);
 429 #       else
 430                                         panic("acquire_spinlock_nocheck(): Failed to acquire "
 431                                                 "spinlock %p for a long time (value: %" B_PRIx32 ")",
 432                                                 lock, lock->lock);
 433 #       endif
 434                                         count = 0;
 435                                 }
 436
 437                                 cpu_wait(&lock->lock, 0);
 438                         }
 439
 440                         if (atomic_get_and_set(&lock->lock, 1) == 0)
 441                                 break;
 442                 }
 443
 444 #       if DEBUG_SPINLOCKS
 445                 push_lock_caller(arch_debug_get_caller(), lock);
 446 #       endif
 447 #endif
 448         } else {
 449 #if DEBUG_SPINLOCKS
 450                 int32 oldValue = atomic_get_and_set(&lock->lock, 1);
 451                 if (oldValue != 0) {
 452                         panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice "
 453                                 "on non-SMP system (last caller: %p, value %" B_PRIx32 ")",
 454                                 lock, find_lock_caller(lock), oldValue);
 455                 }
 456
 457                 push_lock_caller(arch_debug_get_caller(), lock);
 458 #endif
 459         }
 460 }
 461
 462
 463 /*!     Equivalent to acquire_spinlock(), save for currentCPU parameter. */
 464 static void
 465 acquire_spinlock_cpu(int32 currentCPU, spinlock *lock)
 466 {
 467 #if DEBUG_SPINLOCKS
 468         if (are_interrupts_enabled()) {
 469                 panic("acquire_spinlock_cpu: attempt to acquire lock %p with "
 470                         "interrupts enabled", lock);
 471         }
 472 #endif
 473
 474         if (sNumCPUs > 1) {
 475 #if B_DEBUG_SPINLOCK_CONTENTION
 476                 while (atomic_add(&lock->lock, 1) != 0)
 477                         process_all_pending_ici(currentCPU);
 478 #else
 479                 while (1) {
 480                         uint32 count = 0;
 481                         while (lock->lock != 0) {
 482                                 if (++count == SPINLOCK_DEADLOCK_COUNT) {
 483 #       if DEBUG_SPINLOCKS
 484                                         panic("acquire_spinlock_cpu(): Failed to acquire spinlock "
 485                                                 "%p for a long time (last caller: %p, value: %" B_PRIx32
 486                                                 ")", lock, find_lock_caller(lock), lock->lock);
 487 #       else
 488                                         panic("acquire_spinlock_cpu(): Failed to acquire spinlock "
 489                                                 "%p for a long time (value: %" B_PRIx32 ")", lock,
 490                                                 lock->lock);
 491 #       endif
 492                                         count = 0;
 493                                 }
 494
 495                                 process_all_pending_ici(currentCPU);
 496                                 cpu_wait(&lock->lock, 0);
 497                         }
 498                         if (atomic_get_and_set(&lock->lock, 1) == 0)
 499                                 break;
 500                 }
 501
 502 #       if DEBUG_SPINLOCKS
 503                 push_lock_caller(arch_debug_get_caller(), lock);
 504 #       endif
 505 #endif
 506         } else {
 507 #if DEBUG_SPINLOCKS
 508                 int32 oldValue = atomic_get_and_set(&lock->lock, 1);
 509                 if (oldValue != 0) {
 510                         panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on "
 511                                 "non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock,
 512                                 find_lock_caller(lock), oldValue);
 513                 }
 514
 515                 push_lock_caller(arch_debug_get_caller(), lock);
 516 #endif
 517         }
 518 }
 519
 520
 521 void
 522 release_spinlock(spinlock *lock)
 523 {
 524 #if DEBUG_SPINLOCK_LATENCIES
 525         test_latency(lock);
 526 #endif
 527
 528         if (sNumCPUs > 1) {
 529                 if (are_interrupts_enabled())
 530                         panic("release_spinlock: attempt to release lock %p with "
 531                                 "interrupts enabled\n", lock);
 532 #if B_DEBUG_SPINLOCK_CONTENTION
 533                 {
 534                         int32 count = atomic_and(&lock->lock, 0) - 1;
 535                         if (count < 0) {
 536                                 panic("release_spinlock: lock %p was already released\n", lock);
 537                         } else {
 538                                 // add to the total count -- deal with carry manually
 539                                 if ((uint32)atomic_add(&lock->count_low, count) + count
 540                                                 < (uint32)count) {
 541                                         atomic_add(&lock->count_high, 1);
 542                                 }
 543                         }
 544                 }
 545 #elif DEBUG_SPINLOCKS
 546                 if (atomic_get_and_set(&lock->lock, 0) != 1)
 547                         panic("release_spinlock: lock %p was already released\n", lock);
 548 #else
 549                 atomic_set(&lock->lock, 0);
 550 #endif
 551         } else {
 552 #if DEBUG_SPINLOCKS
 553                 if (are_interrupts_enabled()) {
 554                         panic("release_spinlock: attempt to release lock %p with "
 555                                 "interrupts enabled\n", lock);
 556                 }
 557                 if (atomic_get_and_set(&lock->lock, 0) != 1)
 558                         panic("release_spinlock: lock %p was already released\n", lock);
 559 #endif
 560 #if DEBUG_SPINLOCK_LATENCIES
 561                 test_latency(lock);
 562 #endif
 563         }
 564 }
 565
 566
 567 bool
 568 try_acquire_write_spinlock(rw_spinlock* lock)
 569 {
 570 #if DEBUG_SPINLOCKS
 571         if (are_interrupts_enabled()) {
 572                 panic("try_acquire_write_spinlock: attempt to acquire lock %p with "
 573                         "interrupts enabled", lock);
 574         }
 575
 576         if (sNumCPUs < 2 && lock->lock != 0) {
 577                 panic("try_acquire_write_spinlock(): attempt to acquire lock %p twice "
 578                         "on non-SMP system", lock);
 579         }
 580 #endif
 581
 582         return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
 583 }
 584
 585
 586 void
 587 acquire_write_spinlock(rw_spinlock* lock)
 588 {
 589 #if DEBUG_SPINLOCKS
 590         if (are_interrupts_enabled()) {
 591                 panic("acquire_write_spinlock: attempt to acquire lock %p with "
 592                         "interrupts enabled", lock);
 593         }
 594 #endif
 595
 596         uint32 count = 0;
 597         int currentCPU = smp_get_current_cpu();
 598         while (true) {
 599                 if (try_acquire_write_spinlock(lock))
 600                         break;
 601
 602                 while (lock->lock != 0) {
 603                         if (++count == SPINLOCK_DEADLOCK_COUNT) {
 604                                 panic("acquire_write_spinlock(): Failed to acquire spinlock %p "
 605                                         "for a long time!", lock);
 606                                 count = 0;
 607                         }
 608
 609                         process_all_pending_ici(currentCPU);
 610                         cpu_wait(&lock->lock, 0);
 611                 }
 612         }
 613 }
 614
 615
 616 void
 617 release_write_spinlock(rw_spinlock* lock)
 618 {
 619 #if DEBUG_SPINLOCKS
 620         uint32 previous = atomic_get_and_set(&lock->lock, 0);
 621         if ((previous & 1u << 31) == 0) {
 622                 panic("release_write_spinlock: lock %p was already released (value: "
 623                         "%#" B_PRIx32 ")\n", lock, previous);
 624         }
 625 #else
 626         atomic_set(&lock->lock, 0);
 627 #endif
 628 }
 629
 630
 631 bool
 632 try_acquire_read_spinlock(rw_spinlock* lock)
 633 {
 634 #if DEBUG_SPINLOCKS
 635         if (are_interrupts_enabled()) {
 636                 panic("try_acquire_read_spinlock: attempt to acquire lock %p with "
 637                         "interrupts enabled", lock);
 638         }
 639
 640         if (sNumCPUs < 2 && lock->lock != 0) {
 641                 panic("try_acquire_read_spinlock(): attempt to acquire lock %p twice "
 642                         "on non-SMP system", lock);
 643         }
 644 #endif
 645
 646         uint32 previous = atomic_add(&lock->lock, 1);
 647         return (previous & (1u << 31)) == 0;
 648 }
 649
 650
 651 void
 652 acquire_read_spinlock(rw_spinlock* lock)
 653 {
 654 #if DEBUG_SPINLOCKS
 655         if (are_interrupts_enabled()) {
 656                 panic("acquire_read_spinlock: attempt to acquire lock %p with "
 657                         "interrupts enabled", lock);
 658         }
 659 #endif
 660
 661         uint32 count = 0;
 662         int currentCPU = smp_get_current_cpu();
 663         while (1) {
 664                 if (try_acquire_read_spinlock(lock))
 665                         break;
 666
 667                 while ((lock->lock & (1u << 31)) != 0) {
 668                         if (++count == SPINLOCK_DEADLOCK_COUNT) {
 669                                 panic("acquire_read_spinlock(): Failed to acquire spinlock %p "
 670                                         "for a long time!", lock);
 671                                 count = 0;
 672                         }
 673
 674                         process_all_pending_ici(currentCPU);
 675                         cpu_wait(&lock->lock, 0);
 676                 }
 677         }
 678 }
 679
 680
 681 void
 682 release_read_spinlock(rw_spinlock* lock)
 683 {
 684 #if DEBUG_SPINLOCKS
 685         uint32 previous = atomic_add(&lock->lock, -1);
 686         if ((previous & 1u << 31) != 0) {
 687                 panic("release_read_spinlock: lock %p was already released (value:"
 688                         " %#" B_PRIx32 ")\n", lock, previous);
 689         }
 690 #else
 691         atomic_add(&lock->lock, -1);
 692 #endif
 693
 694 }
 695
 696
 697 bool
 698 try_acquire_write_seqlock(seqlock* lock) {
 699         bool succeed = try_acquire_spinlock(&lock->lock);
 700         if (succeed)
 701                 atomic_add((int32*)&lock->count, 1);
 702         return succeed;
 703 }
 704
 705
 706 void
 707 acquire_write_seqlock(seqlock* lock) {
 708         acquire_spinlock(&lock->lock);
 709         atomic_add((int32*)&lock->count, 1);
 710 }
 711
 712
 713 void
 714 release_write_seqlock(seqlock* lock) {
 715         atomic_add((int32*)&lock->count, 1);
 716         release_spinlock(&lock->lock);
 717 }
 718
 719
 720 uint32
 721 acquire_read_seqlock(seqlock* lock) {
 722         return atomic_get((int32*)&lock->count);
 723 }
 724
 725
 726 bool
 727 release_read_seqlock(seqlock* lock, uint32 count) {
 728         memory_read_barrier();
 729
 730         uint32 current = *(volatile int32*)&lock->count;
 731
 732         if (count % 2 == 1 || current != count) {
 733                 cpu_pause();
 734                 return false;
 735         }
 736
 737         return true;
 738 }
 739
 740
 741 /*!     Finds a free message and gets it.
 742         NOTE: has side effect of disabling interrupts
 743         return value is the former interrupt state
 744 */
 745 static cpu_status
 746 find_free_message(struct smp_msg** msg)
 747 {
 748         cpu_status state;
 749
 750         TRACE("find_free_message: entry\n");
 751
 752 retry:
 753         while (sFreeMessageCount <= 0)
 754                 cpu_pause();
 755
 756         state = disable_interrupts();
 757         acquire_spinlock(&sFreeMessageSpinlock);
 758
 759         if (sFreeMessageCount <= 0) {
 760                 // someone grabbed one while we were getting the lock,
 761                 // go back to waiting for it
 762                 release_spinlock(&sFreeMessageSpinlock);
 763                 restore_interrupts(state);
 764                 goto retry;
 765         }
 766
 767         *msg = sFreeMessages;
 768         sFreeMessages = (*msg)->next;
 769         sFreeMessageCount--;
 770
 771         release_spinlock(&sFreeMessageSpinlock);
 772
 773         TRACE("find_free_message: returning msg %p\n", *msg);
 774
 775         return state;
 776 }
 777
 778
 779 /*!     Similar to find_free_message(), but expects the interrupts to be disabled
 780         already.
 781 */
 782 static void
 783 find_free_message_interrupts_disabled(int32 currentCPU,
 784         struct smp_msg** _message)
 785 {
 786         TRACE("find_free_message_interrupts_disabled: entry\n");
 787
 788         acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock);
 789         while (sFreeMessageCount <= 0) {
 790                 release_spinlock(&sFreeMessageSpinlock);
 791                 process_all_pending_ici(currentCPU);
 792                 cpu_pause();
 793                 acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock);
 794         }
 795
 796         *_message = sFreeMessages;
 797         sFreeMessages = (*_message)->next;
 798         sFreeMessageCount--;
 799
 800         release_spinlock(&sFreeMessageSpinlock);
 801
 802         TRACE("find_free_message_interrupts_disabled: returning msg %p\n",
 803                 *_message);
 804 }
 805
 806
 807 static void
 808 return_free_message(struct smp_msg* msg)
 809 {
 810         TRACE("return_free_message: returning msg %p\n", msg);
 811
 812         acquire_spinlock_nocheck(&sFreeMessageSpinlock);
 813         msg->next = sFreeMessages;
 814         sFreeMessages = msg;
 815         sFreeMessageCount++;
 816         release_spinlock(&sFreeMessageSpinlock);
 817 }
 818
 819
 820 static struct smp_msg*
 821 check_for_message(int currentCPU, mailbox_source& sourceMailbox)
 822 {
 823         if (!sICIEnabled)
 824                 return NULL;
 825
 826         struct smp_msg* msg = atomic_pointer_get(&sCPUMessages[currentCPU]);
 827         if (msg != NULL) {
 828                 do {
 829                         cpu_pause();
 830                         msg = atomic_pointer_get(&sCPUMessages[currentCPU]);
 831                         ASSERT(msg != NULL);
 832                 } while (atomic_pointer_test_and_set(&sCPUMessages[currentCPU],
 833                                 msg->next, msg) != msg);
 834
 835                 TRACE(" cpu %d: found msg %p in cpu mailbox\n", currentCPU, msg);
 836                 sourceMailbox = MAILBOX_LOCAL;
 837         } else if (atomic_get(&get_cpu_struct()->ici_counter)
 838                 != atomic_get(&sBroadcastMessageCounter)) {
 839
 840                 // try getting one from the broadcast mailbox
 841                 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
 842
 843                 msg = sBroadcastMessages;
 844                 while (msg != NULL) {
 845                         if (!msg->proc_bitmap.GetBit(currentCPU)) {
 846                                 // we have handled this one already
 847                                 msg = msg->next;
 848                                 continue;
 849                         }
 850
 851                         // mark it so we wont try to process this one again
 852                         msg->proc_bitmap.ClearBitAtomic(currentCPU);
 853                         atomic_add(&gCPU[currentCPU].ici_counter, 1);
 854
 855                         sourceMailbox = MAILBOX_BCAST;
 856                         break;
 857                 }
 858                 release_spinlock(&sBroadcastMessageSpinlock);
 859
 860                 if (msg != NULL) {
 861                         TRACE(" cpu %d: found msg %p in broadcast mailbox\n", currentCPU,
 862                                 msg);
 863                 }
 864         }
 865         return msg;
 866 }
 867
 868
 869 static void
 870 finish_message_processing(int currentCPU, struct smp_msg* msg,
 871         mailbox_source sourceMailbox)
 872 {
 873         if (atomic_add(&msg->ref_count, -1) != 1)
 874                 return;
 875
 876         // we were the last one to decrement the ref_count
 877         // it's our job to remove it from the list & possibly clean it up
 878
 879         // clean up the message
 880         if (sourceMailbox == MAILBOX_BCAST)
 881                 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
 882
 883         TRACE("cleaning up message %p\n", msg);
 884
 885         if (sourceMailbox != MAILBOX_BCAST) {
 886                 // local mailbox -- the message has already been removed in
 887                 // check_for_message()
 888         } else if (msg == sBroadcastMessages) {
 889                 sBroadcastMessages = msg->next;
 890         } else {
 891                 // we need to walk to find the message in the list.
 892                 // we can't use any data found when previously walking through
 893                 // the list, since the list may have changed. But, we are guaranteed
 894                 // to at least have msg in it.
 895                 struct smp_msg* last = NULL;
 896                 struct smp_msg* msg1;
 897
 898                 msg1 = sBroadcastMessages;
 899                 while (msg1 != NULL && msg1 != msg) {
 900                         last = msg1;
 901                         msg1 = msg1->next;
 902                 }
 903
 904                 // by definition, last must be something
 905                 if (msg1 == msg && last != NULL)
 906                         last->next = msg->next;
 907                 else
 908                         panic("last == NULL or msg != msg1");
 909         }
 910
 911         if (sourceMailbox == MAILBOX_BCAST)
 912                 release_spinlock(&sBroadcastMessageSpinlock);
 913
 914         if ((msg->flags & SMP_MSG_FLAG_FREE_ARG) != 0 && msg->data_ptr != NULL)
 915                 free(msg->data_ptr);
 916
 917         if ((msg->flags & SMP_MSG_FLAG_SYNC) != 0) {
 918                 atomic_set(&msg->done, 1);
 919                 // the caller cpu should now free the message
 920         } else {
 921                 // in the !SYNC case, we get to free the message
 922                 return_free_message(msg);
 923         }
 924 }
 925
 926
 927 static status_t
 928 process_pending_ici(int32 currentCPU)
 929 {
 930         mailbox_source sourceMailbox;
 931         struct smp_msg* msg = check_for_message(currentCPU, sourceMailbox);
 932         if (msg == NULL)
 933                 return B_ENTRY_NOT_FOUND;
 934
 935         TRACE("  cpu %ld message = %ld\n", currentCPU, msg->message);
 936
 937         bool haltCPU = false;
 938
 939         switch (msg->message) {
 940                 case SMP_MSG_INVALIDATE_PAGE_RANGE:
 941                         arch_cpu_invalidate_TLB_range(msg->data, msg->data2);
 942                         break;
 943                 case SMP_MSG_INVALIDATE_PAGE_LIST:
 944                         arch_cpu_invalidate_TLB_list((addr_t*)msg->data, (int)msg->data2);
 945                         break;
 946                 case SMP_MSG_USER_INVALIDATE_PAGES:
 947                         arch_cpu_user_TLB_invalidate();
 948                         break;
 949                 case SMP_MSG_GLOBAL_INVALIDATE_PAGES:
 950                         arch_cpu_global_TLB_invalidate();
 951                         break;
 952                 case SMP_MSG_CPU_HALT:
 953                         haltCPU = true;
 954                         break;
 955                 case SMP_MSG_CALL_FUNCTION:
 956                 {
 957                         smp_call_func func = (smp_call_func)msg->data_ptr;
 958                         func(msg->data, currentCPU, msg->data2, msg->data3);
 959                         break;
 960                 }
 961                 case SMP_MSG_RESCHEDULE:
 962                         scheduler_reschedule_ici();
 963                         break;
 964
 965                 default:
 966                         dprintf("smp_intercpu_int_handler: got unknown message %" B_PRId32 "\n",
 967                                 msg->message);
 968                         break;
 969         }
 970
 971         // finish dealing with this message, possibly removing it from the list
 972         finish_message_processing(currentCPU, msg, sourceMailbox);
 973
 974         // special case for the halt message
 975         if (haltCPU)
 976                 debug_trap_cpu_in_kdl(currentCPU, false);
 977
 978         return B_OK;
 979 }
 980
 981
 982 #if B_DEBUG_SPINLOCK_CONTENTION
 983
 984
 985 static uint64
 986 get_spinlock_counter(spinlock* lock)
 987 {
 988         uint32 high;
 989         uint32 low;
 990         do {
 991                 high = (uint32)atomic_get(&lock->count_high);
 992                 low = (uint32)atomic_get(&lock->count_low);
 993         } while (high != atomic_get(&lock->count_high));
 994
 995         return ((uint64)high << 32) | low;
 996 }
 997
 998
 999 static status_t
1000 spinlock_contention_syscall(const char* subsystem, uint32 function,
1001         void* buffer, size_t bufferSize)
1002 {
1003         spinlock_contention_info info;
1004
1005         if (function != GET_SPINLOCK_CONTENTION_INFO)
1006                 return B_BAD_VALUE;
1007
1008         if (bufferSize < sizeof(spinlock_contention_info))
1009                 return B_BAD_VALUE;
1010
1011         info.thread_spinlock_counter = get_spinlock_counter(&gThreadSpinlock);
1012         info.team_spinlock_counter = get_spinlock_counter(&gTeamSpinlock);
1013
1014         if (!IS_USER_ADDRESS(buffer)
1015                 || user_memcpy(buffer, &info, sizeof(info)) != B_OK) {
1016                 return B_BAD_ADDRESS;
1017         }
1018
1019         return B_OK;
1020 }
1021
1022
1023 #endif  // B_DEBUG_SPINLOCK_CONTENTION
1024
1025
1026 static void
1027 process_early_cpu_call(int32 cpu)
1028 {
1029         sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu);
1030         sEarlyCPUCallSet.ClearBitAtomic(cpu);
1031         atomic_add(&sEarlyCPUCallCount, 1);
1032 }
1033
1034
1035 static void
1036 call_all_cpus_early(void (*function)(void*, int), void* cookie)
1037 {
1038         if (sNumCPUs > 1) {
1039                 sEarlyCPUCallFunction = function;
1040                 sEarlyCPUCallCookie = cookie;
1041
1042                 atomic_set(&sEarlyCPUCallCount, 1);
1043                 sEarlyCPUCallSet.SetAll();
1044                 sEarlyCPUCallSet.ClearBit(0);
1045
1046                 // wait for all CPUs to finish
1047                 while (sEarlyCPUCallCount < sNumCPUs)
1048                         cpu_wait(&sEarlyCPUCallCount, sNumCPUs);
1049         }
1050
1051         function(cookie, 0);
1052 }
1053
1054
1055 //      #pragma mark -
1056
1057
1058 int
1059 smp_intercpu_int_handler(int32 cpu)
1060 {
1061         TRACE("smp_intercpu_int_handler: entry on cpu %ld\n", cpu);
1062
1063         process_all_pending_ici(cpu);
1064
1065         TRACE("smp_intercpu_int_handler: done on cpu %ld\n", cpu);
1066
1067         return B_HANDLED_INTERRUPT;
1068 }
1069
1070
1071 void
1072 smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2,
1073         addr_t data3, void* dataPointer, uint32 flags)
1074 {
1075         struct smp_msg *msg;
1076
1077         TRACE("smp_send_ici: target 0x%lx, mess 0x%lx, data 0x%lx, data2 0x%lx, "
1078                 "data3 0x%lx, ptr %p, flags 0x%lx\n", targetCPU, message, data, data2,
1079                 data3, dataPointer, flags);
1080
1081         if (sICIEnabled) {
1082                 int state;
1083                 int currentCPU;
1084
1085                 // find_free_message leaves interrupts disabled
1086                 state = find_free_message(&msg);
1087
1088                 currentCPU = smp_get_current_cpu();
1089                 if (targetCPU == currentCPU) {
1090                         return_free_message(msg);
1091                         restore_interrupts(state);
1092                         return; // nope, cant do that
1093                 }
1094
1095                 // set up the message
1096                 msg->message = message;
1097                 msg->data = data;
1098                 msg->data2 = data2;
1099                 msg->data3 = data3;
1100                 msg->data_ptr = dataPointer;
1101                 msg->ref_count = 1;
1102                 msg->flags = flags;
1103                 msg->done = 0;
1104
1105                 // stick it in the appropriate cpu's mailbox
1106                 struct smp_msg* next;
1107                 do {
1108                         cpu_pause();
1109                         next = atomic_pointer_get(&sCPUMessages[targetCPU]);
1110                         msg->next = next;
1111                 } while (atomic_pointer_test_and_set(&sCPUMessages[targetCPU], msg,
1112                                 next) != next);
1113
1114                 arch_smp_send_ici(targetCPU);
1115
1116                 if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
1117                         // wait for the other cpu to finish processing it
1118                         // the interrupt handler will ref count it to <0
1119                         // if the message is sync after it has removed it from the mailbox
1120                         while (msg->done == 0) {
1121                                 process_all_pending_ici(currentCPU);
1122                                 cpu_wait(&msg->done, 1);
1123                         }
1124                         // for SYNC messages, it's our responsibility to put it
1125                         // back into the free list
1126                         return_free_message(msg);
1127                 }
1128
1129                 restore_interrupts(state);
1130         }
1131 }
1132
1133
1134 void
1135 smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
1136         addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
1137 {
1138         if (!sICIEnabled)
1139                 return;
1140
1141         int currentCPU = smp_get_current_cpu();
1142
1143         // find_free_message leaves interrupts disabled
1144         struct smp_msg *msg;
1145         int state = find_free_message(&msg);
1146
1147         msg->proc_bitmap = cpuMask;
1148         msg->proc_bitmap.ClearBit(currentCPU);
1149
1150         int32 targetCPUs = 0;
1151         for (int32 i = 0; i < sNumCPUs; i++) {
1152                 if (msg->proc_bitmap.GetBit(i))
1153                         targetCPUs++;
1154         }
1155
1156         if (targetCPUs == 0) {
1157                 panic("smp_send_multicast_ici(): 0 CPU mask");
1158                 return;
1159         }
1160
1161         msg->message = message;
1162         msg->data = data;
1163         msg->data2 = data2;
1164         msg->data3 = data3;
1165         msg->data_ptr = dataPointer;
1166         msg->ref_count = targetCPUs;
1167         msg->flags = flags;
1168         msg->done = 0;
1169
1170         bool broadcast = targetCPUs == sNumCPUs - 1;
1171
1172         // stick it in the broadcast mailbox
1173         acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
1174         msg->next = sBroadcastMessages;
1175         sBroadcastMessages = msg;
1176         release_spinlock(&sBroadcastMessageSpinlock);
1177
1178         atomic_add(&sBroadcastMessageCounter, 1);
1179         for (int32 i = 0; i < sNumCPUs; i++) {
1180                 if (!cpuMask.GetBit(i))
1181                         atomic_add(&gCPU[i].ici_counter, 1);
1182         }
1183
1184         if (broadcast)
1185                 arch_smp_send_broadcast_ici();
1186         else
1187                 arch_smp_send_multicast_ici(cpuMask);
1188
1189         if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
1190                 // wait for the other cpus to finish processing it
1191                 // the interrupt handler will ref count it to <0
1192                 // if the message is sync after it has removed it from the mailbox
1193                 while (msg->done == 0) {
1194                         process_all_pending_ici(currentCPU);
1195                         cpu_wait(&msg->done, 1);
1196                 }
1197
1198                 // for SYNC messages, it's our responsibility to put it
1199                 // back into the free list
1200                 return_free_message(msg);
1201         }
1202
1203         restore_interrupts(state);
1204 }
1205
1206
1207 void
1208 smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
1209         void *dataPointer, uint32 flags)
1210 {
1211         struct smp_msg *msg;
1212
1213         TRACE("smp_send_broadcast_ici: cpu %ld mess 0x%lx, data 0x%lx, data2 "
1214                 "0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", smp_get_current_cpu(),
1215                 message, data, data2, data3, dataPointer, flags);
1216
1217         if (sICIEnabled) {
1218                 int state;
1219                 int currentCPU;
1220
1221                 // find_free_message leaves interrupts disabled
1222                 state = find_free_message(&msg);
1223
1224                 currentCPU = smp_get_current_cpu();
1225
1226                 msg->message = message;
1227                 msg->data = data;
1228                 msg->data2 = data2;
1229                 msg->data3 = data3;
1230                 msg->data_ptr = dataPointer;
1231                 msg->ref_count = sNumCPUs - 1;
1232                 msg->flags = flags;
1233                 msg->proc_bitmap.SetAll();
1234                 msg->proc_bitmap.ClearBit(currentCPU);
1235                 msg->done = 0;
1236
1237                 TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast "
1238                         "mbox\n", currentCPU, msg);
1239
1240                 // stick it in the appropriate cpu's mailbox
1241                 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
1242                 msg->next = sBroadcastMessages;
1243                 sBroadcastMessages = msg;
1244                 release_spinlock(&sBroadcastMessageSpinlock);
1245
1246                 atomic_add(&sBroadcastMessageCounter, 1);
1247                 atomic_add(&gCPU[currentCPU].ici_counter, 1);
1248
1249                 arch_smp_send_broadcast_ici();
1250
1251                 TRACE("smp_send_broadcast_ici: sent interrupt\n");
1252
1253                 if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
1254                         // wait for the other cpus to finish processing it
1255                         // the interrupt handler will ref count it to <0
1256                         // if the message is sync after it has removed it from the mailbox
1257                         TRACE("smp_send_broadcast_ici: waiting for ack\n");
1258
1259                         while (msg->done == 0) {
1260                                 process_all_pending_ici(currentCPU);
1261                                 cpu_wait(&msg->done, 1);
1262                         }
1263
1264                         TRACE("smp_send_broadcast_ici: returning message to free list\n");
1265
1266                         // for SYNC messages, it's our responsibility to put it
1267                         // back into the free list
1268                         return_free_message(msg);
1269                 }
1270
1271                 restore_interrupts(state);
1272         }
1273
1274         TRACE("smp_send_broadcast_ici: done\n");
1275 }
1276
1277
1278 void
1279 smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
1280         addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
1281 {
1282         if (!sICIEnabled)
1283                 return;
1284
1285         TRACE("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, "
1286                 "data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n",
1287                 currentCPU, message, data, data2, data3, dataPointer, flags);
1288
1289         struct smp_msg *msg;
1290         find_free_message_interrupts_disabled(currentCPU, &msg);
1291
1292         msg->message = message;
1293         msg->data = data;
1294         msg->data2 = data2;
1295         msg->data3 = data3;
1296         msg->data_ptr = dataPointer;
1297         msg->ref_count = sNumCPUs - 1;
1298         msg->flags = flags;
1299         msg->proc_bitmap.SetAll();
1300         msg->proc_bitmap.ClearBit(currentCPU);
1301         msg->done = 0;
1302
1303         TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p "
1304                 "into broadcast mbox\n", currentCPU, msg);
1305
1306         // stick it in the appropriate cpu's mailbox
1307         acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
1308         msg->next = sBroadcastMessages;
1309         sBroadcastMessages = msg;
1310         release_spinlock(&sBroadcastMessageSpinlock);
1311
1312         atomic_add(&sBroadcastMessageCounter, 1);
1313         atomic_add(&gCPU[currentCPU].ici_counter, 1);
1314
1315         arch_smp_send_broadcast_ici();
1316
1317         TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n",
1318                 currentCPU);
1319
1320         if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
1321                 // wait for the other cpus to finish processing it
1322                 // the interrupt handler will ref count it to <0
1323                 // if the message is sync after it has removed it from the mailbox
1324                 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for "
1325                         "ack\n", currentCPU);
1326
1327                 while (msg->done == 0) {
1328                         process_all_pending_ici(currentCPU);
1329                         cpu_wait(&msg->done, 1);
1330                 }
1331
1332                 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: returning "
1333                         "message to free list\n", currentCPU);
1334
1335                 // for SYNC messages, it's our responsibility to put it
1336                 // back into the free list
1337                 return_free_message(msg);
1338         }
1339
1340         TRACE("smp_send_broadcast_ici_interrupts_disabled: done\n");
1341 }
1342
1343
1344 /*!     Spin on non-boot CPUs until smp_wake_up_non_boot_cpus() has been called.
1345
1346         \param cpu The index of the calling CPU.
1347         \param rendezVous A rendez-vous variable to make sure that the boot CPU
1348                 does not return before all other CPUs have started waiting.
1349         \return \c true on the boot CPU, \c false otherwise.
1350 */
1351 bool
1352 smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous)
1353 {
1354         if (cpu == 0) {
1355                 smp_cpu_rendezvous(rendezVous);
1356                 return true;
1357         }
1358
1359         smp_cpu_rendezvous(rendezVous);
1360
1361         while (sBootCPUSpin == 0) {
1362                 if (sEarlyCPUCallSet.GetBit(cpu))
1363                         process_early_cpu_call(cpu);
1364
1365                 cpu_pause();
1366         }
1367
1368         return false;
1369 }
1370
1371
1372 void
1373 smp_wake_up_non_boot_cpus()
1374 {
1375         // ICIs were previously being ignored
1376         if (sNumCPUs > 1)
1377                 sICIEnabled = true;
1378
1379         // resume non boot CPUs
1380         atomic_set(&sBootCPUSpin, 1);
1381 }
1382
1383
1384 /*!     Spin until all CPUs have reached the rendez-vous point.
1385
1386         The rendez-vous variable \c *var must have been initialized to 0 before the
1387         function is called. The variable will be non-null when the function returns.
1388
1389         Note that when the function returns on one CPU, it only means that all CPU
1390         have already entered the function. It does not mean that the variable can
1391         already be reset. Only when all CPUs have returned (which would have to be
1392         ensured via another rendez-vous) the variable can be reset.
1393 */
1394 void
1395 smp_cpu_rendezvous(uint32* var)
1396 {
1397         atomic_add((int32*)var, 1);
1398
1399         while (*var < (uint32)sNumCPUs)
1400                 cpu_wait((int32*)var, sNumCPUs);
1401 }
1402
1403
1404 status_t
1405 smp_init(kernel_args* args)
1406 {
1407         TRACE("smp_init: entry\n");
1408
1409 #if DEBUG_SPINLOCK_LATENCIES
1410         sEnableLatencyCheck
1411                 = !get_safemode_boolean(B_SAFEMODE_DISABLE_LATENCY_CHECK, false);
1412 #endif
1413
1414 #if DEBUG_SPINLOCKS
1415         add_debugger_command_etc("spinlock", &dump_spinlock,
1416                 "Dump info on a spinlock",
1417                 "\n"
1418                 "Dumps info on a spinlock.\n", 0);
1419 #endif
1420         add_debugger_command_etc("ici", &dump_ici_messages,
1421                 "Dump info on pending ICI messages",
1422                 "\n"
1423                 "Dumps info on pending ICI messages.\n", 0);
1424         add_debugger_command_etc("ici_message", &dump_ici_message,
1425                 "Dump info on an ICI message",
1426                 "\n"
1427                 "Dumps info on an ICI message.\n", 0);
1428
1429         if (args->num_cpus > 1) {
1430                 sFreeMessages = NULL;
1431                 sFreeMessageCount = 0;
1432                 for (int i = 0; i < MSG_POOL_SIZE; i++) {
1433                         struct smp_msg* msg
1434                                 = (struct smp_msg*)malloc(sizeof(struct smp_msg));
1435                         if (msg == NULL) {
1436                                 panic("error creating smp mailboxes\n");
1437                                 return B_ERROR;
1438                         }
1439                         memset(msg, 0, sizeof(struct smp_msg));
1440                         msg->next = sFreeMessages;
1441                         sFreeMessages = msg;
1442                         sFreeMessageCount++;
1443                 }
1444                 sNumCPUs = args->num_cpus;
1445         }
1446         TRACE("smp_init: calling arch_smp_init\n");
1447
1448         return arch_smp_init(args);
1449 }
1450
1451
1452 status_t
1453 smp_per_cpu_init(kernel_args* args, int32 cpu)
1454 {
1455         return arch_smp_per_cpu_init(args, cpu);
1456 }
1457
1458
1459 status_t
1460 smp_init_post_generic_syscalls(void)
1461 {
1462 #if B_DEBUG_SPINLOCK_CONTENTION
1463         return register_generic_syscall(SPINLOCK_CONTENTION,
1464                 &spinlock_contention_syscall, 0, 0);
1465 #else
1466         return B_OK;
1467 #endif
1468 }
1469
1470
1471 void
1472 smp_set_num_cpus(int32 numCPUs)
1473 {
1474         sNumCPUs = numCPUs;
1475 }
1476
1477
1478 int32
1479 smp_get_num_cpus()
1480 {
1481         return sNumCPUs;
1482 }
1483
1484
1485 int32
1486 smp_get_current_cpu(void)
1487 {
1488         return thread_get_current_thread()->cpu->cpu_num;
1489 }
1490
1491
1492 // #pragma mark - public exported functions
1493
1494
1495 void
1496 call_all_cpus(void (*func)(void*, int), void* cookie)
1497 {
1498         cpu_status state = disable_interrupts();
1499
1500         // if inter-CPU communication is not yet enabled, use the early mechanism
1501         if (!sICIEnabled) {
1502                 call_all_cpus_early(func, cookie);
1503                 restore_interrupts(state);
1504                 return;
1505         }
1506
1507         if (smp_get_num_cpus() > 1) {
1508                 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie,
1509                         0, 0, (void*)func, SMP_MSG_FLAG_ASYNC);
1510         }
1511
1512         // we need to call this function ourselves as well
1513         func(cookie, smp_get_current_cpu());
1514
1515         restore_interrupts(state);
1516 }
1517
1518
1519 void
1520 call_all_cpus_sync(void (*func)(void*, int), void* cookie)
1521 {
1522         cpu_status state = disable_interrupts();
1523
1524         // if inter-CPU communication is not yet enabled, use the early mechanism
1525         if (!sICIEnabled) {
1526                 call_all_cpus_early(func, cookie);
1527                 restore_interrupts(state);
1528                 return;
1529         }
1530
1531         if (smp_get_num_cpus() > 1) {
1532                 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie,
1533                         0, 0, (void*)func, SMP_MSG_FLAG_SYNC);
1534         }
1535
1536         // we need to call this function ourselves as well
1537         func(cookie, smp_get_current_cpu());
1538
1539         restore_interrupts(state);
1540 }
1541
1542
1543 #undef memory_read_barrier
1544 #undef memory_write_barrier
1545
1546
1547 void
1548 memory_read_barrier()
1549 {
1550         memory_read_barrier_inline();
1551 }
1552
1553
1554 void
1555 memory_write_barrier()
1556 {
1557         memory_write_barrier_inline();
1558 }
1559