cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor.h"
  29 #include "sysemu.h"
  30 #include "gdbstub.h"
  31 #include "dma.h"
  32 #include "kvm.h"
  33
  34 #include "qemu-thread.h"
  35 #include "cpus.h"
  36
  37 #ifndef _WIN32
  38 #include "compatfd.h"
  39 #endif
  40
  41 #ifdef SIGRTMIN
  42 #define SIG_IPI (SIGRTMIN+4)
  43 #else
  44 #define SIG_IPI SIGUSR1
  45 #endif
  46
  47 #ifdef CONFIG_LINUX
  48
  49 #include <sys/prctl.h>
  50
  51 #ifndef PR_MCE_KILL
  52 #define PR_MCE_KILL 33
  53 #endif
  54
  55 #ifndef PR_MCE_KILL_SET
  56 #define PR_MCE_KILL_SET 1
  57 #endif
  58
  59 #ifndef PR_MCE_KILL_EARLY
  60 #define PR_MCE_KILL_EARLY 1
  61 #endif
  62
  63 #endif /* CONFIG_LINUX */
  64
  65 static CPUState *next_cpu;
  66
  67 /***********************************************************/
  68 void hw_error(const char *fmt, ...)
  69 {
  70     va_list ap;
  71     CPUState *env;
  72
  73     va_start(ap, fmt);
  74     fprintf(stderr, "qemu: hardware error: ");
  75     vfprintf(stderr, fmt, ap);
  76     fprintf(stderr, "\n");
  77     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  78         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
  79 #ifdef TARGET_I386
  80         cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
  81 #else
  82         cpu_dump_state(env, stderr, fprintf, 0);
  83 #endif
  84     }
  85     va_end(ap);
  86     abort();
  87 }
  88
  89 void cpu_synchronize_all_states(void)
  90 {
  91     CPUState *cpu;
  92
  93     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
  94         cpu_synchronize_state(cpu);
  95     }
  96 }
  97
  98 void cpu_synchronize_all_post_reset(void)
  99 {
 100     CPUState *cpu;
 101
 102     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 103         cpu_synchronize_post_reset(cpu);
 104     }
 105 }
 106
 107 void cpu_synchronize_all_post_init(void)
 108 {
 109     CPUState *cpu;
 110
 111     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 112         cpu_synchronize_post_init(cpu);
 113     }
 114 }
 115
 116 int cpu_is_stopped(CPUState *env)
 117 {
 118     return !vm_running || env->stopped;
 119 }
 120
 121 static void do_vm_stop(int reason)
 122 {
 123     if (vm_running) {
 124         cpu_disable_ticks();
 125         vm_running = 0;
 126         pause_all_vcpus();
 127         vm_state_notify(0, reason);
 128         qemu_aio_flush();
 129         bdrv_flush_all();
 130         monitor_protocol_event(QEVENT_STOP, NULL);
 131     }
 132 }
 133
 134 static int cpu_can_run(CPUState *env)
 135 {
 136     if (env->stop) {
 137         return 0;
 138     }
 139     if (env->stopped || !vm_running) {
 140         return 0;
 141     }
 142     return 1;
 143 }
 144
 145 static bool cpu_thread_is_idle(CPUState *env)
 146 {
 147     if (env->stop || env->queued_work_first) {
 148         return false;
 149     }
 150     if (env->stopped || !vm_running) {
 151         return true;
 152     }
 153     if (!env->halted || qemu_cpu_has_work(env) ||
 154         (kvm_enabled() && kvm_irqchip_in_kernel())) {
 155         return false;
 156     }
 157     return true;
 158 }
 159
 160 bool all_cpu_threads_idle(void)
 161 {
 162     CPUState *env;
 163
 164     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 165         if (!cpu_thread_is_idle(env)) {
 166             return false;
 167         }
 168     }
 169     return true;
 170 }
 171
 172 static void cpu_handle_guest_debug(CPUState *env)
 173 {
 174     gdb_set_stop_cpu(env);
 175     qemu_system_debug_request();
 176 #ifdef CONFIG_IOTHREAD
 177     env->stopped = 1;
 178 #endif
 179 }
 180
 181 #ifdef CONFIG_IOTHREAD
 182 static void cpu_signal(int sig)
 183 {
 184     if (cpu_single_env) {
 185         cpu_exit(cpu_single_env);
 186     }
 187     exit_request = 1;
 188 }
 189 #endif
 190
 191 #ifdef CONFIG_LINUX
 192 static void sigbus_reraise(void)
 193 {
 194     sigset_t set;
 195     struct sigaction action;
 196
 197     memset(&action, 0, sizeof(action));
 198     action.sa_handler = SIG_DFL;
 199     if (!sigaction(SIGBUS, &action, NULL)) {
 200         raise(SIGBUS);
 201         sigemptyset(&set);
 202         sigaddset(&set, SIGBUS);
 203         sigprocmask(SIG_UNBLOCK, &set, NULL);
 204     }
 205     perror("Failed to re-raise SIGBUS!\n");
 206     abort();
 207 }
 208
 209 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 210                            void *ctx)
 211 {
 212     if (kvm_on_sigbus(siginfo->ssi_code,
 213                       (void *)(intptr_t)siginfo->ssi_addr)) {
 214         sigbus_reraise();
 215     }
 216 }
 217
 218 static void qemu_init_sigbus(void)
 219 {
 220     struct sigaction action;
 221
 222     memset(&action, 0, sizeof(action));
 223     action.sa_flags = SA_SIGINFO;
 224     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 225     sigaction(SIGBUS, &action, NULL);
 226
 227     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 228 }
 229
 230 static void qemu_kvm_eat_signals(CPUState *env)
 231 {
 232     struct timespec ts = { 0, 0 };
 233     siginfo_t siginfo;
 234     sigset_t waitset;
 235     sigset_t chkset;
 236     int r;
 237
 238     sigemptyset(&waitset);
 239     sigaddset(&waitset, SIG_IPI);
 240     sigaddset(&waitset, SIGBUS);
 241
 242     do {
 243         r = sigtimedwait(&waitset, &siginfo, &ts);
 244         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 245             perror("sigtimedwait");
 246             exit(1);
 247         }
 248
 249         switch (r) {
 250         case SIGBUS:
 251             if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
 252                 sigbus_reraise();
 253             }
 254             break;
 255         default:
 256             break;
 257         }
 258
 259         r = sigpending(&chkset);
 260         if (r == -1) {
 261             perror("sigpending");
 262             exit(1);
 263         }
 264     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 265
 266 #ifndef CONFIG_IOTHREAD
 267     if (sigismember(&chkset, SIGIO) || sigismember(&chkset, SIGALRM)) {
 268         qemu_notify_event();
 269     }
 270 #endif
 271 }
 272
 273 #else /* !CONFIG_LINUX */
 274
 275 static void qemu_init_sigbus(void)
 276 {
 277 }
 278
 279 static void qemu_kvm_eat_signals(CPUState *env)
 280 {
 281 }
 282 #endif /* !CONFIG_LINUX */
 283
 284 #ifndef _WIN32
 285 static int io_thread_fd = -1;
 286
 287 static void qemu_event_increment(void)
 288 {
 289     /* Write 8 bytes to be compatible with eventfd.  */
 290     static const uint64_t val = 1;
 291     ssize_t ret;
 292
 293     if (io_thread_fd == -1) {
 294         return;
 295     }
 296     do {
 297         ret = write(io_thread_fd, &val, sizeof(val));
 298     } while (ret < 0 && errno == EINTR);
 299
 300     /* EAGAIN is fine, a read must be pending.  */
 301     if (ret < 0 && errno != EAGAIN) {
 302         fprintf(stderr, "qemu_event_increment: write() failed: %s\n",
 303                 strerror(errno));
 304         exit (1);
 305     }
 306 }
 307
 308 static void qemu_event_read(void *opaque)
 309 {
 310     int fd = (intptr_t)opaque;
 311     ssize_t len;
 312     char buffer[512];
 313
 314     /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
 315     do {
 316         len = read(fd, buffer, sizeof(buffer));
 317     } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
 318 }
 319
 320 static int qemu_event_init(void)
 321 {
 322     int err;
 323     int fds[2];
 324
 325     err = qemu_eventfd(fds);
 326     if (err == -1) {
 327         return -errno;
 328     }
 329     err = fcntl_setfl(fds[0], O_NONBLOCK);
 330     if (err < 0) {
 331         goto fail;
 332     }
 333     err = fcntl_setfl(fds[1], O_NONBLOCK);
 334     if (err < 0) {
 335         goto fail;
 336     }
 337     qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
 338                          (void *)(intptr_t)fds[0]);
 339
 340     io_thread_fd = fds[1];
 341     return 0;
 342
 343 fail:
 344     close(fds[0]);
 345     close(fds[1]);
 346     return err;
 347 }
 348
 349 static void dummy_signal(int sig)
 350 {
 351 }
 352
 353 /* If we have signalfd, we mask out the signals we want to handle and then
 354  * use signalfd to listen for them.  We rely on whatever the current signal
 355  * handler is to dispatch the signals when we receive them.
 356  */
 357 static void sigfd_handler(void *opaque)
 358 {
 359     int fd = (intptr_t)opaque;
 360     struct qemu_signalfd_siginfo info;
 361     struct sigaction action;
 362     ssize_t len;
 363
 364     while (1) {
 365         do {
 366             len = read(fd, &info, sizeof(info));
 367         } while (len == -1 && errno == EINTR);
 368
 369         if (len == -1 && errno == EAGAIN) {
 370             break;
 371         }
 372
 373         if (len != sizeof(info)) {
 374             printf("read from sigfd returned %zd: %m\n", len);
 375             return;
 376         }
 377
 378         sigaction(info.ssi_signo, NULL, &action);
 379         if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction) {
 380             action.sa_sigaction(info.ssi_signo,
 381                                 (siginfo_t *)&info, NULL);
 382         } else if (action.sa_handler) {
 383             action.sa_handler(info.ssi_signo);
 384         }
 385     }
 386 }
 387
 388 static int qemu_signal_init(void)
 389 {
 390     int sigfd;
 391     sigset_t set;
 392
 393 #ifdef CONFIG_IOTHREAD
 394     /* SIGUSR2 used by posix-aio-compat.c */
 395     sigemptyset(&set);
 396     sigaddset(&set, SIGUSR2);
 397     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 398
 399     /*
 400      * SIG_IPI must be blocked in the main thread and must not be caught
 401      * by sigwait() in the signal thread. Otherwise, the cpu thread will
 402      * not catch it reliably.
 403      */
 404     sigemptyset(&set);
 405     sigaddset(&set, SIG_IPI);
 406     pthread_sigmask(SIG_BLOCK, &set, NULL);
 407
 408     sigemptyset(&set);
 409     sigaddset(&set, SIGIO);
 410     sigaddset(&set, SIGALRM);
 411     sigaddset(&set, SIGBUS);
 412 #else
 413     sigemptyset(&set);
 414     sigaddset(&set, SIGBUS);
 415     if (kvm_enabled()) {
 416         /*
 417          * We need to process timer signals synchronously to avoid a race
 418          * between exit_request check and KVM vcpu entry.
 419          */
 420         sigaddset(&set, SIGIO);
 421         sigaddset(&set, SIGALRM);
 422     }
 423 #endif
 424     pthread_sigmask(SIG_BLOCK, &set, NULL);
 425
 426     sigfd = qemu_signalfd(&set);
 427     if (sigfd == -1) {
 428         fprintf(stderr, "failed to create signalfd\n");
 429         return -errno;
 430     }
 431
 432     fcntl_setfl(sigfd, O_NONBLOCK);
 433
 434     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 435                          (void *)(intptr_t)sigfd);
 436
 437     return 0;
 438 }
 439
 440 static void qemu_kvm_init_cpu_signals(CPUState *env)
 441 {
 442     int r;
 443     sigset_t set;
 444     struct sigaction sigact;
 445
 446     memset(&sigact, 0, sizeof(sigact));
 447     sigact.sa_handler = dummy_signal;
 448     sigaction(SIG_IPI, &sigact, NULL);
 449
 450 #ifdef CONFIG_IOTHREAD
 451     pthread_sigmask(SIG_BLOCK, NULL, &set);
 452     sigdelset(&set, SIG_IPI);
 453     sigdelset(&set, SIGBUS);
 454     r = kvm_set_signal_mask(env, &set);
 455     if (r) {
 456         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 457         exit(1);
 458     }
 459 #else
 460     sigemptyset(&set);
 461     sigaddset(&set, SIG_IPI);
 462     sigaddset(&set, SIGIO);
 463     sigaddset(&set, SIGALRM);
 464     pthread_sigmask(SIG_BLOCK, &set, NULL);
 465
 466     pthread_sigmask(SIG_BLOCK, NULL, &set);
 467     sigdelset(&set, SIGIO);
 468     sigdelset(&set, SIGALRM);
 469 #endif
 470     sigdelset(&set, SIG_IPI);
 471     sigdelset(&set, SIGBUS);
 472     r = kvm_set_signal_mask(env, &set);
 473     if (r) {
 474         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 475         exit(1);
 476     }
 477 }
 478
 479 static void qemu_tcg_init_cpu_signals(void)
 480 {
 481 #ifdef CONFIG_IOTHREAD
 482     sigset_t set;
 483     struct sigaction sigact;
 484
 485     memset(&sigact, 0, sizeof(sigact));
 486     sigact.sa_handler = cpu_signal;
 487     sigaction(SIG_IPI, &sigact, NULL);
 488
 489     sigemptyset(&set);
 490     sigaddset(&set, SIG_IPI);
 491     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 492 #endif
 493 }
 494
 495 #else /* _WIN32 */
 496
 497 HANDLE qemu_event_handle;
 498
 499 static void dummy_event_handler(void *opaque)
 500 {
 501 }
 502
 503 static int qemu_event_init(void)
 504 {
 505     qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
 506     if (!qemu_event_handle) {
 507         fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
 508         return -1;
 509     }
 510     qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
 511     return 0;
 512 }
 513
 514 static void qemu_event_increment(void)
 515 {
 516     if (!SetEvent(qemu_event_handle)) {
 517         fprintf(stderr, "qemu_event_increment: SetEvent failed: %ld\n",
 518                 GetLastError());
 519         exit (1);
 520     }
 521 }
 522
 523 static int qemu_signal_init(void)
 524 {
 525     return 0;
 526 }
 527
 528 static void qemu_kvm_init_cpu_signals(CPUState *env)
 529 {
 530     abort();
 531 }
 532
 533 static void qemu_tcg_init_cpu_signals(void)
 534 {
 535 }
 536 #endif /* _WIN32 */
 537
 538 #ifndef CONFIG_IOTHREAD
 539 int qemu_init_main_loop(void)
 540 {
 541     int ret;
 542
 543     ret = qemu_signal_init();
 544     if (ret) {
 545         return ret;
 546     }
 547
 548     qemu_init_sigbus();
 549
 550     return qemu_event_init();
 551 }
 552
 553 void qemu_main_loop_start(void)
 554 {
 555 }
 556
 557 void qemu_init_vcpu(void *_env)
 558 {
 559     CPUState *env = _env;
 560     int r;
 561
 562     env->nr_cores = smp_cores;
 563     env->nr_threads = smp_threads;
 564
 565     if (kvm_enabled()) {
 566         r = kvm_init_vcpu(env);
 567         if (r < 0) {
 568             fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 569             exit(1);
 570         }
 571         qemu_kvm_init_cpu_signals(env);
 572     } else {
 573         qemu_tcg_init_cpu_signals();
 574     }
 575 }
 576
 577 int qemu_cpu_is_self(void *env)
 578 {
 579     return 1;
 580 }
 581
 582 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 583 {
 584     func(data);
 585 }
 586
 587 void resume_all_vcpus(void)
 588 {
 589 }
 590
 591 void pause_all_vcpus(void)
 592 {
 593 }
 594
 595 void qemu_cpu_kick(void *env)
 596 {
 597 }
 598
 599 void qemu_cpu_kick_self(void)
 600 {
 601 #ifndef _WIN32
 602     assert(cpu_single_env);
 603
 604     raise(SIG_IPI);
 605 #else
 606     abort();
 607 #endif
 608 }
 609
 610 void qemu_notify_event(void)
 611 {
 612     CPUState *env = cpu_single_env;
 613
 614     qemu_event_increment ();
 615     if (env) {
 616         cpu_exit(env);
 617     }
 618     if (next_cpu && env != next_cpu) {
 619         cpu_exit(next_cpu);
 620     }
 621     exit_request = 1;
 622 }
 623
 624 void qemu_mutex_lock_iothread(void) {}
 625 void qemu_mutex_unlock_iothread(void) {}
 626
 627 void cpu_stop_current(void)
 628 {
 629 }
 630
 631 void vm_stop(int reason)
 632 {
 633     do_vm_stop(reason);
 634 }
 635
 636 #else /* CONFIG_IOTHREAD */
 637
 638 QemuMutex qemu_global_mutex;
 639 static QemuCond qemu_io_proceeded_cond;
 640 static bool iothread_requesting_mutex;
 641
 642 static QemuThread io_thread;
 643
 644 static QemuThread *tcg_cpu_thread;
 645 static QemuCond *tcg_halt_cond;
 646
 647 static int qemu_system_ready;
 648 /* cpu creation */
 649 static QemuCond qemu_cpu_cond;
 650 /* system init */
 651 static QemuCond qemu_system_cond;
 652 static QemuCond qemu_pause_cond;
 653 static QemuCond qemu_work_cond;
 654
 655 int qemu_init_main_loop(void)
 656 {
 657     int ret;
 658
 659     qemu_init_sigbus();
 660
 661     ret = qemu_signal_init();
 662     if (ret) {
 663         return ret;
 664     }
 665
 666     /* Note eventfd must be drained before signalfd handlers run */
 667     ret = qemu_event_init();
 668     if (ret) {
 669         return ret;
 670     }
 671
 672     qemu_cond_init(&qemu_cpu_cond);
 673     qemu_cond_init(&qemu_system_cond);
 674     qemu_cond_init(&qemu_pause_cond);
 675     qemu_cond_init(&qemu_work_cond);
 676     qemu_cond_init(&qemu_io_proceeded_cond);
 677     qemu_mutex_init(&qemu_global_mutex);
 678     qemu_mutex_lock(&qemu_global_mutex);
 679
 680     qemu_thread_get_self(&io_thread);
 681
 682     return 0;
 683 }
 684
 685 void qemu_main_loop_start(void)
 686 {
 687     qemu_system_ready = 1;
 688     qemu_cond_broadcast(&qemu_system_cond);
 689 }
 690
 691 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 692 {
 693     struct qemu_work_item wi;
 694
 695     if (qemu_cpu_is_self(env)) {
 696         func(data);
 697         return;
 698     }
 699
 700     wi.func = func;
 701     wi.data = data;
 702     if (!env->queued_work_first) {
 703         env->queued_work_first = &wi;
 704     } else {
 705         env->queued_work_last->next = &wi;
 706     }
 707     env->queued_work_last = &wi;
 708     wi.next = NULL;
 709     wi.done = false;
 710
 711     qemu_cpu_kick(env);
 712     while (!wi.done) {
 713         CPUState *self_env = cpu_single_env;
 714
 715         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 716         cpu_single_env = self_env;
 717     }
 718 }
 719
 720 static void flush_queued_work(CPUState *env)
 721 {
 722     struct qemu_work_item *wi;
 723
 724     if (!env->queued_work_first) {
 725         return;
 726     }
 727
 728     while ((wi = env->queued_work_first)) {
 729         env->queued_work_first = wi->next;
 730         wi->func(wi->data);
 731         wi->done = true;
 732     }
 733     env->queued_work_last = NULL;
 734     qemu_cond_broadcast(&qemu_work_cond);
 735 }
 736
 737 static void qemu_wait_io_event_common(CPUState *env)
 738 {
 739     if (env->stop) {
 740         env->stop = 0;
 741         env->stopped = 1;
 742         qemu_cond_signal(&qemu_pause_cond);
 743     }
 744     flush_queued_work(env);
 745     env->thread_kicked = false;
 746 }
 747
 748 static void qemu_tcg_wait_io_event(void)
 749 {
 750     CPUState *env;
 751
 752     while (all_cpu_threads_idle()) {
 753        /* Start accounting real time to the virtual clock if the CPUs
 754           are idle.  */
 755         qemu_clock_warp(vm_clock);
 756         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 757     }
 758
 759     while (iothread_requesting_mutex) {
 760         qemu_cond_wait(&qemu_io_proceeded_cond, &qemu_global_mutex);
 761     }
 762
 763     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 764         qemu_wait_io_event_common(env);
 765     }
 766 }
 767
 768 static void qemu_kvm_wait_io_event(CPUState *env)
 769 {
 770     while (cpu_thread_is_idle(env)) {
 771         qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
 772     }
 773
 774     qemu_kvm_eat_signals(env);
 775     qemu_wait_io_event_common(env);
 776 }
 777
 778 static void *qemu_kvm_cpu_thread_fn(void *arg)
 779 {
 780     CPUState *env = arg;
 781     int r;
 782
 783     qemu_mutex_lock(&qemu_global_mutex);
 784     qemu_thread_get_self(env->thread);
 785     env->thread_id = qemu_get_thread_id();
 786
 787     r = kvm_init_vcpu(env);
 788     if (r < 0) {
 789         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 790         exit(1);
 791     }
 792
 793     qemu_kvm_init_cpu_signals(env);
 794
 795     /* signal CPU creation */
 796     env->created = 1;
 797     qemu_cond_signal(&qemu_cpu_cond);
 798
 799     /* and wait for machine initialization */
 800     while (!qemu_system_ready) {
 801         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 802     }
 803
 804     while (1) {
 805         if (cpu_can_run(env)) {
 806             r = kvm_cpu_exec(env);
 807             if (r == EXCP_DEBUG) {
 808                 cpu_handle_guest_debug(env);
 809             }
 810         }
 811         qemu_kvm_wait_io_event(env);
 812     }
 813
 814     return NULL;
 815 }
 816
 817 static void *qemu_tcg_cpu_thread_fn(void *arg)
 818 {
 819     CPUState *env = arg;
 820
 821     qemu_tcg_init_cpu_signals();
 822     qemu_thread_get_self(env->thread);
 823
 824     /* signal CPU creation */
 825     qemu_mutex_lock(&qemu_global_mutex);
 826     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 827         env->thread_id = qemu_get_thread_id();
 828         env->created = 1;
 829     }
 830     qemu_cond_signal(&qemu_cpu_cond);
 831
 832     /* and wait for machine initialization */
 833     while (!qemu_system_ready) {
 834         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 835     }
 836
 837     while (1) {
 838         cpu_exec_all();
 839         if (use_icount && qemu_next_icount_deadline() <= 0) {
 840             qemu_notify_event();
 841         }
 842         qemu_tcg_wait_io_event();
 843     }
 844
 845     return NULL;
 846 }
 847
 848 static void qemu_cpu_kick_thread(CPUState *env)
 849 {
 850 #ifndef _WIN32
 851     int err;
 852
 853     err = pthread_kill(env->thread->thread, SIG_IPI);
 854     if (err) {
 855         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
 856         exit(1);
 857     }
 858 #else /* _WIN32 */
 859     if (!qemu_cpu_is_self(env)) {
 860         SuspendThread(env->thread->thread);
 861         cpu_signal(0);
 862         ResumeThread(env->thread->thread);
 863     }
 864 #endif
 865 }
 866
 867 void qemu_cpu_kick(void *_env)
 868 {
 869     CPUState *env = _env;
 870
 871     qemu_cond_broadcast(env->halt_cond);
 872     if (!env->thread_kicked) {
 873         qemu_cpu_kick_thread(env);
 874         env->thread_kicked = true;
 875     }
 876 }
 877
 878 void qemu_cpu_kick_self(void)
 879 {
 880 #ifndef _WIN32
 881     assert(cpu_single_env);
 882
 883     if (!cpu_single_env->thread_kicked) {
 884         qemu_cpu_kick_thread(cpu_single_env);
 885         cpu_single_env->thread_kicked = true;
 886     }
 887 #else
 888     abort();
 889 #endif
 890 }
 891
 892 int qemu_cpu_is_self(void *_env)
 893 {
 894     CPUState *env = _env;
 895
 896     return qemu_thread_is_self(env->thread);
 897 }
 898
 899 void qemu_mutex_lock_iothread(void)
 900 {
 901     if (kvm_enabled()) {
 902         qemu_mutex_lock(&qemu_global_mutex);
 903     } else {
 904         iothread_requesting_mutex = true;
 905         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 906             qemu_cpu_kick_thread(first_cpu);
 907             qemu_mutex_lock(&qemu_global_mutex);
 908         }
 909         iothread_requesting_mutex = false;
 910         qemu_cond_broadcast(&qemu_io_proceeded_cond);
 911     }
 912 }
 913
 914 void qemu_mutex_unlock_iothread(void)
 915 {
 916     qemu_mutex_unlock(&qemu_global_mutex);
 917 }
 918
 919 static int all_vcpus_paused(void)
 920 {
 921     CPUState *penv = first_cpu;
 922
 923     while (penv) {
 924         if (!penv->stopped) {
 925             return 0;
 926         }
 927         penv = (CPUState *)penv->next_cpu;
 928     }
 929
 930     return 1;
 931 }
 932
 933 void pause_all_vcpus(void)
 934 {
 935     CPUState *penv = first_cpu;
 936
 937     while (penv) {
 938         penv->stop = 1;
 939         qemu_cpu_kick(penv);
 940         penv = (CPUState *)penv->next_cpu;
 941     }
 942
 943     while (!all_vcpus_paused()) {
 944         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
 945         penv = first_cpu;
 946         while (penv) {
 947             qemu_cpu_kick(penv);
 948             penv = (CPUState *)penv->next_cpu;
 949         }
 950     }
 951 }
 952
 953 void resume_all_vcpus(void)
 954 {
 955     CPUState *penv = first_cpu;
 956
 957     while (penv) {
 958         penv->stop = 0;
 959         penv->stopped = 0;
 960         qemu_cpu_kick(penv);
 961         penv = (CPUState *)penv->next_cpu;
 962     }
 963 }
 964
 965 static void qemu_tcg_init_vcpu(void *_env)
 966 {
 967     CPUState *env = _env;
 968
 969     /* share a single thread for all cpus with TCG */
 970     if (!tcg_cpu_thread) {
 971         env->thread = qemu_mallocz(sizeof(QemuThread));
 972         env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 973         qemu_cond_init(env->halt_cond);
 974         qemu_thread_create(env->thread, qemu_tcg_cpu_thread_fn, env);
 975         while (env->created == 0) {
 976             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 977         }
 978         tcg_cpu_thread = env->thread;
 979         tcg_halt_cond = env->halt_cond;
 980     } else {
 981         env->thread = tcg_cpu_thread;
 982         env->halt_cond = tcg_halt_cond;
 983     }
 984 }
 985
 986 static void qemu_kvm_start_vcpu(CPUState *env)
 987 {
 988     env->thread = qemu_mallocz(sizeof(QemuThread));
 989     env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 990     qemu_cond_init(env->halt_cond);
 991     qemu_thread_create(env->thread, qemu_kvm_cpu_thread_fn, env);
 992     while (env->created == 0) {
 993         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 994     }
 995 }
 996
 997 void qemu_init_vcpu(void *_env)
 998 {
 999     CPUState *env = _env;
1000
1001     env->nr_cores = smp_cores;
1002     env->nr_threads = smp_threads;
1003     if (kvm_enabled()) {
1004         qemu_kvm_start_vcpu(env);
1005     } else {
1006         qemu_tcg_init_vcpu(env);
1007     }
1008 }
1009
1010 void qemu_notify_event(void)
1011 {
1012     qemu_event_increment();
1013 }
1014
1015 void cpu_stop_current(void)
1016 {
1017     if (cpu_single_env) {
1018         cpu_single_env->stop = 0;
1019         cpu_single_env->stopped = 1;
1020         cpu_exit(cpu_single_env);
1021         qemu_cond_signal(&qemu_pause_cond);
1022     }
1023 }
1024
1025 void vm_stop(int reason)
1026 {
1027     if (!qemu_thread_is_self(&io_thread)) {
1028         qemu_system_vmstop_request(reason);
1029         /*
1030          * FIXME: should not return to device code in case
1031          * vm_stop() has been requested.
1032          */
1033         cpu_stop_current();
1034         return;
1035     }
1036     do_vm_stop(reason);
1037 }
1038
1039 #endif
1040
1041 static int tcg_cpu_exec(CPUState *env)
1042 {
1043     int ret;
1044 #ifdef CONFIG_PROFILER
1045     int64_t ti;
1046 #endif
1047
1048 #ifdef CONFIG_PROFILER
1049     ti = profile_getclock();
1050 #endif
1051     if (use_icount) {
1052         int64_t count;
1053         int decr;
1054         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1055         env->icount_decr.u16.low = 0;
1056         env->icount_extra = 0;
1057         count = qemu_icount_round(qemu_next_icount_deadline());
1058         qemu_icount += count;
1059         decr = (count > 0xffff) ? 0xffff : count;
1060         count -= decr;
1061         env->icount_decr.u16.low = decr;
1062         env->icount_extra = count;
1063     }
1064     ret = cpu_exec(env);
1065 #ifdef CONFIG_PROFILER
1066     qemu_time += profile_getclock() - ti;
1067 #endif
1068     if (use_icount) {
1069         /* Fold pending instructions back into the
1070            instruction counter, and clear the interrupt flag.  */
1071         qemu_icount -= (env->icount_decr.u16.low
1072                         + env->icount_extra);
1073         env->icount_decr.u32 = 0;
1074         env->icount_extra = 0;
1075     }
1076     return ret;
1077 }
1078
1079 bool cpu_exec_all(void)
1080 {
1081     int r;
1082
1083     /* Account partial waits to the vm_clock.  */
1084     qemu_clock_warp(vm_clock);
1085
1086     if (next_cpu == NULL) {
1087         next_cpu = first_cpu;
1088     }
1089     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1090         CPUState *env = next_cpu;
1091
1092         qemu_clock_enable(vm_clock,
1093                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1094
1095 #ifndef CONFIG_IOTHREAD
1096         if (qemu_alarm_pending()) {
1097             break;
1098         }
1099 #endif
1100         if (cpu_can_run(env)) {
1101             if (kvm_enabled()) {
1102                 r = kvm_cpu_exec(env);
1103                 qemu_kvm_eat_signals(env);
1104             } else {
1105                 r = tcg_cpu_exec(env);
1106             }
1107             if (r == EXCP_DEBUG) {
1108                 cpu_handle_guest_debug(env);
1109                 break;
1110             }
1111         } else if (env->stop || env->stopped) {
1112             break;
1113         }
1114     }
1115     exit_request = 0;
1116     return !all_cpu_threads_idle();
1117 }
1118
1119 void set_numa_modes(void)
1120 {
1121     CPUState *env;
1122     int i;
1123
1124     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1125         for (i = 0; i < nb_numa_nodes; i++) {
1126             if (node_cpumask[i] & (1 << env->cpu_index)) {
1127                 env->numa_node = i;
1128             }
1129         }
1130     }
1131 }
1132
1133 void set_cpu_log(const char *optarg)
1134 {
1135     int mask;
1136     const CPULogItem *item;
1137
1138     mask = cpu_str_to_log_mask(optarg);
1139     if (!mask) {
1140         printf("Log items (comma separated):\n");
1141         for (item = cpu_log_items; item->mask != 0; item++) {
1142             printf("%-10s %s\n", item->name, item->help);
1143         }
1144         exit(1);
1145     }
1146     cpu_set_log(mask);
1147 }
1148
1149 void set_cpu_log_filename(const char *optarg)
1150 {
1151     cpu_set_log_filename(optarg);
1152 }
1153
1154 /* Return the virtual CPU time, based on the instruction counter.  */
1155 int64_t cpu_get_icount(void)
1156 {
1157     int64_t icount;
1158     CPUState *env = cpu_single_env;;
1159
1160     icount = qemu_icount;
1161     if (env) {
1162         if (!can_do_io(env)) {
1163             fprintf(stderr, "Bad clock read\n");
1164         }
1165         icount -= (env->icount_decr.u16.low + env->icount_extra);
1166     }
1167     return qemu_icount_bias + (icount << icount_time_shift);
1168 }
1169
1170 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1171 {
1172     /* XXX: implement xxx_cpu_list for targets that still miss it */
1173 #if defined(cpu_list_id)
1174     cpu_list_id(f, cpu_fprintf, optarg);
1175 #elif defined(cpu_list)
1176     cpu_list(f, cpu_fprintf); /* deprecated */
1177 #endif
1178 }