qemu/qemu-kvm.c

   1 /*
   2  * qemu/kvm integration
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  *
   6  * Licensed under the terms of the GNU GPL version 2 or higher.
   7  */
   8 #include "config.h"
   9 #include "config-host.h"
  10
  11 int kvm_allowed = 1;
  12 int kvm_irqchip = 1;
  13 int kvm_pit = 1;
  14
  15 #include <assert.h>
  16 #include <string.h>
  17 #include "hw/hw.h"
  18 #include "sysemu.h"
  19 #include "qemu-common.h"
  20 #include "console.h"
  21 #include "block.h"
  22
  23 #include "qemu-kvm.h"
  24 #include <libkvm.h>
  25 #include <pthread.h>
  26 #include <sys/utsname.h>
  27 #include <sys/syscall.h>
  28 #include <sys/mman.h>
  29
  30 #define bool _Bool
  31 #define false 0
  32 #define true 1
  33
  34 extern void perror(const char *s);
  35
  36 kvm_context_t kvm_context;
  37
  38 extern int smp_cpus;
  39
  40 pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
  41 pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
  42 pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
  43 pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
  44 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
  45 __thread struct vcpu_info *vcpu;
  46
  47 static int qemu_system_ready;
  48
  49 #define SIG_IPI (SIGRTMIN+4)
  50
  51 struct qemu_kvm_work_item {
  52     struct qemu_kvm_work_item *next;
  53     void (*func)(void *data);
  54     void *data;
  55     bool done;
  56 };
  57
  58 struct vcpu_info {
  59     CPUState *env;
  60     int sipi_needed;
  61     int init;
  62     pthread_t thread;
  63     int signalled;
  64     int stop;
  65     int stopped;
  66     int created;
  67     struct qemu_kvm_work_item *queued_work_first, *queued_work_last;
  68 } vcpu_info[256];
  69
  70 pthread_t io_thread;
  71 static int io_thread_fd = -1;
  72 static int io_thread_sigfd = -1;
  73
  74 static int kvm_debug_stop_requested;
  75
  76 static inline unsigned long kvm_get_thread_id(void)
  77 {
  78     return syscall(SYS_gettid);
  79 }
  80
  81 static void qemu_cond_wait(pthread_cond_t *cond)
  82 {
  83     CPUState *env = cpu_single_env;
  84     static const struct timespec ts = {
  85         .tv_sec = 0,
  86         .tv_nsec = 100000,
  87     };
  88
  89     pthread_cond_timedwait(cond, &qemu_mutex, &ts);
  90     /* If we're the I/O thread, some other thread may be waiting for aio
  91      * completion */
  92     if (!vcpu)
  93         qemu_aio_poll(NULL);
  94     cpu_single_env = env;
  95 }
  96
  97 CPUState *qemu_kvm_cpu_env(int index)
  98 {
  99     return vcpu_info[index].env;
 100 }
 101
 102 static void sig_ipi_handler(int n)
 103 {
 104 }
 105
 106 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 107 {
 108     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
 109     struct qemu_kvm_work_item wi;
 110
 111     if (vi == vcpu) {
 112         func(data);
 113         return;
 114     }
 115
 116     wi.func = func;
 117     wi.data = data;
 118     if (!vi->queued_work_first)
 119         vi->queued_work_first = &wi;
 120     else
 121         vi->queued_work_last->next = &wi;
 122     vi->queued_work_last = &wi;
 123     wi.next = NULL;
 124     wi.done = false;
 125
 126     pthread_kill(vi->thread, SIG_IPI);
 127     while (!wi.done)
 128         qemu_cond_wait(&qemu_work_cond);
 129 }
 130
 131 void kvm_update_interrupt_request(CPUState *env)
 132 {
 133     int signal = 0;
 134
 135     if (env) {
 136         if (!vcpu)
 137             signal = 1;
 138         if (vcpu && env != vcpu->env && !vcpu_info[env->cpu_index].signalled)
 139             signal = 1;
 140
 141         if (signal) {
 142             vcpu_info[env->cpu_index].signalled = 1;
 143                 if (vcpu_info[env->cpu_index].thread)
 144                     pthread_kill(vcpu_info[env->cpu_index].thread, SIG_IPI);
 145         }
 146     }
 147 }
 148
 149 void kvm_update_after_sipi(CPUState *env)
 150 {
 151     vcpu_info[env->cpu_index].sipi_needed = 1;
 152     kvm_update_interrupt_request(env);
 153 }
 154
 155 void kvm_apic_init(CPUState *env)
 156 {
 157     if (env->cpu_index != 0)
 158         vcpu_info[env->cpu_index].init = 1;
 159     kvm_update_interrupt_request(env);
 160 }
 161
 162 #include <signal.h>
 163
 164 static int try_push_interrupts(void *opaque)
 165 {
 166     return kvm_arch_try_push_interrupts(opaque);
 167 }
 168
 169 static void post_kvm_run(void *opaque, int vcpu)
 170 {
 171
 172     pthread_mutex_lock(&qemu_mutex);
 173     kvm_arch_post_kvm_run(opaque, vcpu);
 174 }
 175
 176 static int pre_kvm_run(void *opaque, int vcpu)
 177 {
 178     CPUState *env = qemu_kvm_cpu_env(vcpu);
 179
 180     kvm_arch_pre_kvm_run(opaque, vcpu);
 181
 182     if (env->interrupt_request & CPU_INTERRUPT_EXIT)
 183         return 1;
 184     pthread_mutex_unlock(&qemu_mutex);
 185     return 0;
 186 }
 187
 188 static void kvm_do_load_registers(void *_env)
 189 {
 190     CPUState *env = _env;
 191
 192     kvm_arch_load_regs(env);
 193 }
 194
 195 void kvm_load_registers(CPUState *env)
 196 {
 197     if (kvm_enabled() && qemu_system_ready)
 198         on_vcpu(env, kvm_do_load_registers, env);
 199 }
 200
 201 static void kvm_do_save_registers(void *_env)
 202 {
 203     CPUState *env = _env;
 204
 205     kvm_arch_save_regs(env);
 206 }
 207
 208 void kvm_save_registers(CPUState *env)
 209 {
 210     if (kvm_enabled())
 211         on_vcpu(env, kvm_do_save_registers, env);
 212 }
 213
 214 int kvm_cpu_exec(CPUState *env)
 215 {
 216     int r;
 217
 218     r = kvm_run(kvm_context, env->cpu_index);
 219     if (r < 0) {
 220         printf("kvm_run returned %d\n", r);
 221         exit(1);
 222     }
 223
 224     return 0;
 225 }
 226
 227 extern int vm_running;
 228
 229 static int has_work(CPUState *env)
 230 {
 231     if (!vm_running || (env && vcpu_info[env->cpu_index].stopped))
 232         return 0;
 233     if (!env->halted)
 234         return 1;
 235     return kvm_arch_has_work(env);
 236 }
 237
 238 static void flush_queued_work(CPUState *env)
 239 {
 240     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
 241     struct qemu_kvm_work_item *wi;
 242
 243     if (!vi->queued_work_first)
 244         return;
 245
 246     while ((wi = vi->queued_work_first)) {
 247         vi->queued_work_first = wi->next;
 248         wi->func(wi->data);
 249         wi->done = true;
 250     }
 251     vi->queued_work_last = NULL;
 252     pthread_cond_broadcast(&qemu_work_cond);
 253 }
 254
 255 static void kvm_main_loop_wait(CPUState *env, int timeout)
 256 {
 257     struct timespec ts;
 258     int r, e;
 259     siginfo_t siginfo;
 260     sigset_t waitset;
 261
 262     pthread_mutex_unlock(&qemu_mutex);
 263
 264     ts.tv_sec = timeout / 1000;
 265     ts.tv_nsec = (timeout % 1000) * 1000000;
 266     sigemptyset(&waitset);
 267     sigaddset(&waitset, SIG_IPI);
 268
 269     r = sigtimedwait(&waitset, &siginfo, &ts);
 270     e = errno;
 271
 272     pthread_mutex_lock(&qemu_mutex);
 273
 274     if (r == -1 && !(e == EAGAIN || e == EINTR)) {
 275         printf("sigtimedwait: %s\n", strerror(e));
 276         exit(1);
 277     }
 278
 279     cpu_single_env = env;
 280     flush_queued_work(env);
 281
 282     if (vcpu_info[env->cpu_index].stop) {
 283         vcpu_info[env->cpu_index].stop = 0;
 284         vcpu_info[env->cpu_index].stopped = 1;
 285         pthread_cond_signal(&qemu_pause_cond);
 286     }
 287
 288     vcpu_info[env->cpu_index].signalled = 0;
 289 }
 290
 291 static int all_threads_paused(void)
 292 {
 293     int i;
 294
 295     for (i = 0; i < smp_cpus; ++i)
 296         if (vcpu_info[i].stop)
 297             return 0;
 298     return 1;
 299 }
 300
 301 static void pause_all_threads(void)
 302 {
 303     int i;
 304
 305     assert(!cpu_single_env);
 306
 307     for (i = 0; i < smp_cpus; ++i) {
 308         vcpu_info[i].stop = 1;
 309         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 310     }
 311     while (!all_threads_paused())
 312         qemu_cond_wait(&qemu_pause_cond);
 313 }
 314
 315 static void resume_all_threads(void)
 316 {
 317     int i;
 318
 319     assert(!cpu_single_env);
 320
 321     for (i = 0; i < smp_cpus; ++i) {
 322         vcpu_info[i].stop = 0;
 323         vcpu_info[i].stopped = 0;
 324         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 325     }
 326 }
 327
 328 static void kvm_vm_state_change_handler(void *context, int running)
 329 {
 330     if (running)
 331         resume_all_threads();
 332     else
 333         pause_all_threads();
 334 }
 335
 336 static void update_regs_for_sipi(CPUState *env)
 337 {
 338     kvm_arch_update_regs_for_sipi(env);
 339     vcpu_info[env->cpu_index].sipi_needed = 0;
 340     vcpu_info[env->cpu_index].init = 0;
 341 }
 342
 343 static void update_regs_for_init(CPUState *env)
 344 {
 345     cpu_reset(env);
 346     kvm_arch_load_regs(env);
 347 }
 348
 349 static void setup_kernel_sigmask(CPUState *env)
 350 {
 351     sigset_t set;
 352
 353     sigemptyset(&set);
 354     sigaddset(&set, SIGUSR2);
 355     sigaddset(&set, SIGIO);
 356     sigaddset(&set, SIGALRM);
 357     sigprocmask(SIG_BLOCK, &set, NULL);
 358
 359     sigprocmask(SIG_BLOCK, NULL, &set);
 360     sigdelset(&set, SIG_IPI);
 361
 362     kvm_set_signal_mask(kvm_context, env->cpu_index, &set);
 363 }
 364
 365 void qemu_kvm_system_reset(void)
 366 {
 367     int i;
 368
 369     pause_all_threads();
 370
 371     qemu_system_reset();
 372
 373     for (i = 0; i < smp_cpus; ++i)
 374         kvm_arch_cpu_reset(vcpu_info[i].env);
 375
 376     resume_all_threads();
 377 }
 378
 379 static int kvm_main_loop_cpu(CPUState *env)
 380 {
 381     struct vcpu_info *info = &vcpu_info[env->cpu_index];
 382
 383     setup_kernel_sigmask(env);
 384
 385     pthread_mutex_lock(&qemu_mutex);
 386     if (kvm_irqchip_in_kernel(kvm_context))
 387         env->halted = 0;
 388
 389     kvm_qemu_init_env(env);
 390 #ifdef TARGET_I386
 391     kvm_tpr_vcpu_start(env);
 392 #endif
 393
 394     cpu_single_env = env;
 395     kvm_load_registers(env);
 396
 397     while (1) {
 398         while (!has_work(env))
 399             kvm_main_loop_wait(env, 1000);
 400         if (env->interrupt_request & CPU_INTERRUPT_HARD)
 401             env->halted = 0;
 402         if (!kvm_irqchip_in_kernel(kvm_context) && info->sipi_needed)
 403             update_regs_for_sipi(env);
 404         if (!kvm_irqchip_in_kernel(kvm_context) && info->init)
 405             update_regs_for_init(env);
 406         if (!env->halted && !info->init)
 407             kvm_cpu_exec(env);
 408         env->interrupt_request &= ~CPU_INTERRUPT_EXIT;
 409         kvm_main_loop_wait(env, 0);
 410     }
 411     pthread_mutex_unlock(&qemu_mutex);
 412     return 0;
 413 }
 414
 415 static void *ap_main_loop(void *_env)
 416 {
 417     CPUState *env = _env;
 418     sigset_t signals;
 419
 420     vcpu = &vcpu_info[env->cpu_index];
 421     vcpu->env = env;
 422     vcpu->env->thread_id = kvm_get_thread_id();
 423     sigfillset(&signals);
 424     sigprocmask(SIG_BLOCK, &signals, NULL);
 425     kvm_create_vcpu(kvm_context, env->cpu_index);
 426     kvm_qemu_init_env(env);
 427
 428     /* signal VCPU creation */
 429     pthread_mutex_lock(&qemu_mutex);
 430     vcpu->created = 1;
 431     pthread_cond_signal(&qemu_vcpu_cond);
 432
 433     /* and wait for machine initialization */
 434     while (!qemu_system_ready)
 435         qemu_cond_wait(&qemu_system_cond);
 436     pthread_mutex_unlock(&qemu_mutex);
 437
 438     kvm_main_loop_cpu(env);
 439     return NULL;
 440 }
 441
 442 void kvm_init_new_ap(int cpu, CPUState *env)
 443 {
 444     pthread_create(&vcpu_info[cpu].thread, NULL, ap_main_loop, env);
 445
 446     while (vcpu_info[cpu].created == 0)
 447         qemu_cond_wait(&qemu_vcpu_cond);
 448 }
 449
 450 int kvm_init_ap(void)
 451 {
 452 #ifdef TARGET_I386
 453     kvm_tpr_opt_setup();
 454 #endif
 455     qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);
 456
 457     signal(SIG_IPI, sig_ipi_handler);
 458     return 0;
 459 }
 460
 461 void qemu_kvm_notify_work(void)
 462 {
 463     uint64_t value = 1;
 464     char buffer[8];
 465     size_t offset = 0;
 466
 467     if (io_thread_fd == -1)
 468         return;
 469
 470     memcpy(buffer, &value, sizeof(value));
 471
 472     while (offset < 8) {
 473         ssize_t len;
 474
 475         len = write(io_thread_fd, buffer + offset, 8 - offset);
 476         if (len == -1 && errno == EINTR)
 477             continue;
 478
 479         if (len <= 0)
 480             break;
 481
 482         offset += len;
 483     }
 484
 485     if (offset != 8)
 486         fprintf(stderr, "failed to notify io thread\n");
 487 }
 488
 489 /* If we have signalfd, we mask out the signals we want to handle and then
 490  * use signalfd to listen for them.  We rely on whatever the current signal
 491  * handler is to dispatch the signals when we receive them.
 492  */
 493
 494 static void sigfd_handler(void *opaque)
 495 {
 496     int fd = (unsigned long)opaque;
 497     struct signalfd_siginfo info;
 498     struct sigaction action;
 499     ssize_t len;
 500
 501     while (1) {
 502         do {
 503             len = read(fd, &info, sizeof(info));
 504         } while (len == -1 && errno == EINTR);
 505
 506         if (len == -1 && errno == EAGAIN)
 507             break;
 508
 509         if (len != sizeof(info)) {
 510             printf("read from sigfd returned %ld: %m\n", len);
 511             return;
 512         }
 513
 514         sigaction(info.ssi_signo, NULL, &action);
 515         if (action.sa_handler)
 516             action.sa_handler(info.ssi_signo);
 517
 518     }
 519 }
 520
 521 /* Used to break IO thread out of select */
 522 static void io_thread_wakeup(void *opaque)
 523 {
 524     int fd = (unsigned long)opaque;
 525     char buffer[8];
 526     size_t offset = 0;
 527
 528     while (offset < 8) {
 529         ssize_t len;
 530
 531         len = read(fd, buffer + offset, 8 - offset);
 532         if (len == -1 && errno == EINTR)
 533             continue;
 534
 535         if (len <= 0)
 536             break;
 537
 538         offset += len;
 539     }
 540 }
 541
 542 int kvm_main_loop(void)
 543 {
 544     int fds[2];
 545     sigset_t mask;
 546     int sigfd;
 547
 548     io_thread = pthread_self();
 549     qemu_system_ready = 1;
 550
 551     if (kvm_eventfd(fds) == -1) {
 552         fprintf(stderr, "failed to create eventfd\n");
 553         return -errno;
 554     }
 555
 556     qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
 557                          (void *)(unsigned long)fds[0]);
 558
 559     io_thread_fd = fds[1];
 560
 561     sigemptyset(&mask);
 562     sigaddset(&mask, SIGIO);
 563     sigaddset(&mask, SIGALRM);
 564     sigprocmask(SIG_BLOCK, &mask, NULL);
 565
 566     sigfd = kvm_signalfd(&mask);
 567     if (sigfd == -1) {
 568         fprintf(stderr, "failed to create signalfd\n");
 569         return -errno;
 570     }
 571
 572     fcntl(sigfd, F_SETFL, O_NONBLOCK);
 573
 574     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 575                          (void *)(unsigned long)sigfd);
 576
 577     pthread_cond_broadcast(&qemu_system_cond);
 578
 579     io_thread_sigfd = sigfd;
 580     cpu_single_env = NULL;
 581
 582     while (1) {
 583         main_loop_wait(1000);
 584         if (qemu_shutdown_requested())
 585             break;
 586         else if (qemu_powerdown_requested())
 587             qemu_system_powerdown();
 588         else if (qemu_reset_requested())
 589             qemu_kvm_system_reset();
 590         else if (kvm_debug_stop_requested) {
 591             vm_stop(EXCP_DEBUG);
 592             kvm_debug_stop_requested = 0;
 593         }
 594     }
 595
 596     pause_all_threads();
 597     pthread_mutex_unlock(&qemu_mutex);
 598
 599     return 0;
 600 }
 601
 602 static int kvm_debug(void *opaque, int vcpu)
 603 {
 604     kvm_debug_stop_requested = 1;
 605     vcpu_info[vcpu].stopped = 1;
 606     return 1;
 607 }
 608
 609 static int kvm_inb(void *opaque, uint16_t addr, uint8_t *data)
 610 {
 611     *data = cpu_inb(0, addr);
 612     return 0;
 613 }
 614
 615 static int kvm_inw(void *opaque, uint16_t addr, uint16_t *data)
 616 {
 617     *data = cpu_inw(0, addr);
 618     return 0;
 619 }
 620
 621 static int kvm_inl(void *opaque, uint16_t addr, uint32_t *data)
 622 {
 623     *data = cpu_inl(0, addr);
 624     return 0;
 625 }
 626
 627 #define PM_IO_BASE 0xb000
 628
 629 static int kvm_outb(void *opaque, uint16_t addr, uint8_t data)
 630 {
 631     if (addr == 0xb2) {
 632         switch (data) {
 633         case 0: {
 634             cpu_outb(0, 0xb3, 0);
 635             break;
 636         }
 637         case 0xf0: {
 638             unsigned x;
 639
 640             /* enable acpi */
 641             x = cpu_inw(0, PM_IO_BASE + 4);
 642             x &= ~1;
 643             cpu_outw(0, PM_IO_BASE + 4, x);
 644             break;
 645         }
 646         case 0xf1: {
 647             unsigned x;
 648
 649             /* enable acpi */
 650             x = cpu_inw(0, PM_IO_BASE + 4);
 651             x |= 1;
 652             cpu_outw(0, PM_IO_BASE + 4, x);
 653             break;
 654         }
 655         default:
 656             break;
 657         }
 658         return 0;
 659     }
 660     cpu_outb(0, addr, data);
 661     return 0;
 662 }
 663
 664 static int kvm_outw(void *opaque, uint16_t addr, uint16_t data)
 665 {
 666     cpu_outw(0, addr, data);
 667     return 0;
 668 }
 669
 670 static int kvm_outl(void *opaque, uint16_t addr, uint32_t data)
 671 {
 672     cpu_outl(0, addr, data);
 673     return 0;
 674 }
 675
 676 static int kvm_mmio_read(void *opaque, uint64_t addr, uint8_t *data, int len)
 677 {
 678         cpu_physical_memory_rw(addr, data, len, 0);
 679         return 0;
 680 }
 681
 682 static int kvm_mmio_write(void *opaque, uint64_t addr, uint8_t *data, int len)
 683 {
 684         cpu_physical_memory_rw(addr, data, len, 1);
 685         return 0;
 686 }
 687
 688 static int kvm_io_window(void *opaque)
 689 {
 690     return 1;
 691 }
 692
 693
 694 static int kvm_halt(void *opaque, int vcpu)
 695 {
 696     return kvm_arch_halt(opaque, vcpu);
 697 }
 698
 699 static int kvm_shutdown(void *opaque, int vcpu)
 700 {
 701     /* stop the current vcpu from going back to guest mode */
 702     vcpu_info[cpu_single_env->cpu_index].stopped = 1;
 703
 704     qemu_system_reset_request();
 705     return 1;
 706 }
 707
 708 static struct kvm_callbacks qemu_kvm_ops = {
 709     .debug = kvm_debug,
 710     .inb   = kvm_inb,
 711     .inw   = kvm_inw,
 712     .inl   = kvm_inl,
 713     .outb  = kvm_outb,
 714     .outw  = kvm_outw,
 715     .outl  = kvm_outl,
 716     .mmio_read = kvm_mmio_read,
 717     .mmio_write = kvm_mmio_write,
 718     .halt  = kvm_halt,
 719     .shutdown = kvm_shutdown,
 720     .io_window = kvm_io_window,
 721     .try_push_interrupts = try_push_interrupts,
 722     .post_kvm_run = post_kvm_run,
 723     .pre_kvm_run = pre_kvm_run,
 724 #ifdef TARGET_I386
 725     .tpr_access = handle_tpr_access,
 726 #endif
 727 #ifdef TARGET_PPC
 728     .powerpc_dcr_read = handle_powerpc_dcr_read,
 729     .powerpc_dcr_write = handle_powerpc_dcr_write,
 730 #endif
 731 };
 732
 733 int kvm_qemu_init()
 734 {
 735     /* Try to initialize kvm */
 736     kvm_context = kvm_init(&qemu_kvm_ops, cpu_single_env);
 737     if (!kvm_context) {
 738         return -1;
 739     }
 740     pthread_mutex_lock(&qemu_mutex);
 741
 742     return 0;
 743 }
 744
 745 int kvm_qemu_create_context(void)
 746 {
 747     int r;
 748     if (!kvm_irqchip) {
 749         kvm_disable_irqchip_creation(kvm_context);
 750     }
 751     if (!kvm_pit) {
 752         kvm_disable_pit_creation(kvm_context);
 753     }
 754     if (kvm_create(kvm_context, phys_ram_size, (void**)&phys_ram_base) < 0) {
 755         kvm_qemu_destroy();
 756         return -1;
 757     }
 758     r = kvm_arch_qemu_create_context();
 759     if(r <0)
 760         kvm_qemu_destroy();
 761     return 0;
 762 }
 763
 764 void kvm_qemu_destroy(void)
 765 {
 766     kvm_finalize(kvm_context);
 767 }
 768
 769 void kvm_cpu_register_physical_memory(target_phys_addr_t start_addr,
 770                                       unsigned long size,
 771                                       unsigned long phys_offset)
 772 {
 773     int r = 0;
 774     if (!(phys_offset & ~TARGET_PAGE_MASK)) {
 775         r = kvm_is_allocated_mem(kvm_context, start_addr, size);
 776         if (r)
 777             return;
 778         r = kvm_is_intersecting_mem(kvm_context, start_addr);
 779         if (r)
 780             kvm_create_mem_hole(kvm_context, start_addr, size);
 781         r = kvm_register_phys_mem(kvm_context, start_addr,
 782                                             phys_ram_base + phys_offset,
 783                                             size, 0);
 784     }
 785     if (phys_offset & IO_MEM_ROM) {
 786         phys_offset &= ~IO_MEM_ROM;
 787         r = kvm_is_intersecting_mem(kvm_context, start_addr);
 788         if (r)
 789             kvm_create_mem_hole(kvm_context, start_addr, size);
 790         r = kvm_register_phys_mem(kvm_context, start_addr,
 791                                             phys_ram_base + phys_offset,
 792                                             size, 0);
 793     }
 794     if (r < 0) {
 795         printf("kvm_cpu_register_physical_memory: failed\n");
 796         exit(1);
 797     }
 798     return;
 799 }
 800
 801 int kvm_setup_guest_memory(void *area, unsigned long size)
 802 {
 803     int ret = 0;
 804
 805 #ifdef MADV_DONTFORK
 806     if (kvm_enabled() && !kvm_has_sync_mmu(kvm_context))
 807         ret = madvise(area, size, MADV_DONTFORK);
 808 #endif
 809
 810     if (ret)
 811         perror ("madvise");
 812
 813     return ret;
 814 }
 815
 816 int kvm_qemu_check_extension(int ext)
 817 {
 818     return kvm_check_extension(kvm_context, ext);
 819 }
 820
 821 int kvm_qemu_init_env(CPUState *cenv)
 822 {
 823     return kvm_arch_qemu_init_env(cenv);
 824 }
 825
 826 struct kvm_guest_debug_data {
 827     struct kvm_debug_guest dbg;
 828     int err;
 829 };
 830
 831 void kvm_invoke_guest_debug(void *data)
 832 {
 833     struct kvm_guest_debug_data *dbg_data = data;
 834
 835     dbg_data->err = kvm_guest_debug(kvm_context, cpu_single_env->cpu_index,
 836                                     &dbg_data->dbg);
 837 }
 838
 839 int kvm_update_debugger(CPUState *env)
 840 {
 841     struct kvm_guest_debug_data data;
 842     int i;
 843
 844     memset(data.dbg.breakpoints, 0, sizeof(data.dbg.breakpoints));
 845
 846     data.dbg.enabled = 0;
 847     if (env->nb_breakpoints || env->singlestep_enabled) {
 848         data.dbg.enabled = 1;
 849         for (i = 0; i < 4 && i < env->nb_breakpoints; ++i) {
 850             data.dbg.breakpoints[i].enabled = 1;
 851             data.dbg.breakpoints[i].address = env->breakpoints[i];
 852         }
 853         data.dbg.singlestep = env->singlestep_enabled;
 854     }
 855     on_vcpu(env, kvm_invoke_guest_debug, &data);
 856     return data.err;
 857 }
 858
 859
 860 /*
 861  * dirty pages logging
 862  */
 863 /* FIXME: use unsigned long pointer instead of unsigned char */
 864 unsigned char *kvm_dirty_bitmap = NULL;
 865 int kvm_physical_memory_set_dirty_tracking(int enable)
 866 {
 867     int r = 0;
 868
 869     if (!kvm_enabled())
 870         return 0;
 871
 872     if (enable) {
 873         if (!kvm_dirty_bitmap) {
 874             unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
 875             kvm_dirty_bitmap = qemu_malloc(bitmap_size);
 876             if (kvm_dirty_bitmap == NULL) {
 877                 perror("Failed to allocate dirty pages bitmap");
 878                 r=-1;
 879             }
 880             else {
 881                 r = kvm_dirty_pages_log_enable_all(kvm_context);
 882             }
 883         }
 884     }
 885     else {
 886         if (kvm_dirty_bitmap) {
 887             r = kvm_dirty_pages_log_reset(kvm_context);
 888             qemu_free(kvm_dirty_bitmap);
 889             kvm_dirty_bitmap = NULL;
 890         }
 891     }
 892     return r;
 893 }
 894
 895 /* get kvm's dirty pages bitmap and update qemu's */
 896 int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 897                                   unsigned char *bitmap,
 898                                   unsigned int offset,
 899                                   unsigned long mem_size)
 900 {
 901     unsigned int i, j, n=0;
 902     unsigned char c;
 903     unsigned page_number, addr, addr1;
 904     unsigned int len = ((mem_size/TARGET_PAGE_SIZE) + 7) / 8;
 905
 906     /*
 907      * bitmap-traveling is faster than memory-traveling (for addr...)
 908      * especially when most of the memory is not dirty.
 909      */
 910     for (i=0; i<len; i++) {
 911         c = bitmap[i];
 912         while (c>0) {
 913             j = ffsl(c) - 1;
 914             c &= ~(1u<<j);
 915             page_number = i * 8 + j;
 916             addr1 = page_number * TARGET_PAGE_SIZE;
 917             addr  = offset + addr1;
 918             cpu_physical_memory_set_dirty(addr);
 919             n++;
 920         }
 921     }
 922     return 0;
 923 }
 924 int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
 925                             void *bitmap, void *opaque)
 926 {
 927     return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
 928 }
 929
 930 /*
 931  * get kvm's dirty pages bitmap and update qemu's
 932  * we only care about physical ram, which resides in slots 0 and 3
 933  */
 934 int kvm_update_dirty_pages_log(void)
 935 {
 936     int r = 0;
 937
 938
 939     r = kvm_get_dirty_pages_range(kvm_context, 0, phys_ram_size,
 940                                   kvm_dirty_bitmap, NULL,
 941                                   kvm_get_dirty_bitmap_cb);
 942     return r;
 943 }
 944
 945 int kvm_get_phys_ram_page_bitmap(unsigned char *bitmap)
 946 {
 947     unsigned int bsize  = BITMAP_SIZE(phys_ram_size);
 948     unsigned int brsize = BITMAP_SIZE(ram_size);
 949     unsigned int extra_pages = (phys_ram_size - ram_size) / TARGET_PAGE_SIZE;
 950     unsigned int extra_bytes = (extra_pages +7)/8;
 951     unsigned int hole_start = BITMAP_SIZE(0xa0000);
 952     unsigned int hole_end   = BITMAP_SIZE(0xc0000);
 953
 954     memset(bitmap, 0xFF, brsize + extra_bytes);
 955     memset(bitmap + hole_start, 0, hole_end - hole_start);
 956     memset(bitmap + brsize + extra_bytes, 0, bsize - brsize - extra_bytes);
 957
 958     return 0;
 959 }
 960
 961 #ifdef KVM_CAP_IRQCHIP
 962
 963 int kvm_set_irq(int irq, int level)
 964 {
 965     return kvm_set_irq_level(kvm_context, irq, level);
 966 }
 967
 968 #endif
 969
 970 int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
 971 {
 972     return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
 973 }
 974
 975 void *kvm_cpu_create_phys_mem(target_phys_addr_t start_addr,
 976                               unsigned long size, int log, int writable)
 977 {
 978     return kvm_create_phys_mem(kvm_context, start_addr, size, log, writable);
 979 }
 980
 981 void kvm_cpu_destroy_phys_mem(target_phys_addr_t start_addr,
 982                               unsigned long size)
 983 {
 984     kvm_destroy_phys_mem(kvm_context, start_addr, size);
 985 }
 986
 987 void kvm_mutex_unlock(void)
 988 {
 989     assert(!cpu_single_env);
 990     pthread_mutex_unlock(&qemu_mutex);
 991 }
 992
 993 void kvm_mutex_lock(void)
 994 {
 995     pthread_mutex_lock(&qemu_mutex);
 996     cpu_single_env = NULL;
 997 }
 998
 999 int qemu_kvm_register_coalesced_mmio(target_phys_addr_t addr, unsigned int size)
1000 {
1001     return kvm_register_coalesced_mmio(kvm_context, addr, size);
1002 }
1003
1004 int qemu_kvm_unregister_coalesced_mmio(target_phys_addr_t addr,
1005                                        unsigned int size)
1006 {
1007     return kvm_unregister_coalesced_mmio(kvm_context, addr, size);
1008 }