libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include "libkvm.h"
  38
  39 #if defined(__x86_64__) || defined(__i386__)
  40 #include "kvm-x86.h"
  41 #endif
  42
  43 #if defined(__ia64__)
  44 #include "kvm-ia64.h"
  45 #endif
  46
  47 #if defined(__powerpc__)
  48 #include "kvm-powerpc.h"
  49 #endif
  50
  51 int kvm_abi = EXPECTED_KVM_API_VERSION;
  52 int kvm_page_size;
  53
  54 struct slot_info {
  55         unsigned long phys_addr;
  56         unsigned long len;
  57         int user_alloc;
  58         unsigned long userspace_addr;
  59         unsigned flags;
  60 };
  61
  62 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  63
  64 void init_slots(void)
  65 {
  66         int i;
  67
  68         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  69                 slots[i].len = 0;
  70 }
  71
  72 int get_free_slot(kvm_context_t kvm)
  73 {
  74         int i;
  75         int tss_ext;
  76
  77 #ifdef KVM_CAP_SET_TSS_ADDR
  78         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  79 #else
  80         tss_ext = 0;
  81 #endif
  82
  83         /*
  84          * on older kernels where the set tss ioctl is not supprted we must save
  85          * slot 0 to hold the extended memory, as the vmx will use the last 3
  86          * pages of this slot.
  87          */
  88         if (tss_ext > 0)
  89                 i = 0;
  90         else
  91                 i = 1;
  92
  93         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  94                 if (!slots[i].len)
  95                         return i;
  96         return -1;
  97 }
  98
  99 void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 100                    int user_alloc, unsigned long userspace_addr, unsigned flags)
 101 {
 102         slots[slot].phys_addr = phys_addr;
 103         slots[slot].len = len;
 104         slots[slot].user_alloc = user_alloc;
 105         slots[slot].userspace_addr = userspace_addr;
 106         slots[slot].flags = flags;
 107 }
 108
 109 void free_slot(int slot)
 110 {
 111         slots[slot].len = 0;
 112 }
 113
 114 int get_slot(unsigned long phys_addr)
 115 {
 116         int i;
 117
 118         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 119                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 120                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 121                         return i;
 122         }
 123         return -1;
 124 }
 125
 126 int get_intersecting_slot(unsigned long phys_addr)
 127 {
 128         int i;
 129
 130         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 131                 if (slots[i].len && slots[i].phys_addr < phys_addr &&
 132                     (slots[i].phys_addr + slots[i].len) > phys_addr)
 133                         return i;
 134         return -1;
 135 }
 136
 137 /*
 138  * dirty pages logging control
 139  */
 140 static int kvm_dirty_pages_log_change(kvm_context_t kvm, unsigned long phys_addr
 141                                       , __u32 flag)
 142 {
 143         int r;
 144         int slot;
 145
 146         slot = get_slot(phys_addr);
 147         if (slot == -1) {
 148                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 149                 return 1;
 150         }
 151         flag |= slots[slot].flags;
 152 #ifdef KVM_CAP_USER_MEMORY
 153         if (slots[slot].user_alloc) {
 154                 struct kvm_userspace_memory_region mem = {
 155                         .slot = slot,
 156                         .memory_size = slots[slot].len,
 157                         .guest_phys_addr = slots[slot].phys_addr,
 158                         .userspace_addr = slots[slot].userspace_addr,
 159                         .flags = flag,
 160                 };
 161                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 162         }
 163 #endif
 164         if (!slots[slot].user_alloc) {
 165                 struct kvm_memory_region mem = {
 166                         .slot = slot,
 167                         .memory_size = slots[slot].len,
 168                         .guest_phys_addr = slots[slot].phys_addr,
 169                         .flags = flag,
 170                 };
 171                 r = ioctl(kvm->vm_fd, KVM_SET_MEMORY_REGION, &mem);
 172         }
 173         if (r == -1)
 174                 fprintf(stderr, "%s: %m\n", __FUNCTION__);
 175         return r;
 176 }
 177
 178 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm, __u32 flag)
 179 {
 180         int i, r;
 181
 182         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 183                 if (slots[i].len)
 184                         r = kvm_dirty_pages_log_change(kvm, slots[i].phys_addr,
 185                                                        flag);
 186         }
 187         return r;
 188 }
 189
 190 /**
 191  * Enable dirty page logging for all memory regions
 192  */
 193 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 194 {
 195         if (kvm->dirty_pages_log_all)
 196                 return 0;
 197         kvm->dirty_pages_log_all = 1;
 198         return kvm_dirty_pages_log_change_all(kvm, KVM_MEM_LOG_DIRTY_PAGES);
 199 }
 200
 201 /**
 202  * Enable dirty page logging only for memory regions that were created with
 203  *     dirty logging enabled (disable for all other memory regions).
 204  */
 205 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 206 {
 207         if (!kvm->dirty_pages_log_all)
 208                 return 0;
 209         kvm->dirty_pages_log_all = 0;
 210         return kvm_dirty_pages_log_change_all(kvm, 0);
 211 }
 212
 213
 214 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 215                        void *opaque)
 216 {
 217         int fd;
 218         kvm_context_t kvm;
 219         int r;
 220
 221         fd = open("/dev/kvm", O_RDWR);
 222         if (fd == -1) {
 223                 perror("open /dev/kvm");
 224                 return NULL;
 225         }
 226         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 227         if (r == -1) {
 228             fprintf(stderr, "kvm kernel version too old: "
 229                     "KVM_GET_API_VERSION ioctl not supported\n");
 230             goto out_close;
 231         }
 232         if (r < EXPECTED_KVM_API_VERSION) {
 233                 fprintf(stderr, "kvm kernel version too old: "
 234                         "We expect API version %d or newer, but got "
 235                         "version %d\n",
 236                         EXPECTED_KVM_API_VERSION, r);
 237             goto out_close;
 238         }
 239         if (r > EXPECTED_KVM_API_VERSION) {
 240             fprintf(stderr, "kvm userspace version too old\n");
 241             goto out_close;
 242         }
 243         kvm_abi = r;
 244         kvm_page_size = getpagesize();
 245         kvm = malloc(sizeof(*kvm));
 246         kvm->fd = fd;
 247         kvm->vm_fd = -1;
 248         kvm->callbacks = callbacks;
 249         kvm->opaque = opaque;
 250         kvm->dirty_pages_log_all = 0;
 251         kvm->no_irqchip_creation = 0;
 252
 253         return kvm;
 254  out_close:
 255         close(fd);
 256         return NULL;
 257 }
 258
 259 void kvm_finalize(kvm_context_t kvm)
 260 {
 261         if (kvm->vcpu_fd[0] != -1)
 262                 close(kvm->vcpu_fd[0]);
 263         if (kvm->vm_fd != -1)
 264                 close(kvm->vm_fd);
 265         close(kvm->fd);
 266         free(kvm);
 267 }
 268
 269 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 270 {
 271         kvm->no_irqchip_creation = 1;
 272 }
 273
 274 void kvm_disable_pit_creation(kvm_context_t kvm)
 275 {
 276         kvm->no_pit_creation = 1;
 277 }
 278
 279 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 280 {
 281         long mmap_size;
 282         int r;
 283
 284         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 285         if (r == -1) {
 286                 r = -errno;
 287                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 288                 return r;
 289         }
 290         kvm->vcpu_fd[slot] = r;
 291         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 292         if (mmap_size == -1) {
 293                 r = -errno;
 294                 fprintf(stderr, "get vcpu mmap size: %m\n");
 295                 return r;
 296         }
 297         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 298                               kvm->vcpu_fd[slot], 0);
 299         if (kvm->run[slot] == MAP_FAILED) {
 300                 r = -errno;
 301                 fprintf(stderr, "mmap vcpu area: %m\n");
 302                 return r;
 303         }
 304         return 0;
 305 }
 306
 307 int kvm_create_vm(kvm_context_t kvm)
 308 {
 309         int fd = kvm->fd;
 310
 311         kvm->vcpu_fd[0] = -1;
 312
 313         fd = ioctl(fd, KVM_CREATE_VM, 0);
 314         if (fd == -1) {
 315                 fprintf(stderr, "kvm_create_vm: %m\n");
 316                 return -1;
 317         }
 318         kvm->vm_fd = fd;
 319         return 0;
 320 }
 321
 322 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 323                                        unsigned long phys_mem_bytes,
 324                                        void **vm_mem)
 325 {
 326         unsigned long memory = (phys_mem_bytes + PAGE_SIZE - 1) & PAGE_MASK;
 327         int r;
 328
 329 #ifdef KVM_CAP_USER_MEMORY
 330         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 331         if (r > 0)
 332                 return 0;
 333         else
 334 #endif
 335                 r = kvm_alloc_kernel_memory(kvm, memory, vm_mem);
 336         if (r < 0)
 337                 return r;
 338
 339         r = kvm_arch_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 340         if (r < 0)
 341                 return r;
 342
 343         kvm->physical_memory = *vm_mem;
 344         return 0;
 345 }
 346
 347 int kvm_check_extension(kvm_context_t kvm, int ext)
 348 {
 349         int ret;
 350
 351         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 352         if (ret > 0)
 353                 return 1;
 354         return 0;
 355 }
 356
 357 void kvm_create_irqchip(kvm_context_t kvm)
 358 {
 359         int r;
 360
 361         kvm->irqchip_in_kernel = 0;
 362 #ifdef KVM_CAP_IRQCHIP
 363         if (!kvm->no_irqchip_creation) {
 364                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 365                 if (r > 0) {    /* kernel irqchip supported */
 366                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 367                         if (r >= 0)
 368                                 kvm->irqchip_in_kernel = 1;
 369                         else
 370                                 printf("Create kernel PIC irqchip failed\n");
 371                 }
 372         }
 373 #endif
 374 }
 375
 376 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 377 {
 378         int r;
 379
 380         r = kvm_create_vm(kvm);
 381         if (r < 0)
 382                 return r;
 383         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 384         if (r < 0)
 385                 return r;
 386         init_slots();
 387         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 388         if (r < 0)
 389                 return r;
 390         kvm_create_irqchip(kvm);
 391         r = kvm_create_vcpu(kvm, 0);
 392         if (r < 0)
 393                 return r;
 394
 395         return 0;
 396 }
 397
 398
 399 #ifdef KVM_CAP_USER_MEMORY
 400
 401 void *kvm_create_userspace_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 402                         unsigned long len, int log, int writable)
 403 {
 404         int r;
 405         int prot = PROT_READ;
 406         void *ptr;
 407         struct kvm_userspace_memory_region memory = {
 408                 .memory_size = len,
 409                 .guest_phys_addr = phys_start,
 410                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 411         };
 412
 413         if (writable)
 414                 prot |= PROT_WRITE;
 415
 416         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 417         if (ptr == MAP_FAILED) {
 418                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 419                 return 0;
 420         }
 421
 422         memset(ptr, 0, len);
 423
 424         memory.userspace_addr = (unsigned long)ptr;
 425         memory.slot = get_free_slot(kvm);
 426         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 427         if (r == -1) {
 428                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 429                 return 0;
 430         }
 431         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 432                       1, memory.userspace_addr, memory.flags);
 433
 434         return ptr;
 435 }
 436
 437 #endif
 438
 439 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 440                           unsigned long len, int log, int writable)
 441 {
 442 #ifdef KVM_CAP_USER_MEMORY
 443         int r;
 444
 445         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 446         if (r > 0)
 447                 return kvm_create_userspace_phys_mem(kvm, phys_start, len,
 448                                                                 log, writable);
 449         else
 450 #endif
 451                 return kvm_create_kernel_phys_mem(kvm, phys_start, len,
 452                                                                 log, writable);
 453 }
 454
 455 int kvm_is_intersecting_mem(kvm_context_t kvm, unsigned long phys_start)
 456 {
 457         return get_intersecting_slot(phys_start) != -1;
 458 }
 459
 460 int kvm_is_allocated_mem(kvm_context_t kvm, unsigned long phys_start,
 461                          unsigned long len)
 462 {
 463         int slot;
 464
 465         slot = get_slot(phys_start);
 466         if (slot == -1)
 467                 return 0;
 468         if (slots[slot].len == len)
 469                 return 1;
 470         return 0;
 471 }
 472
 473 int kvm_create_mem_hole(kvm_context_t kvm, unsigned long phys_start,
 474                         unsigned long len)
 475 {
 476 #ifdef KVM_CAP_USER_MEMORY
 477         int slot;
 478         int r;
 479         struct kvm_userspace_memory_region rmslot;
 480         struct kvm_userspace_memory_region newslot1;
 481         struct kvm_userspace_memory_region newslot2;
 482
 483         len = (len + PAGE_SIZE - 1) & PAGE_MASK;
 484
 485         slot = get_intersecting_slot(phys_start);
 486         /* no need to create hole, as there is already hole */
 487         if (slot == -1)
 488                 return 0;
 489
 490         memset(&rmslot, 0, sizeof(struct kvm_userspace_memory_region));
 491         memset(&newslot1, 0, sizeof(struct kvm_userspace_memory_region));
 492         memset(&newslot2, 0, sizeof(struct kvm_userspace_memory_region));
 493
 494         rmslot.guest_phys_addr = slots[slot].phys_addr;
 495         rmslot.slot = slot;
 496
 497         newslot1.guest_phys_addr = slots[slot].phys_addr;
 498         newslot1.memory_size = phys_start - slots[slot].phys_addr;
 499         newslot1.slot = slot;
 500         newslot1.userspace_addr = slots[slot].userspace_addr;
 501         newslot1.flags = slots[slot].flags;
 502
 503         newslot2.guest_phys_addr = newslot1.guest_phys_addr +
 504                                    newslot1.memory_size + len;
 505         newslot2.memory_size = slots[slot].phys_addr +
 506                                slots[slot].len - newslot2.guest_phys_addr;
 507         newslot2.userspace_addr = newslot1.userspace_addr +
 508                                   newslot1.memory_size;
 509         newslot2.slot = get_free_slot(kvm);
 510         newslot2.flags = newslot1.flags;
 511
 512         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &rmslot);
 513         if (r == -1) {
 514                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 515                 return -1;
 516         }
 517         free_slot(slot);
 518
 519         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot1);
 520         if (r == -1) {
 521                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 522                 return -1;
 523         }
 524         register_slot(newslot1.slot, newslot1.guest_phys_addr,
 525                       newslot1.memory_size, 1, newslot1.userspace_addr,
 526                       newslot1.flags);
 527
 528         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot2);
 529         if (r == -1) {
 530                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 531                 return -1;
 532         }
 533         register_slot(newslot2.slot, newslot2.guest_phys_addr,
 534                       newslot2.memory_size, 1, newslot2.userspace_addr,
 535                       newslot2.flags);
 536 #endif
 537         return 0;
 538 }
 539
 540 int kvm_register_userspace_phys_mem(kvm_context_t kvm,
 541                         unsigned long phys_start, void *userspace_addr,
 542                         unsigned long len, int log)
 543 {
 544
 545 #ifdef KVM_CAP_USER_MEMORY
 546         struct kvm_userspace_memory_region memory = {
 547                 .memory_size = len,
 548                 .guest_phys_addr = phys_start,
 549                 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
 550                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 551         };
 552         int r;
 553
 554         if (!kvm->physical_memory)
 555                 kvm->physical_memory = userspace_addr - phys_start;
 556
 557         memory.slot = get_free_slot(kvm);
 558         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 559         if (r == -1) {
 560                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 561                 return -1;
 562         }
 563         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 564                       1, memory.userspace_addr, memory.flags);
 565         return 0;
 566 #else
 567         return -ENOSYS;
 568 #endif
 569 }
 570
 571
 572 /* destroy/free a whole slot.
 573  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 574  */
 575 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 576                           unsigned long len)
 577 {
 578         int slot;
 579
 580         slot = get_slot(phys_start);
 581
 582         if (slot >= KVM_MAX_NUM_MEM_REGIONS) {
 583                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 584                         __FUNCTION__, slot);
 585                 return;
 586         }
 587         if (phys_start != slots[slot].phys_addr) {
 588                 fprintf(stderr,
 589                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 590                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 591                 phys_start = slots[slot].phys_addr;
 592         }
 593         kvm_create_phys_mem(kvm, phys_start, 0, 0, 0);
 594 }
 595
 596 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 597 {
 598         int r;
 599         struct kvm_dirty_log log = {
 600                 .slot = slot,
 601         };
 602
 603         log.dirty_bitmap = buf;
 604
 605         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 606         if (r == -1)
 607                 return -errno;
 608         return 0;
 609 }
 610
 611 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 612 {
 613         int slot;
 614
 615         slot = get_slot(phys_addr);
 616         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 617 }
 618
 619 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 620 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 621
 622 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 623                               unsigned long len, void *buf, void *opaque,
 624                               int (*cb)(unsigned long start, unsigned long len,
 625                                         void*bitmap, void *opaque))
 626 {
 627         int i;
 628         int r;
 629         unsigned long end_addr = phys_addr + len;
 630
 631         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 632                 if ((slots[i].len && slots[i].phys_addr >= phys_addr) &&
 633                     (slots[i].phys_addr + slots[i].len  <= end_addr)) {
 634                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 635                         if (r)
 636                                 return r;
 637                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 638                         if (r)
 639                                 return r;
 640                 }
 641         }
 642         return 0;
 643 }
 644
 645 #ifdef KVM_CAP_IRQCHIP
 646
 647 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level)
 648 {
 649         struct kvm_irq_level event;
 650         int r;
 651
 652         if (!kvm->irqchip_in_kernel)
 653                 return 0;
 654         event.level = level;
 655         event.irq = irq;
 656         r = ioctl(kvm->vm_fd, KVM_IRQ_LINE, &event);
 657         if (r == -1)
 658                 perror("kvm_set_irq_level");
 659         return 1;
 660 }
 661
 662 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 663 {
 664         int r;
 665
 666         if (!kvm->irqchip_in_kernel)
 667                 return 0;
 668         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 669         if (r == -1) {
 670                 r = -errno;
 671                 perror("kvm_get_irqchip\n");
 672         }
 673         return r;
 674 }
 675
 676 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 677 {
 678         int r;
 679
 680         if (!kvm->irqchip_in_kernel)
 681                 return 0;
 682         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 683         if (r == -1) {
 684                 r = -errno;
 685                 perror("kvm_set_irqchip\n");
 686         }
 687         return r;
 688 }
 689
 690 #endif
 691
 692 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 693 {
 694         uint16_t addr = run->io.port;
 695         int r;
 696         int i;
 697         void *p = (void *)run + run->io.data_offset;
 698
 699         for (i = 0; i < run->io.count; ++i) {
 700                 switch (run->io.direction) {
 701                 case KVM_EXIT_IO_IN:
 702                         switch (run->io.size) {
 703                         case 1:
 704                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 705                                 break;
 706                         case 2:
 707                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 708                                 break;
 709                         case 4:
 710                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 711                                 break;
 712                         default:
 713                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 714                                 return -EMSGSIZE;
 715                         }
 716                         break;
 717                 case KVM_EXIT_IO_OUT:
 718                         switch (run->io.size) {
 719                         case 1:
 720                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 721                                                      *(uint8_t *)p);
 722                                 break;
 723                         case 2:
 724                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 725                                                      *(uint16_t *)p);
 726                                 break;
 727                         case 4:
 728                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 729                                                      *(uint32_t *)p);
 730                                 break;
 731                         default:
 732                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 733                                 return -EMSGSIZE;
 734                         }
 735                         break;
 736                 default:
 737                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 738                         return -EPROTO;
 739                 }
 740
 741                 p += run->io.size;
 742         }
 743
 744         return 0;
 745 }
 746
 747 int handle_debug(kvm_context_t kvm, int vcpu)
 748 {
 749         return kvm->callbacks->debug(kvm->opaque, vcpu);
 750 }
 751
 752 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 753 {
 754     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 755 }
 756
 757 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 758 {
 759     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 760 }
 761
 762 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 763 {
 764     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 765 }
 766
 767 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 768 {
 769     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 770 }
 771
 772 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 773 {
 774     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 775 }
 776
 777 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 778 {
 779     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 780 }
 781
 782 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 783 {
 784         unsigned long addr = kvm_run->mmio.phys_addr;
 785         void *data = kvm_run->mmio.data;
 786
 787         /* hack: Red Hat 7.1 generates these weird accesses. */
 788         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 789             return 0;
 790
 791         if (kvm_run->mmio.is_write)
 792                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 793                                         kvm_run->mmio.len);
 794         else
 795                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 796                                         kvm_run->mmio.len);
 797 }
 798
 799 int handle_io_window(kvm_context_t kvm)
 800 {
 801         return kvm->callbacks->io_window(kvm->opaque);
 802 }
 803
 804 int handle_halt(kvm_context_t kvm, int vcpu)
 805 {
 806         return kvm->callbacks->halt(kvm->opaque, vcpu);
 807 }
 808
 809 int handle_shutdown(kvm_context_t kvm, int vcpu)
 810 {
 811         return kvm->callbacks->shutdown(kvm->opaque, vcpu);
 812 }
 813
 814 int try_push_interrupts(kvm_context_t kvm)
 815 {
 816         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 817 }
 818
 819 void post_kvm_run(kvm_context_t kvm, int vcpu)
 820 {
 821         kvm->callbacks->post_kvm_run(kvm->opaque, vcpu);
 822 }
 823
 824 int pre_kvm_run(kvm_context_t kvm, int vcpu)
 825 {
 826         return kvm->callbacks->pre_kvm_run(kvm->opaque, vcpu);
 827 }
 828
 829 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 830 {
 831         struct kvm_run *run = kvm->run[vcpu];
 832
 833         return run->if_flag;
 834 }
 835
 836 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 837 {
 838         struct kvm_run *run = kvm->run[vcpu];
 839
 840         return run->ready_for_interrupt_injection;
 841 }
 842
 843 int kvm_run(kvm_context_t kvm, int vcpu)
 844 {
 845         int r;
 846         int fd = kvm->vcpu_fd[vcpu];
 847         struct kvm_run *run = kvm->run[vcpu];
 848
 849 again:
 850         if (!kvm->irqchip_in_kernel)
 851                 run->request_interrupt_window = try_push_interrupts(kvm);
 852         r = pre_kvm_run(kvm, vcpu);
 853         if (r)
 854             return r;
 855         r = ioctl(fd, KVM_RUN, 0);
 856
 857         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 858                 r = -errno;
 859                 post_kvm_run(kvm, vcpu);
 860                 printf("kvm_run: %s\n", strerror(-r));
 861                 return r;
 862         }
 863
 864         post_kvm_run(kvm, vcpu);
 865
 866         if (r == -1) {
 867                 r = handle_io_window(kvm);
 868                 goto more;
 869         }
 870         if (1) {
 871                 switch (run->exit_reason) {
 872                 case KVM_EXIT_UNKNOWN:
 873                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 874                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 875                         kvm_show_regs(kvm, vcpu);
 876                         abort();
 877                         break;
 878                 case KVM_EXIT_FAIL_ENTRY:
 879                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 880                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 881                         return -ENOEXEC;
 882                         break;
 883                 case KVM_EXIT_EXCEPTION:
 884                         fprintf(stderr, "exception %d (%x)\n",
 885                                run->ex.exception,
 886                                run->ex.error_code);
 887                         kvm_show_regs(kvm, vcpu);
 888                         kvm_show_code(kvm, vcpu);
 889                         abort();
 890                         break;
 891                 case KVM_EXIT_IO:
 892                         r = handle_io(kvm, run, vcpu);
 893                         break;
 894                 case KVM_EXIT_DEBUG:
 895                         r = handle_debug(kvm, vcpu);
 896                         break;
 897                 case KVM_EXIT_MMIO:
 898                         r = handle_mmio(kvm, run);
 899                         break;
 900                 case KVM_EXIT_HLT:
 901                         r = handle_halt(kvm, vcpu);
 902                         break;
 903                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 904                         break;
 905                 case KVM_EXIT_SHUTDOWN:
 906                         r = handle_shutdown(kvm, vcpu);
 907                         break;
 908                 default:
 909                         if (kvm_arch_run(run, kvm, vcpu)) {
 910                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 911                                                         run->exit_reason);
 912                                 kvm_show_regs(kvm, vcpu);
 913                                 abort();
 914                         }
 915                         break;
 916                 }
 917         }
 918 more:
 919         if (!r)
 920                 goto again;
 921         return r;
 922 }
 923
 924 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 925 {
 926         struct kvm_interrupt intr;
 927
 928         intr.irq = irq;
 929         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 930 }
 931
 932 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
 933 {
 934         return ioctl(kvm->vcpu_fd[vcpu], KVM_DEBUG_GUEST, dbg);
 935 }
 936
 937 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
 938 {
 939         struct kvm_signal_mask *sigmask;
 940         int r;
 941
 942         if (!sigset) {
 943                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
 944                 if (r == -1)
 945                         r = -errno;
 946                 return r;
 947         }
 948         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
 949         if (!sigmask)
 950                 return -ENOMEM;
 951
 952         sigmask->len = 8;
 953         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
 954         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
 955         if (r == -1)
 956                 r = -errno;
 957         free(sigmask);
 958         return r;
 959 }
 960
 961 int kvm_irqchip_in_kernel(kvm_context_t kvm)
 962 {
 963     return kvm->irqchip_in_kernel;
 964 }
 965
 966 int kvm_pit_in_kernel(kvm_context_t kvm)
 967 {
 968         return kvm->pit_in_kernel;
 969 }