libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include <inttypes.h>
  38 #include "libkvm.h"
  39
  40 #if defined(__x86_64__) || defined(__i386__)
  41 #include "kvm-x86.h"
  42 #endif
  43
  44 #if defined(__ia64__)
  45 #include "kvm-ia64.h"
  46 #endif
  47
  48 #if defined(__powerpc__)
  49 #include "kvm-powerpc.h"
  50 #endif
  51
  52 #if defined(__s390__)
  53 #include "kvm-s390.h"
  54 #endif
  55
  56 //#define DEBUG_MEMREG
  57 #ifdef  DEBUG_MEMREG
  58 #define DPRINTF(fmt, args...) \
  59         do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
  60 #else
  61 #define DPRINTF(fmt, args...) do {} while (0)
  62 #endif
  63
  64
  65 int kvm_abi = EXPECTED_KVM_API_VERSION;
  66 int kvm_page_size;
  67
  68 struct slot_info {
  69         unsigned long phys_addr;
  70         unsigned long len;
  71         unsigned long userspace_addr;
  72         unsigned flags;
  73         int logging_count;
  74 };
  75
  76 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  77
  78 static void init_slots(void)
  79 {
  80         int i;
  81
  82         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  83                 slots[i].len = 0;
  84 }
  85
  86 static int get_free_slot(kvm_context_t kvm)
  87 {
  88         int i;
  89         int tss_ext;
  90
  91 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
  92         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  93 #else
  94         tss_ext = 0;
  95 #endif
  96
  97         /*
  98          * on older kernels where the set tss ioctl is not supprted we must save
  99          * slot 0 to hold the extended memory, as the vmx will use the last 3
 100          * pages of this slot.
 101          */
 102         if (tss_ext > 0)
 103                 i = 0;
 104         else
 105                 i = 1;
 106
 107         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
 108                 if (!slots[i].len)
 109                         return i;
 110         return -1;
 111 }
 112
 113 static void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 114                    unsigned long userspace_addr, unsigned flags)
 115 {
 116         slots[slot].phys_addr = phys_addr;
 117         slots[slot].len = len;
 118         slots[slot].userspace_addr = userspace_addr;
 119         slots[slot].flags = flags;
 120 }
 121
 122 static void free_slot(int slot)
 123 {
 124         slots[slot].len = 0;
 125         slots[slot].logging_count = 0;
 126 }
 127
 128 static int get_slot(unsigned long phys_addr)
 129 {
 130         int i;
 131
 132         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 133                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 134                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 135                         return i;
 136         }
 137         return -1;
 138 }
 139
 140 /* Returns -1 if this slot is not totally contained on any other,
 141  * and the number of the slot otherwise */
 142 static int get_container_slot(uint64_t phys_addr, unsigned long size)
 143 {
 144         int i;
 145
 146         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 147                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 148                     (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
 149                         return i;
 150         return -1;
 151 }
 152
 153 int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr, unsigned long size)
 154 {
 155         int slot = get_container_slot(phys_addr, size);
 156         if (slot == -1)
 157                 return 0;
 158         return 1;
 159 }
 160
 161 /*
 162  * dirty pages logging control
 163  */
 164 static int kvm_dirty_pages_log_change(kvm_context_t kvm,
 165                                       unsigned long phys_addr,
 166                                       unsigned flags,
 167                                       unsigned mask)
 168 {
 169         int r = -1;
 170         int slot = get_slot(phys_addr);
 171
 172         if (slot == -1) {
 173                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 174                 return 1;
 175         }
 176
 177         flags = (slots[slot].flags & ~mask) | flags;
 178         if (flags == slots[slot].flags)
 179                 return 0;
 180         slots[slot].flags = flags;
 181
 182         {
 183                 struct kvm_userspace_memory_region mem = {
 184                         .slot = slot,
 185                         .memory_size = slots[slot].len,
 186                         .guest_phys_addr = slots[slot].phys_addr,
 187                         .userspace_addr = slots[slot].userspace_addr,
 188                         .flags = slots[slot].flags,
 189                 };
 190
 191
 192                 DPRINTF("slot %d start %llx len %llx flags %x\n",
 193                         mem.slot,
 194                         mem.guest_phys_addr,
 195                         mem.memory_size,
 196                         mem.flags);
 197                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 198                 if (r == -1)
 199                         fprintf(stderr, "%s: %m\n", __FUNCTION__);
 200         }
 201         return r;
 202 }
 203
 204 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
 205                                           int (*change)(kvm_context_t kvm,
 206                                                         uint64_t start,
 207                                                         uint64_t len))
 208 {
 209         int i, r;
 210
 211         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 212                 if (slots[i].len)
 213                         r = change(kvm, slots[i].phys_addr, slots[i].len);
 214         }
 215         return r;
 216 }
 217
 218 int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm,
 219                                     uint64_t phys_addr,
 220                                     uint64_t len)
 221 {
 222         int slot = get_slot(phys_addr);
 223
 224         DPRINTF("start %"PRIx64" len %"PRIx64"\n", phys_addr, len);
 225         if (slot == -1) {
 226                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 227                 return -EINVAL;
 228         }
 229
 230         if (slots[slot].logging_count++)
 231                 return 0;
 232
 233         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 234                                           KVM_MEM_LOG_DIRTY_PAGES,
 235                                           KVM_MEM_LOG_DIRTY_PAGES);
 236 }
 237
 238 int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm,
 239                                      uint64_t phys_addr,
 240                                      uint64_t len)
 241 {
 242         int slot = get_slot(phys_addr);
 243
 244         if (slot == -1) {
 245                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 246                 return -EINVAL;
 247         }
 248
 249         if (--slots[slot].logging_count)
 250                 return 0;
 251
 252         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 253                                           0,
 254                                           KVM_MEM_LOG_DIRTY_PAGES);
 255 }
 256
 257 /**
 258  * Enable dirty page logging for all memory regions
 259  */
 260 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 261 {
 262         if (kvm->dirty_pages_log_all)
 263                 return 0;
 264         kvm->dirty_pages_log_all = 1;
 265         return kvm_dirty_pages_log_change_all(kvm,
 266                                               kvm_dirty_pages_log_enable_slot);
 267 }
 268
 269 /**
 270  * Enable dirty page logging only for memory regions that were created with
 271  *     dirty logging enabled (disable for all other memory regions).
 272  */
 273 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 274 {
 275         if (!kvm->dirty_pages_log_all)
 276                 return 0;
 277         kvm->dirty_pages_log_all = 0;
 278         return kvm_dirty_pages_log_change_all(kvm,
 279                                               kvm_dirty_pages_log_disable_slot);
 280 }
 281
 282
 283 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 284                        void *opaque)
 285 {
 286         int fd;
 287         kvm_context_t kvm;
 288         int r;
 289
 290         fd = open("/dev/kvm", O_RDWR);
 291         if (fd == -1) {
 292                 perror("open /dev/kvm");
 293                 return NULL;
 294         }
 295         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 296         if (r == -1) {
 297             fprintf(stderr, "kvm kernel version too old: "
 298                     "KVM_GET_API_VERSION ioctl not supported\n");
 299             goto out_close;
 300         }
 301         if (r < EXPECTED_KVM_API_VERSION) {
 302                 fprintf(stderr, "kvm kernel version too old: "
 303                         "We expect API version %d or newer, but got "
 304                         "version %d\n",
 305                         EXPECTED_KVM_API_VERSION, r);
 306             goto out_close;
 307         }
 308         if (r > EXPECTED_KVM_API_VERSION) {
 309             fprintf(stderr, "kvm userspace version too old\n");
 310             goto out_close;
 311         }
 312         kvm_abi = r;
 313         kvm_page_size = getpagesize();
 314         kvm = malloc(sizeof(*kvm));
 315         if (kvm == NULL)
 316                 goto out_close;
 317         memset(kvm, 0, sizeof(*kvm));
 318         kvm->fd = fd;
 319         kvm->vm_fd = -1;
 320         kvm->callbacks = callbacks;
 321         kvm->opaque = opaque;
 322         kvm->dirty_pages_log_all = 0;
 323         kvm->no_irqchip_creation = 0;
 324         kvm->no_pit_creation = 0;
 325
 326         return kvm;
 327  out_close:
 328         close(fd);
 329         return NULL;
 330 }
 331
 332 void kvm_finalize(kvm_context_t kvm)
 333 {
 334         if (kvm->vcpu_fd[0] != -1)
 335                 close(kvm->vcpu_fd[0]);
 336         if (kvm->vm_fd != -1)
 337                 close(kvm->vm_fd);
 338         close(kvm->fd);
 339         free(kvm);
 340 }
 341
 342 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 343 {
 344         kvm->no_irqchip_creation = 1;
 345 }
 346
 347 void kvm_disable_pit_creation(kvm_context_t kvm)
 348 {
 349         kvm->no_pit_creation = 1;
 350 }
 351
 352 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 353 {
 354         long mmap_size;
 355         int r;
 356
 357         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 358         if (r == -1) {
 359                 r = -errno;
 360                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 361                 return r;
 362         }
 363         kvm->vcpu_fd[slot] = r;
 364         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 365         if (mmap_size == -1) {
 366                 r = -errno;
 367                 fprintf(stderr, "get vcpu mmap size: %m\n");
 368                 return r;
 369         }
 370         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 371                               kvm->vcpu_fd[slot], 0);
 372         if (kvm->run[slot] == MAP_FAILED) {
 373                 r = -errno;
 374                 fprintf(stderr, "mmap vcpu area: %m\n");
 375                 return r;
 376         }
 377         return 0;
 378 }
 379
 380 int kvm_create_vm(kvm_context_t kvm)
 381 {
 382         int fd = kvm->fd;
 383
 384 #ifdef KVM_CAP_IRQ_ROUTING
 385         kvm->irq_routes = malloc(sizeof(*kvm->irq_routes));
 386         if (!kvm->irq_routes)
 387                 return -ENOMEM;
 388         memset(kvm->irq_routes, 0, sizeof(*kvm->irq_routes));
 389         kvm->nr_allocated_irq_routes = 0;
 390 #endif
 391
 392         kvm->vcpu_fd[0] = -1;
 393
 394         fd = ioctl(fd, KVM_CREATE_VM, 0);
 395         if (fd == -1) {
 396                 fprintf(stderr, "kvm_create_vm: %m\n");
 397                 return -1;
 398         }
 399         kvm->vm_fd = fd;
 400         return 0;
 401 }
 402
 403 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 404                                        unsigned long phys_mem_bytes,
 405                                        void **vm_mem)
 406 {
 407 #ifdef KVM_CAP_USER_MEMORY
 408         int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 409         if (r > 0)
 410                 return 0;
 411         fprintf(stderr, "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
 412 #else
 413 #error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
 414 #endif
 415         return -1;
 416 }
 417
 418 int kvm_check_extension(kvm_context_t kvm, int ext)
 419 {
 420         int ret;
 421
 422         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 423         if (ret > 0)
 424                 return 1;
 425         return 0;
 426 }
 427
 428 void kvm_create_irqchip(kvm_context_t kvm)
 429 {
 430         int r;
 431
 432         kvm->irqchip_in_kernel = 0;
 433 #ifdef KVM_CAP_IRQCHIP
 434         if (!kvm->no_irqchip_creation) {
 435                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 436                 if (r > 0) {    /* kernel irqchip supported */
 437                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 438                         if (r >= 0) {
 439                                 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE;
 440 #if defined(KVM_CAP_IRQ_INJECT_STATUS) && defined(KVM_IRQ_LINE_STATUS)
 441                                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
 442                           KVM_CAP_IRQ_INJECT_STATUS);
 443                                 if (r > 0)
 444                                         kvm->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
 445 #endif
 446                                 kvm->irqchip_in_kernel = 1;
 447                         }
 448                         else
 449                                 fprintf(stderr, "Create kernel PIC irqchip failed\n");
 450                 }
 451         }
 452 #endif
 453 }
 454
 455 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 456 {
 457         int r;
 458
 459         r = kvm_create_vm(kvm);
 460         if (r < 0)
 461                 return r;
 462         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 463         if (r < 0)
 464                 return r;
 465         init_slots();
 466         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 467         if (r < 0)
 468                 return r;
 469         kvm_create_irqchip(kvm);
 470
 471         return 0;
 472 }
 473
 474
 475 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 476                           unsigned long len, int log, int writable)
 477 {
 478         int r;
 479         int prot = PROT_READ;
 480         void *ptr;
 481         struct kvm_userspace_memory_region memory = {
 482                 .memory_size = len,
 483                 .guest_phys_addr = phys_start,
 484                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 485         };
 486
 487         if (writable)
 488                 prot |= PROT_WRITE;
 489
 490 #if !defined(__s390__)
 491         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 492 #else
 493         ptr = mmap(LIBKVM_S390_ORIGIN, len, prot | PROT_EXEC,
 494                 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 495 #endif
 496         if (ptr == MAP_FAILED) {
 497                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 498                 return 0;
 499         }
 500
 501         memset(ptr, 0, len);
 502
 503         memory.userspace_addr = (unsigned long)ptr;
 504         memory.slot = get_free_slot(kvm);
 505         DPRINTF("slot %d start %llx len %llx flags %x\n",
 506                 memory.slot,
 507                 memory.guest_phys_addr,
 508                 memory.memory_size,
 509                 memory.flags);
 510         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 511         if (r == -1) {
 512                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 513                 return 0;
 514         }
 515         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 516                       memory.userspace_addr, memory.flags);
 517
 518         return ptr;
 519 }
 520
 521 int kvm_register_phys_mem(kvm_context_t kvm,
 522                           unsigned long phys_start, void *userspace_addr,
 523                           unsigned long len, int log)
 524 {
 525
 526         struct kvm_userspace_memory_region memory = {
 527                 .memory_size = len,
 528                 .guest_phys_addr = phys_start,
 529                 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
 530                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 531         };
 532         int r;
 533
 534         memory.slot = get_free_slot(kvm);
 535         DPRINTF("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %lx\n",
 536                 memory.guest_phys_addr, memory.memory_size,
 537                 memory.userspace_addr, memory.slot, memory.flags);
 538         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 539         if (r == -1) {
 540                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 541                 return -1;
 542         }
 543         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 544                       memory.userspace_addr, memory.flags);
 545         return 0;
 546 }
 547
 548
 549 /* destroy/free a whole slot.
 550  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 551  */
 552 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 553                           unsigned long len)
 554 {
 555         int slot;
 556         int r;
 557         struct kvm_userspace_memory_region memory = {
 558                 .memory_size = 0,
 559                 .guest_phys_addr = phys_start,
 560                 .userspace_addr = 0,
 561                 .flags = 0,
 562         };
 563
 564         slot = get_slot(phys_start);
 565
 566         if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
 567                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 568                         __FUNCTION__, slot);
 569                 return;
 570         }
 571         if (phys_start != slots[slot].phys_addr) {
 572                 fprintf(stderr,
 573                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 574                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 575                 phys_start = slots[slot].phys_addr;
 576         }
 577
 578         memory.slot = slot;
 579         DPRINTF("slot %d start %llx len %llx flags %x\n",
 580                 memory.slot,
 581                 memory.guest_phys_addr,
 582                 memory.memory_size,
 583                 memory.flags);
 584         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 585         if (r == -1) {
 586                 fprintf(stderr, "destroy_userspace_phys_mem: %s",
 587                         strerror(errno));
 588                 return;
 589         }
 590
 591         free_slot(memory.slot);
 592 }
 593
 594 void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr, unsigned long size)
 595 {
 596
 597         int slot = get_container_slot(phys_addr, size);
 598
 599         if (slot != -1) {
 600                 DPRINTF("Unregistering memory region %llx (%lx)\n", phys_addr, size);
 601                 kvm_destroy_phys_mem(kvm, phys_addr, size);
 602                 return;
 603         }
 604 }
 605
 606 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 607 {
 608         int r;
 609         struct kvm_dirty_log log = {
 610                 .slot = slot,
 611         };
 612
 613         log.dirty_bitmap = buf;
 614
 615         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 616         if (r == -1)
 617                 return -errno;
 618         return 0;
 619 }
 620
 621 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 622 {
 623         int slot;
 624
 625         slot = get_slot(phys_addr);
 626         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 627 }
 628
 629 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 630 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 631
 632 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 633                               unsigned long len, void *buf, void *opaque,
 634                               int (*cb)(unsigned long start, unsigned long len,
 635                                         void*bitmap, void *opaque))
 636 {
 637         int i;
 638         int r;
 639         unsigned long end_addr = phys_addr + len;
 640
 641         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 642                 if ((slots[i].len && (uint64_t)slots[i].phys_addr >= phys_addr)
 643                     && ((uint64_t)slots[i].phys_addr + slots[i].len  <= end_addr)) {
 644                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 645                         if (r)
 646                                 return r;
 647                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 648                         if (r)
 649                                 return r;
 650                 }
 651         }
 652         return 0;
 653 }
 654
 655 #ifdef KVM_CAP_IRQCHIP
 656
 657 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level, int *status)
 658 {
 659         struct kvm_irq_level event;
 660         int r;
 661
 662         if (!kvm->irqchip_in_kernel)
 663                 return 0;
 664         event.level = level;
 665         event.irq = irq;
 666         r = ioctl(kvm->vm_fd, kvm->irqchip_inject_ioctl, &event);
 667         if (r == -1)
 668                 perror("kvm_set_irq_level");
 669
 670         if (status) {
 671 #ifdef KVM_CAP_IRQ_INJECT_STATUS
 672                 *status = (kvm->irqchip_inject_ioctl == KVM_IRQ_LINE) ?
 673                         1 : event.status;
 674 #else
 675                 *status = 1;
 676 #endif
 677         }
 678
 679         return 1;
 680 }
 681
 682 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 683 {
 684         int r;
 685
 686         if (!kvm->irqchip_in_kernel)
 687                 return 0;
 688         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 689         if (r == -1) {
 690                 r = -errno;
 691                 perror("kvm_get_irqchip\n");
 692         }
 693         return r;
 694 }
 695
 696 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 697 {
 698         int r;
 699
 700         if (!kvm->irqchip_in_kernel)
 701                 return 0;
 702         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 703         if (r == -1) {
 704                 r = -errno;
 705                 perror("kvm_set_irqchip\n");
 706         }
 707         return r;
 708 }
 709
 710 #endif
 711
 712 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 713 {
 714         uint16_t addr = run->io.port;
 715         int r;
 716         int i;
 717         void *p = (void *)run + run->io.data_offset;
 718
 719         for (i = 0; i < run->io.count; ++i) {
 720                 switch (run->io.direction) {
 721                 case KVM_EXIT_IO_IN:
 722                         switch (run->io.size) {
 723                         case 1:
 724                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 725                                 break;
 726                         case 2:
 727                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 728                                 break;
 729                         case 4:
 730                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 731                                 break;
 732                         default:
 733                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 734                                 return -EMSGSIZE;
 735                         }
 736                         break;
 737                 case KVM_EXIT_IO_OUT:
 738                         switch (run->io.size) {
 739                         case 1:
 740                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 741                                                      *(uint8_t *)p);
 742                                 break;
 743                         case 2:
 744                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 745                                                      *(uint16_t *)p);
 746                                 break;
 747                         case 4:
 748                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 749                                                      *(uint32_t *)p);
 750                                 break;
 751                         default:
 752                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 753                                 return -EMSGSIZE;
 754                         }
 755                         break;
 756                 default:
 757                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 758                         return -EPROTO;
 759                 }
 760
 761                 p += run->io.size;
 762         }
 763
 764         return 0;
 765 }
 766
 767 int handle_debug(kvm_context_t kvm, int vcpu, void *env)
 768 {
 769 #ifdef KVM_CAP_SET_GUEST_DEBUG
 770     struct kvm_run *run = kvm->run[vcpu];
 771
 772     return kvm->callbacks->debug(kvm->opaque, env, &run->debug.arch);
 773 #else
 774     return 0;
 775 #endif
 776 }
 777
 778 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 779 {
 780     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 781 }
 782
 783 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 784 {
 785     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 786 }
 787
 788 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 789 {
 790     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 791 }
 792
 793 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 794 {
 795     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 796 }
 797
 798 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 799 {
 800     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 801 }
 802
 803 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 804 {
 805     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 806 }
 807
 808 #ifdef KVM_CAP_MP_STATE
 809 int kvm_get_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 810 {
 811     int r;
 812
 813     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 814     if (r > 0)
 815         return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_MP_STATE, mp_state);
 816     return -ENOSYS;
 817 }
 818
 819 int kvm_set_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 820 {
 821     int r;
 822
 823     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 824     if (r > 0)
 825         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_MP_STATE, mp_state);
 826     return -ENOSYS;
 827 }
 828 #endif
 829
 830 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 831 {
 832         unsigned long addr = kvm_run->mmio.phys_addr;
 833         void *data = kvm_run->mmio.data;
 834
 835         /* hack: Red Hat 7.1 generates these weird accesses. */
 836         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 837             return 0;
 838
 839         if (kvm_run->mmio.is_write)
 840                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 841                                         kvm_run->mmio.len);
 842         else
 843                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 844                                         kvm_run->mmio.len);
 845 }
 846
 847 int handle_io_window(kvm_context_t kvm)
 848 {
 849         return kvm->callbacks->io_window(kvm->opaque);
 850 }
 851
 852 int handle_halt(kvm_context_t kvm, int vcpu)
 853 {
 854         return kvm->callbacks->halt(kvm->opaque, vcpu);
 855 }
 856
 857 int handle_shutdown(kvm_context_t kvm, void *env)
 858 {
 859         return kvm->callbacks->shutdown(kvm->opaque, env);
 860 }
 861
 862 int try_push_interrupts(kvm_context_t kvm)
 863 {
 864         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 865 }
 866
 867 static inline void push_nmi(kvm_context_t kvm)
 868 {
 869 #ifdef KVM_CAP_USER_NMI
 870         kvm->callbacks->push_nmi(kvm->opaque);
 871 #endif /* KVM_CAP_USER_NMI */
 872 }
 873
 874 void post_kvm_run(kvm_context_t kvm, void *env)
 875 {
 876         kvm->callbacks->post_kvm_run(kvm->opaque, env);
 877 }
 878
 879 int pre_kvm_run(kvm_context_t kvm, void *env)
 880 {
 881         return kvm->callbacks->pre_kvm_run(kvm->opaque, env);
 882 }
 883
 884 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 885 {
 886         struct kvm_run *run = kvm->run[vcpu];
 887
 888         return run->if_flag;
 889 }
 890
 891 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 892 {
 893         struct kvm_run *run = kvm->run[vcpu];
 894
 895         return run->ready_for_interrupt_injection;
 896 }
 897
 898 int kvm_run(kvm_context_t kvm, int vcpu, void *env)
 899 {
 900         int r;
 901         int fd = kvm->vcpu_fd[vcpu];
 902         struct kvm_run *run = kvm->run[vcpu];
 903
 904 again:
 905         push_nmi(kvm);
 906 #if !defined(__s390__)
 907         if (!kvm->irqchip_in_kernel)
 908                 run->request_interrupt_window = try_push_interrupts(kvm);
 909 #endif
 910         r = pre_kvm_run(kvm, env);
 911         if (r)
 912             return r;
 913         r = ioctl(fd, KVM_RUN, 0);
 914
 915         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 916                 r = -errno;
 917                 post_kvm_run(kvm, env);
 918                 fprintf(stderr, "kvm_run: %s\n", strerror(-r));
 919                 return r;
 920         }
 921
 922         post_kvm_run(kvm, env);
 923
 924 #if defined(KVM_CAP_COALESCED_MMIO)
 925         if (kvm->coalesced_mmio) {
 926                 struct kvm_coalesced_mmio_ring *ring = (void *)run +
 927                                                 kvm->coalesced_mmio * PAGE_SIZE;
 928                 while (ring->first != ring->last) {
 929                         kvm->callbacks->mmio_write(kvm->opaque,
 930                                  ring->coalesced_mmio[ring->first].phys_addr,
 931                                 &ring->coalesced_mmio[ring->first].data[0],
 932                                  ring->coalesced_mmio[ring->first].len);
 933                         smp_wmb();
 934                         ring->first = (ring->first + 1) %
 935                                                         KVM_COALESCED_MMIO_MAX;
 936                 }
 937         }
 938 #endif
 939
 940 #if !defined(__s390__)
 941         if (r == -1) {
 942                 r = handle_io_window(kvm);
 943                 goto more;
 944         }
 945 #endif
 946         if (1) {
 947                 switch (run->exit_reason) {
 948                 case KVM_EXIT_UNKNOWN:
 949                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 950                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 951                         kvm_show_regs(kvm, vcpu);
 952                         abort();
 953                         break;
 954                 case KVM_EXIT_FAIL_ENTRY:
 955                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 956                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 957                         kvm_show_regs(kvm, vcpu);
 958                         return -ENOEXEC;
 959                         break;
 960                 case KVM_EXIT_EXCEPTION:
 961                         fprintf(stderr, "exception %d (%x)\n",
 962                                run->ex.exception,
 963                                run->ex.error_code);
 964                         kvm_show_regs(kvm, vcpu);
 965                         kvm_show_code(kvm, vcpu);
 966                         abort();
 967                         break;
 968                 case KVM_EXIT_IO:
 969                         r = handle_io(kvm, run, vcpu);
 970                         break;
 971                 case KVM_EXIT_DEBUG:
 972                         r = handle_debug(kvm, vcpu, env);
 973                         break;
 974                 case KVM_EXIT_MMIO:
 975                         r = handle_mmio(kvm, run);
 976                         break;
 977                 case KVM_EXIT_HLT:
 978                         r = handle_halt(kvm, vcpu);
 979                         break;
 980                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 981                         break;
 982                 case KVM_EXIT_SHUTDOWN:
 983                         r = handle_shutdown(kvm, env);
 984                         break;
 985 #if defined(__s390__)
 986                 case KVM_EXIT_S390_SIEIC:
 987                         r = kvm->callbacks->s390_handle_intercept(kvm, vcpu,
 988                                 run);
 989                         break;
 990                 case KVM_EXIT_S390_RESET:
 991                         r = kvm->callbacks->s390_handle_reset(kvm, vcpu, run);
 992                         break;
 993 #endif
 994                 default:
 995                         if (kvm_arch_run(run, kvm, vcpu)) {
 996                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 997                                                         run->exit_reason);
 998                                 kvm_show_regs(kvm, vcpu);
 999                                 abort();
1000                         }
1001                         break;
1002                 }
1003         }
1004 more:
1005         if (!r)
1006                 goto again;
1007         return r;
1008 }
1009
1010 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
1011 {
1012         struct kvm_interrupt intr;
1013
1014         intr.irq = irq;
1015         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
1016 }
1017
1018 #ifdef KVM_CAP_SET_GUEST_DEBUG
1019 int kvm_set_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_guest_debug *dbg)
1020 {
1021         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_GUEST_DEBUG, dbg);
1022 }
1023 #endif
1024
1025 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
1026 {
1027         struct kvm_signal_mask *sigmask;
1028         int r;
1029
1030         if (!sigset) {
1031                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
1032                 if (r == -1)
1033                         r = -errno;
1034                 return r;
1035         }
1036         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
1037         if (!sigmask)
1038                 return -ENOMEM;
1039
1040         sigmask->len = 8;
1041         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1042         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
1043         if (r == -1)
1044                 r = -errno;
1045         free(sigmask);
1046         return r;
1047 }
1048
1049 int kvm_irqchip_in_kernel(kvm_context_t kvm)
1050 {
1051         return kvm->irqchip_in_kernel;
1052 }
1053
1054 int kvm_pit_in_kernel(kvm_context_t kvm)
1055 {
1056         return kvm->pit_in_kernel;
1057 }
1058
1059 int kvm_has_sync_mmu(kvm_context_t kvm)
1060 {
1061         int r = 0;
1062 #ifdef KVM_CAP_SYNC_MMU
1063         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
1064 #endif
1065         return r;
1066 }
1067
1068 int kvm_inject_nmi(kvm_context_t kvm, int vcpu)
1069 {
1070 #ifdef KVM_CAP_USER_NMI
1071         return ioctl(kvm->vcpu_fd[vcpu], KVM_NMI);
1072 #else
1073         return -ENOSYS;
1074 #endif
1075 }
1076
1077 int kvm_init_coalesced_mmio(kvm_context_t kvm)
1078 {
1079         int r = 0;
1080         kvm->coalesced_mmio = 0;
1081 #ifdef KVM_CAP_COALESCED_MMIO
1082         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
1083         if (r > 0) {
1084                 kvm->coalesced_mmio = r;
1085                 return 0;
1086         }
1087 #endif
1088         return r;
1089 }
1090
1091 int kvm_register_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1092 {
1093 #ifdef KVM_CAP_COALESCED_MMIO
1094         struct kvm_coalesced_mmio_zone zone;
1095         int r;
1096
1097         if (kvm->coalesced_mmio) {
1098
1099                 zone.addr = addr;
1100                 zone.size = size;
1101
1102                 r = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
1103                 if (r == -1) {
1104                         perror("kvm_register_coalesced_mmio_zone");
1105                         return -errno;
1106                 }
1107                 return 0;
1108         }
1109 #endif
1110         return -ENOSYS;
1111 }
1112
1113 int kvm_unregister_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1114 {
1115 #ifdef KVM_CAP_COALESCED_MMIO
1116         struct kvm_coalesced_mmio_zone zone;
1117         int r;
1118
1119         if (kvm->coalesced_mmio) {
1120
1121                 zone.addr = addr;
1122                 zone.size = size;
1123
1124                 r = ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1125                 if (r == -1) {
1126                         perror("kvm_unregister_coalesced_mmio_zone");
1127                         return -errno;
1128                 }
1129                 DPRINTF("Unregistered coalesced mmio region for %llx (%lx)\n", addr, size);
1130                 return 0;
1131         }
1132 #endif
1133         return -ENOSYS;
1134 }
1135
1136 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1137 int kvm_assign_pci_device(kvm_context_t kvm,
1138                           struct kvm_assigned_pci_dev *assigned_dev)
1139 {
1140         int ret;
1141
1142         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
1143         if (ret < 0)
1144                 return -errno;
1145
1146         return ret;
1147 }
1148
1149 int kvm_assign_irq(kvm_context_t kvm,
1150                    struct kvm_assigned_irq *assigned_irq)
1151 {
1152         int ret;
1153
1154         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_IRQ, assigned_irq);
1155         if (ret < 0)
1156                 return -errno;
1157
1158         return ret;
1159 }
1160 #endif
1161
1162 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
1163 int kvm_deassign_pci_device(kvm_context_t kvm,
1164                             struct kvm_assigned_pci_dev *assigned_dev)
1165 {
1166         int ret;
1167
1168         ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_PCI_DEVICE, assigned_dev);
1169         if (ret < 0)
1170                 return -errno;
1171
1172         return ret;
1173 }
1174 #endif
1175
1176 int kvm_destroy_memory_region_works(kvm_context_t kvm)
1177 {
1178         int ret = 0;
1179
1180 #ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
1181         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
1182                     KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
1183         if (ret <= 0)
1184                 ret = 0;
1185 #endif
1186         return ret;
1187 }
1188
1189 int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
1190 {
1191 #ifdef KVM_CAP_REINJECT_CONTROL
1192         int r;
1193         struct kvm_reinject_control control;
1194
1195         control.pit_reinject = pit_reinject;
1196
1197         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_REINJECT_CONTROL);
1198         if (r > 0) {
1199                 r = ioctl(kvm->vm_fd, KVM_REINJECT_CONTROL, &control);
1200                 if (r == -1)
1201                         return -errno;
1202                 return r;
1203         }
1204 #endif
1205         return -ENOSYS;
1206 }
1207
1208 int kvm_has_gsi_routing(kvm_context_t kvm)
1209 {
1210     int r = 0;
1211
1212 #ifdef KVM_CAP_IRQ_ROUTING
1213     r = kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1214 #endif
1215     return r;
1216 }
1217
1218 int kvm_get_gsi_count(kvm_context_t kvm)
1219 {
1220 #ifdef KVM_CAP_IRQ_ROUTING
1221         return kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1222 #else
1223         return -EINVAL;
1224 #endif
1225 }
1226
1227 int kvm_clear_gsi_routes(kvm_context_t kvm)
1228 {
1229 #ifdef KVM_CAP_IRQ_ROUTING
1230         kvm->irq_routes->nr = 0;
1231         return 0;
1232 #else
1233         return -EINVAL;
1234 #endif
1235 }
1236
1237 int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1238 {
1239 #ifdef KVM_CAP_IRQ_ROUTING
1240         struct kvm_irq_routing *z;
1241         struct kvm_irq_routing_entry *e;
1242         int n, size;
1243
1244         if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
1245                 n = kvm->nr_allocated_irq_routes * 2;
1246                 if (n < 64)
1247                         n = 64;
1248                 size = sizeof(struct kvm_irq_routing);
1249                 size += n * sizeof(*e);
1250                 z = realloc(kvm->irq_routes, size);
1251                 if (!z)
1252                         return -ENOMEM;
1253                 kvm->nr_allocated_irq_routes = n;
1254                 kvm->irq_routes = z;
1255         }
1256         n = kvm->irq_routes->nr++;
1257         e = &kvm->irq_routes->entries[n];
1258         memset(e, 0, sizeof(*e));
1259         e->gsi = gsi;
1260         e->type = KVM_IRQ_ROUTING_IRQCHIP;
1261         e->flags = 0;
1262         e->u.irqchip.irqchip = irqchip;
1263         e->u.irqchip.pin = pin;
1264         return 0;
1265 #else
1266         return -ENOSYS;
1267 #endif
1268 }
1269
1270 int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1271 {
1272 #ifdef KVM_CAP_IRQ_ROUTING
1273         struct kvm_irq_routing_entry *e, *p;
1274         int i;
1275
1276         for (i = 0; i < kvm->irq_routes->nr; ++i) {
1277                 e = &kvm->irq_routes->entries[i];
1278                 if (e->type == KVM_IRQ_ROUTING_IRQCHIP
1279                     && e->gsi == gsi
1280                     && e->u.irqchip.irqchip == irqchip
1281                     && e->u.irqchip.pin == pin) {
1282                         p = &kvm->irq_routes->entries[--kvm->irq_routes->nr];
1283                         *e = *p;
1284                         return 0;
1285                 }
1286         }
1287         return -ESRCH;
1288 #else
1289         return -ENOSYS;
1290 #endif
1291 }
1292
1293 int kvm_commit_irq_routes(kvm_context_t kvm)
1294 {
1295 #ifdef KVM_CAP_IRQ_ROUTING
1296         int r;
1297
1298         kvm->irq_routes->flags = 0;
1299         r = ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, kvm->irq_routes);
1300         if (r == -1)
1301                 r = -errno;
1302         return r;
1303 #else
1304         return -ENOSYS;
1305 #endif
1306 }