libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include "libkvm.h"
  38
  39 #if defined(__x86_64__) || defined(__i386__)
  40 #include "kvm-x86.h"
  41 #endif
  42
  43 #if defined(__ia64__)
  44 #include "kvm-ia64.h"
  45 #endif
  46
  47 #if defined(__powerpc__)
  48 #include "kvm-powerpc.h"
  49 #endif
  50
  51 int kvm_abi = EXPECTED_KVM_API_VERSION;
  52 int kvm_page_size;
  53
  54 struct slot_info {
  55         unsigned long phys_addr;
  56         unsigned long len;
  57         int user_alloc;
  58         unsigned long userspace_addr;
  59         unsigned flags;
  60 };
  61
  62 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  63
  64 void init_slots(void)
  65 {
  66         int i;
  67
  68         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  69                 slots[i].len = 0;
  70 }
  71
  72 int get_free_slot(kvm_context_t kvm)
  73 {
  74         int i;
  75         int tss_ext;
  76
  77 #ifdef KVM_CAP_SET_TSS_ADDR
  78         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  79 #else
  80         tss_ext = 0;
  81 #endif
  82
  83         /*
  84          * on older kernels where the set tss ioctl is not supprted we must save
  85          * slot 0 to hold the extended memory, as the vmx will use the last 3
  86          * pages of this slot.
  87          */
  88         if (tss_ext > 0)
  89                 i = 0;
  90         else
  91                 i = 1;
  92
  93         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  94                 if (!slots[i].len)
  95                         return i;
  96         return -1;
  97 }
  98
  99 void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 100                    int user_alloc, unsigned long userspace_addr, unsigned flags)
 101 {
 102         slots[slot].phys_addr = phys_addr;
 103         slots[slot].len = len;
 104         slots[slot].user_alloc = user_alloc;
 105         slots[slot].userspace_addr = userspace_addr;
 106         slots[slot].flags = flags;
 107 }
 108
 109 void free_slot(int slot)
 110 {
 111         slots[slot].len = 0;
 112 }
 113
 114 int get_slot(unsigned long phys_addr)
 115 {
 116         int i;
 117
 118         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 119                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 120                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 121                         return i;
 122         }
 123         return -1;
 124 }
 125
 126 int get_intersecting_slot(unsigned long phys_addr)
 127 {
 128         int i;
 129
 130         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 131                 if (slots[i].len && slots[i].phys_addr < phys_addr &&
 132                     (slots[i].phys_addr + slots[i].len) > phys_addr)
 133                         return i;
 134         return -1;
 135 }
 136
 137 /*
 138  * dirty pages logging control
 139  */
 140 static int kvm_dirty_pages_log_change(kvm_context_t kvm, unsigned long phys_addr
 141                                       , __u32 flag)
 142 {
 143         int r;
 144         int slot;
 145
 146         slot = get_slot(phys_addr);
 147         if (slot == -1) {
 148                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 149                 return 1;
 150         }
 151         flag |= slots[slot].flags;
 152 #ifdef KVM_CAP_USER_MEMORY
 153         if (slots[slot].user_alloc) {
 154                 struct kvm_userspace_memory_region mem = {
 155                         .slot = slot,
 156                         .memory_size = slots[slot].len,
 157                         .guest_phys_addr = slots[slot].phys_addr,
 158                         .userspace_addr = slots[slot].userspace_addr,
 159                         .flags = flag,
 160                 };
 161                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 162         }
 163 #endif
 164         if (!slots[slot].user_alloc) {
 165                 struct kvm_memory_region mem = {
 166                         .slot = slot,
 167                         .memory_size = slots[slot].len,
 168                         .guest_phys_addr = slots[slot].phys_addr,
 169                         .flags = flag,
 170                 };
 171                 r = ioctl(kvm->vm_fd, KVM_SET_MEMORY_REGION, &mem);
 172         }
 173         if (r == -1)
 174                 fprintf(stderr, "%s: %m\n", __FUNCTION__);
 175         return r;
 176 }
 177
 178 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm, __u32 flag)
 179 {
 180         int i, r;
 181
 182         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 183                 if (slots[i].len)
 184                         r = kvm_dirty_pages_log_change(kvm, slots[i].phys_addr,
 185                                                        flag);
 186         }
 187         return r;
 188 }
 189
 190 /**
 191  * Enable dirty page logging for all memory regions
 192  */
 193 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 194 {
 195         if (kvm->dirty_pages_log_all)
 196                 return 0;
 197         kvm->dirty_pages_log_all = 1;
 198         return kvm_dirty_pages_log_change_all(kvm, KVM_MEM_LOG_DIRTY_PAGES);
 199 }
 200
 201 /**
 202  * Enable dirty page logging only for memory regions that were created with
 203  *     dirty logging enabled (disable for all other memory regions).
 204  */
 205 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 206 {
 207         if (!kvm->dirty_pages_log_all)
 208                 return 0;
 209         kvm->dirty_pages_log_all = 0;
 210         return kvm_dirty_pages_log_change_all(kvm, 0);
 211 }
 212
 213
 214 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 215                        void *opaque)
 216 {
 217         int fd;
 218         kvm_context_t kvm;
 219         int r;
 220
 221         fd = open("/dev/kvm", O_RDWR);
 222         if (fd == -1) {
 223                 perror("open /dev/kvm");
 224                 return NULL;
 225         }
 226         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 227         if (r == -1) {
 228             fprintf(stderr, "kvm kernel version too old: "
 229                     "KVM_GET_API_VERSION ioctl not supported\n");
 230             goto out_close;
 231         }
 232         if (r < EXPECTED_KVM_API_VERSION) {
 233                 fprintf(stderr, "kvm kernel version too old: "
 234                         "We expect API version %d or newer, but got "
 235                         "version %d\n",
 236                         EXPECTED_KVM_API_VERSION, r);
 237             goto out_close;
 238         }
 239         if (r > EXPECTED_KVM_API_VERSION) {
 240             fprintf(stderr, "kvm userspace version too old\n");
 241             goto out_close;
 242         }
 243         kvm_abi = r;
 244         kvm_page_size = getpagesize();
 245         kvm = malloc(sizeof(*kvm));
 246         kvm->fd = fd;
 247         kvm->vm_fd = -1;
 248         kvm->callbacks = callbacks;
 249         kvm->opaque = opaque;
 250         kvm->dirty_pages_log_all = 0;
 251         kvm->no_irqchip_creation = 0;
 252
 253         return kvm;
 254  out_close:
 255         close(fd);
 256         return NULL;
 257 }
 258
 259 void kvm_finalize(kvm_context_t kvm)
 260 {
 261         if (kvm->vcpu_fd[0] != -1)
 262                 close(kvm->vcpu_fd[0]);
 263         if (kvm->vm_fd != -1)
 264                 close(kvm->vm_fd);
 265         close(kvm->fd);
 266         free(kvm);
 267 }
 268
 269 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 270 {
 271         kvm->no_irqchip_creation = 1;
 272 }
 273
 274 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 275 {
 276         long mmap_size;
 277         int r;
 278
 279         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 280         if (r == -1) {
 281                 r = -errno;
 282                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 283                 return r;
 284         }
 285         kvm->vcpu_fd[slot] = r;
 286         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 287         if (mmap_size == -1) {
 288                 r = -errno;
 289                 fprintf(stderr, "get vcpu mmap size: %m\n");
 290                 return r;
 291         }
 292         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 293                               kvm->vcpu_fd[slot], 0);
 294         if (kvm->run[slot] == MAP_FAILED) {
 295                 r = -errno;
 296                 fprintf(stderr, "mmap vcpu area: %m\n");
 297                 return r;
 298         }
 299         return 0;
 300 }
 301
 302 int kvm_create_vm(kvm_context_t kvm)
 303 {
 304         int fd = kvm->fd;
 305
 306         kvm->vcpu_fd[0] = -1;
 307
 308         fd = ioctl(fd, KVM_CREATE_VM, 0);
 309         if (fd == -1) {
 310                 fprintf(stderr, "kvm_create_vm: %m\n");
 311                 return -1;
 312         }
 313         kvm->vm_fd = fd;
 314         return 0;
 315 }
 316
 317 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 318                                        unsigned long phys_mem_bytes,
 319                                        void **vm_mem)
 320 {
 321         unsigned long memory = (phys_mem_bytes + PAGE_SIZE - 1) & PAGE_MASK;
 322         int r;
 323
 324 #ifdef KVM_CAP_USER_MEMORY
 325         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 326         if (r > 0)
 327                 return 0;
 328         else
 329 #endif
 330                 r = kvm_alloc_kernel_memory(kvm, memory, vm_mem);
 331         if (r < 0)
 332                 return r;
 333
 334         r = kvm_arch_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 335         if (r < 0)
 336                 return r;
 337
 338         kvm->physical_memory = *vm_mem;
 339         return 0;
 340 }
 341
 342 int kvm_check_extension(kvm_context_t kvm, int ext)
 343 {
 344         int ret;
 345
 346         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 347         if (ret > 0)
 348                 return 1;
 349         return 0;
 350 }
 351
 352 void kvm_create_irqchip(kvm_context_t kvm)
 353 {
 354         int r;
 355
 356         kvm->irqchip_in_kernel = 0;
 357 #ifdef KVM_CAP_IRQCHIP
 358         if (!kvm->no_irqchip_creation) {
 359                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 360                 if (r > 0) {    /* kernel irqchip supported */
 361                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 362                         if (r >= 0)
 363                                 kvm->irqchip_in_kernel = 1;
 364                         else
 365                                 printf("Create kernel PIC irqchip failed\n");
 366                 }
 367         }
 368 #endif
 369 }
 370
 371 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 372 {
 373         int r;
 374
 375         r = kvm_create_vm(kvm);
 376         if (r < 0)
 377                 return r;
 378         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 379         if (r < 0)
 380                 return r;
 381         init_slots();
 382         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 383         if (r < 0)
 384                 return r;
 385         kvm_create_irqchip(kvm);
 386         r = kvm_create_vcpu(kvm, 0);
 387         if (r < 0)
 388                 return r;
 389
 390         return 0;
 391 }
 392
 393
 394 #ifdef KVM_CAP_USER_MEMORY
 395
 396 void *kvm_create_userspace_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 397                         unsigned long len, int log, int writable)
 398 {
 399         int r;
 400         int prot = PROT_READ;
 401         void *ptr;
 402         struct kvm_userspace_memory_region memory = {
 403                 .memory_size = len,
 404                 .guest_phys_addr = phys_start,
 405                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 406         };
 407
 408         if (writable)
 409                 prot |= PROT_WRITE;
 410
 411         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 412         if (ptr == MAP_FAILED) {
 413                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 414                 return 0;
 415         }
 416
 417         memset(ptr, 0, len);
 418
 419         memory.userspace_addr = (unsigned long)ptr;
 420         memory.slot = get_free_slot(kvm);
 421         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 422         if (r == -1) {
 423                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 424                 return 0;
 425         }
 426         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 427                       1, memory.userspace_addr, memory.flags);
 428
 429         return ptr;
 430 }
 431
 432 #endif
 433
 434 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 435                           unsigned long len, int log, int writable)
 436 {
 437 #ifdef KVM_CAP_USER_MEMORY
 438         int r;
 439
 440         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 441         if (r > 0)
 442                 return kvm_create_userspace_phys_mem(kvm, phys_start, len,
 443                                                                 log, writable);
 444         else
 445 #endif
 446                 return kvm_create_kernel_phys_mem(kvm, phys_start, len,
 447                                                                 log, writable);
 448 }
 449
 450 int kvm_is_intersecting_mem(kvm_context_t kvm, unsigned long phys_start)
 451 {
 452         return get_intersecting_slot(phys_start) != -1;
 453 }
 454
 455 int kvm_is_allocated_mem(kvm_context_t kvm, unsigned long phys_start,
 456                          unsigned long len)
 457 {
 458         int slot;
 459
 460         slot = get_slot(phys_start);
 461         if (slot == -1)
 462                 return 0;
 463         if (slots[slot].len == len)
 464                 return 1;
 465         return 0;
 466 }
 467
 468 int kvm_create_mem_hole(kvm_context_t kvm, unsigned long phys_start,
 469                         unsigned long len)
 470 {
 471 #ifdef KVM_CAP_USER_MEMORY
 472         int slot;
 473         int r;
 474         struct kvm_userspace_memory_region rmslot;
 475         struct kvm_userspace_memory_region newslot1;
 476         struct kvm_userspace_memory_region newslot2;
 477
 478         len = (len + PAGE_SIZE - 1) & PAGE_MASK;
 479
 480         slot = get_intersecting_slot(phys_start);
 481         /* no need to create hole, as there is already hole */
 482         if (slot == -1)
 483                 return 0;
 484
 485         memset(&rmslot, 0, sizeof(struct kvm_userspace_memory_region));
 486         memset(&newslot1, 0, sizeof(struct kvm_userspace_memory_region));
 487         memset(&newslot2, 0, sizeof(struct kvm_userspace_memory_region));
 488
 489         rmslot.guest_phys_addr = slots[slot].phys_addr;
 490         rmslot.slot = slot;
 491
 492         newslot1.guest_phys_addr = slots[slot].phys_addr;
 493         newslot1.memory_size = phys_start - slots[slot].phys_addr;
 494         newslot1.slot = slot;
 495         newslot1.userspace_addr = slots[slot].userspace_addr;
 496         newslot1.flags = slots[slot].flags;
 497
 498         newslot2.guest_phys_addr = newslot1.guest_phys_addr +
 499                                    newslot1.memory_size + len;
 500         newslot2.memory_size = slots[slot].phys_addr +
 501                                slots[slot].len - newslot2.guest_phys_addr;
 502         newslot2.userspace_addr = newslot1.userspace_addr +
 503                                   newslot1.memory_size;
 504         newslot2.slot = get_free_slot(kvm);
 505         newslot2.flags = newslot1.flags;
 506
 507         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &rmslot);
 508         if (r == -1) {
 509                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 510                 return -1;
 511         }
 512         free_slot(slot);
 513
 514         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot1);
 515         if (r == -1) {
 516                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 517                 return -1;
 518         }
 519         register_slot(newslot1.slot, newslot1.guest_phys_addr,
 520                       newslot1.memory_size, 1, newslot1.userspace_addr,
 521                       newslot1.flags);
 522
 523         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot2);
 524         if (r == -1) {
 525                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 526                 return -1;
 527         }
 528         register_slot(newslot2.slot, newslot2.guest_phys_addr,
 529                       newslot2.memory_size, 1, newslot2.userspace_addr,
 530                       newslot2.flags);
 531 #endif
 532         return 0;
 533 }
 534
 535 int kvm_register_userspace_phys_mem(kvm_context_t kvm,
 536                         unsigned long phys_start, void *userspace_addr,
 537                         unsigned long len, int log)
 538 {
 539
 540 #ifdef KVM_CAP_USER_MEMORY
 541         struct kvm_userspace_memory_region memory = {
 542                 .memory_size = len,
 543                 .guest_phys_addr = phys_start,
 544                 .userspace_addr = (intptr_t)userspace_addr,
 545                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 546         };
 547         int r;
 548
 549         if (!kvm->physical_memory)
 550                 kvm->physical_memory = userspace_addr - phys_start;
 551
 552         memory.slot = get_free_slot(kvm);
 553         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 554         if (r == -1) {
 555                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 556                 return -1;
 557         }
 558         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 559                       1, memory.userspace_addr, memory.flags);
 560         return 0;
 561 #else
 562         return -ENOSYS;
 563 #endif
 564 }
 565
 566
 567 /* destroy/free a whole slot.
 568  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 569  */
 570 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 571                           unsigned long len)
 572 {
 573         int slot;
 574
 575         slot = get_slot(phys_start);
 576
 577         if (slot >= KVM_MAX_NUM_MEM_REGIONS) {
 578                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 579                         __FUNCTION__, slot);
 580                 return;
 581         }
 582         if (phys_start != slots[slot].phys_addr) {
 583                 fprintf(stderr,
 584                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 585                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 586                 phys_start = slots[slot].phys_addr;
 587         }
 588         kvm_create_phys_mem(kvm, phys_start, 0, 0, 0);
 589 }
 590
 591 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 592 {
 593         int r;
 594         struct kvm_dirty_log log = {
 595                 .slot = slot,
 596         };
 597
 598         log.dirty_bitmap = buf;
 599
 600         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 601         if (r == -1)
 602                 return -errno;
 603         return 0;
 604 }
 605
 606 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 607 {
 608         int slot;
 609
 610         slot = get_slot(phys_addr);
 611         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 612 }
 613
 614 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 615 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 616
 617 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 618                               unsigned long len, void *buf, void *opaque,
 619                               int (*cb)(unsigned long start, unsigned long len,
 620                                         void*bitmap, void *opaque))
 621 {
 622         int i;
 623         int r;
 624         unsigned long end_addr = phys_addr + len;
 625
 626         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 627                 if ((slots[i].len && slots[i].phys_addr >= phys_addr) &&
 628                     (slots[i].phys_addr + slots[i].len  <= end_addr)) {
 629                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 630                         if (r)
 631                                 return r;
 632                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 633                         if (r)
 634                                 return r;
 635                 }
 636         }
 637         return 0;
 638 }
 639
 640 #ifdef KVM_CAP_IRQCHIP
 641
 642 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level)
 643 {
 644         struct kvm_irq_level event;
 645         int r;
 646
 647         if (!kvm->irqchip_in_kernel)
 648                 return 0;
 649         event.level = level;
 650         event.irq = irq;
 651         r = ioctl(kvm->vm_fd, KVM_IRQ_LINE, &event);
 652         if (r == -1)
 653                 perror("kvm_set_irq_level");
 654         return 1;
 655 }
 656
 657 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 658 {
 659         int r;
 660
 661         if (!kvm->irqchip_in_kernel)
 662                 return 0;
 663         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 664         if (r == -1) {
 665                 r = -errno;
 666                 perror("kvm_get_irqchip\n");
 667         }
 668         return r;
 669 }
 670
 671 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 672 {
 673         int r;
 674
 675         if (!kvm->irqchip_in_kernel)
 676                 return 0;
 677         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 678         if (r == -1) {
 679                 r = -errno;
 680                 perror("kvm_set_irqchip\n");
 681         }
 682         return r;
 683 }
 684
 685 #endif
 686
 687 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 688 {
 689         uint16_t addr = run->io.port;
 690         int r;
 691         int i;
 692         void *p = (void *)run + run->io.data_offset;
 693
 694         for (i = 0; i < run->io.count; ++i) {
 695                 switch (run->io.direction) {
 696                 case KVM_EXIT_IO_IN:
 697                         switch (run->io.size) {
 698                         case 1:
 699                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 700                                 break;
 701                         case 2:
 702                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 703                                 break;
 704                         case 4:
 705                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 706                                 break;
 707                         default:
 708                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 709                                 return -EMSGSIZE;
 710                         }
 711                         break;
 712                 case KVM_EXIT_IO_OUT:
 713                         switch (run->io.size) {
 714                         case 1:
 715                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 716                                                      *(uint8_t *)p);
 717                                 break;
 718                         case 2:
 719                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 720                                                      *(uint16_t *)p);
 721                                 break;
 722                         case 4:
 723                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 724                                                      *(uint32_t *)p);
 725                                 break;
 726                         default:
 727                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 728                                 return -EMSGSIZE;
 729                         }
 730                         break;
 731                 default:
 732                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 733                         return -EPROTO;
 734                 }
 735
 736                 p += run->io.size;
 737         }
 738
 739         return 0;
 740 }
 741
 742 int handle_debug(kvm_context_t kvm, int vcpu)
 743 {
 744         return kvm->callbacks->debug(kvm->opaque, vcpu);
 745 }
 746
 747 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 748 {
 749     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 750 }
 751
 752 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 753 {
 754     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 755 }
 756
 757 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 758 {
 759     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 760 }
 761
 762 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 763 {
 764     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 765 }
 766
 767 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 768 {
 769     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 770 }
 771
 772 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 773 {
 774     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 775 }
 776
 777 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 778 {
 779         unsigned long addr = kvm_run->mmio.phys_addr;
 780         void *data = kvm_run->mmio.data;
 781
 782         /* hack: Red Hat 7.1 generates these weird accesses. */
 783         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 784             return 0;
 785
 786         if (kvm_run->mmio.is_write)
 787                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 788                                         kvm_run->mmio.len);
 789         else
 790                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 791                                         kvm_run->mmio.len);
 792 }
 793
 794 int handle_io_window(kvm_context_t kvm)
 795 {
 796         return kvm->callbacks->io_window(kvm->opaque);
 797 }
 798
 799 int handle_halt(kvm_context_t kvm, int vcpu)
 800 {
 801         return kvm->callbacks->halt(kvm->opaque, vcpu);
 802 }
 803
 804 int handle_shutdown(kvm_context_t kvm, int vcpu)
 805 {
 806         return kvm->callbacks->shutdown(kvm->opaque, vcpu);
 807 }
 808
 809 int try_push_interrupts(kvm_context_t kvm)
 810 {
 811         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 812 }
 813
 814 void post_kvm_run(kvm_context_t kvm, int vcpu)
 815 {
 816         kvm->callbacks->post_kvm_run(kvm->opaque, vcpu);
 817 }
 818
 819 int pre_kvm_run(kvm_context_t kvm, int vcpu)
 820 {
 821         return kvm->callbacks->pre_kvm_run(kvm->opaque, vcpu);
 822 }
 823
 824 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 825 {
 826         struct kvm_run *run = kvm->run[vcpu];
 827
 828         return run->if_flag;
 829 }
 830
 831 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 832 {
 833         struct kvm_run *run = kvm->run[vcpu];
 834
 835         return run->ready_for_interrupt_injection;
 836 }
 837
 838 int kvm_run(kvm_context_t kvm, int vcpu)
 839 {
 840         int r;
 841         int fd = kvm->vcpu_fd[vcpu];
 842         struct kvm_run *run = kvm->run[vcpu];
 843
 844 again:
 845         if (!kvm->irqchip_in_kernel)
 846                 run->request_interrupt_window = try_push_interrupts(kvm);
 847         r = pre_kvm_run(kvm, vcpu);
 848         if (r)
 849             return r;
 850         r = ioctl(fd, KVM_RUN, 0);
 851         post_kvm_run(kvm, vcpu);
 852
 853         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 854                 r = -errno;
 855                 printf("kvm_run: %m\n");
 856                 return r;
 857         }
 858         if (r == -1) {
 859                 r = handle_io_window(kvm);
 860                 goto more;
 861         }
 862         if (1) {
 863                 switch (run->exit_reason) {
 864                 case KVM_EXIT_UNKNOWN:
 865                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 866                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 867                         kvm_show_regs(kvm, vcpu);
 868                         abort();
 869                         break;
 870                 case KVM_EXIT_FAIL_ENTRY:
 871                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 872                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 873                         return -ENOEXEC;
 874                         break;
 875                 case KVM_EXIT_EXCEPTION:
 876                         fprintf(stderr, "exception %d (%x)\n",
 877                                run->ex.exception,
 878                                run->ex.error_code);
 879                         kvm_show_regs(kvm, vcpu);
 880                         kvm_show_code(kvm, vcpu);
 881                         abort();
 882                         break;
 883                 case KVM_EXIT_IO:
 884                         r = handle_io(kvm, run, vcpu);
 885                         break;
 886                 case KVM_EXIT_DEBUG:
 887                         r = handle_debug(kvm, vcpu);
 888                         break;
 889                 case KVM_EXIT_MMIO:
 890                         r = handle_mmio(kvm, run);
 891                         break;
 892                 case KVM_EXIT_HLT:
 893                         r = handle_halt(kvm, vcpu);
 894                         break;
 895                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 896                         break;
 897                 case KVM_EXIT_SHUTDOWN:
 898                         r = handle_shutdown(kvm, vcpu);
 899                         break;
 900                 default:
 901                         if (kvm_arch_run(run, kvm, vcpu)) {
 902                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 903                                                         run->exit_reason);
 904                                 kvm_show_regs(kvm, vcpu);
 905                                 abort();
 906                         }
 907                         break;
 908                 }
 909         }
 910 more:
 911         if (!r)
 912                 goto again;
 913         return r;
 914 }
 915
 916 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 917 {
 918         struct kvm_interrupt intr;
 919
 920         intr.irq = irq;
 921         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 922 }
 923
 924 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
 925 {
 926         return ioctl(kvm->vcpu_fd[vcpu], KVM_DEBUG_GUEST, dbg);
 927 }
 928
 929 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
 930 {
 931         struct kvm_signal_mask *sigmask;
 932         int r;
 933
 934         if (!sigset) {
 935                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
 936                 if (r == -1)
 937                         r = -errno;
 938                 return r;
 939         }
 940         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
 941         if (!sigmask)
 942                 return -ENOMEM;
 943
 944         sigmask->len = 8;
 945         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
 946         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
 947         if (r == -1)
 948                 r = -errno;
 949         free(sigmask);
 950         return r;
 951 }
 952
 953 int kvm_irqchip_in_kernel(kvm_context_t kvm)
 954 {
 955     return kvm->irqchip_in_kernel;
 956 }