libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include <inttypes.h>
  38 #include "libkvm.h"
  39
  40 #if defined(__x86_64__) || defined(__i386__)
  41 #include "kvm-x86.h"
  42 #endif
  43
  44 #if defined(__ia64__)
  45 #include "kvm-ia64.h"
  46 #endif
  47
  48 #if defined(__powerpc__)
  49 #include "kvm-powerpc.h"
  50 #endif
  51
  52 #if defined(__s390__)
  53 #include "kvm-s390.h"
  54 #endif
  55
  56 //#define DEBUG_MEMREG
  57 #ifdef  DEBUG_MEMREG
  58 #define DPRINTF(fmt, args...) \
  59         do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
  60 #else
  61 #define DPRINTF(fmt, args...) do {} while (0)
  62 #endif
  63
  64
  65 int kvm_abi = EXPECTED_KVM_API_VERSION;
  66 int kvm_page_size;
  67
  68 struct slot_info {
  69         unsigned long phys_addr;
  70         unsigned long len;
  71         unsigned long userspace_addr;
  72         unsigned flags;
  73         int logging_count;
  74 };
  75
  76 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  77
  78 void init_slots(void)
  79 {
  80         int i;
  81
  82         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  83                 slots[i].len = 0;
  84 }
  85
  86 int get_free_slot(kvm_context_t kvm)
  87 {
  88         int i;
  89         int tss_ext;
  90
  91 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
  92         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  93 #else
  94         tss_ext = 0;
  95 #endif
  96
  97         /*
  98          * on older kernels where the set tss ioctl is not supprted we must save
  99          * slot 0 to hold the extended memory, as the vmx will use the last 3
 100          * pages of this slot.
 101          */
 102         if (tss_ext > 0)
 103                 i = 0;
 104         else
 105                 i = 1;
 106
 107         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
 108                 if (!slots[i].len)
 109                         return i;
 110         return -1;
 111 }
 112
 113 void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 114                    unsigned long userspace_addr, unsigned flags)
 115 {
 116         slots[slot].phys_addr = phys_addr;
 117         slots[slot].len = len;
 118         slots[slot].userspace_addr = userspace_addr;
 119         slots[slot].flags = flags;
 120 }
 121
 122 void free_slot(int slot)
 123 {
 124         slots[slot].len = 0;
 125         slots[slot].logging_count = 0;
 126 }
 127
 128 int get_slot(unsigned long phys_addr)
 129 {
 130         int i;
 131
 132         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 133                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 134                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 135                         return i;
 136         }
 137         return -1;
 138 }
 139
 140 /* Returns -1 if this slot is not totally contained on any other,
 141  * and the number of the slot otherwise */
 142 int get_container_slot(uint64_t phys_addr, unsigned long size)
 143 {
 144         int i;
 145
 146         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 147                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 148                     (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
 149                         return i;
 150         return -1;
 151 }
 152
 153 int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr, unsigned long size)
 154 {
 155         int slot = get_container_slot(phys_addr, size);
 156         if (slot == -1)
 157                 return 0;
 158         return 1;
 159 }
 160
 161 /*
 162  * dirty pages logging control
 163  */
 164 static int kvm_dirty_pages_log_change(kvm_context_t kvm,
 165                                       unsigned long phys_addr,
 166                                       unsigned flags,
 167                                       unsigned mask)
 168 {
 169         int r = -1;
 170         int slot = get_slot(phys_addr);
 171
 172         if (slot == -1) {
 173                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 174                 return 1;
 175         }
 176
 177         flags = (slots[slot].flags & ~mask) | flags;
 178         if (flags == slots[slot].flags)
 179                 return 0;
 180         slots[slot].flags = flags;
 181
 182         {
 183                 struct kvm_userspace_memory_region mem = {
 184                         .slot = slot,
 185                         .memory_size = slots[slot].len,
 186                         .guest_phys_addr = slots[slot].phys_addr,
 187                         .userspace_addr = slots[slot].userspace_addr,
 188                         .flags = slots[slot].flags,
 189                 };
 190
 191
 192                 DPRINTF("slot %d start %llx len %llx flags %x\n",
 193                         mem.slot,
 194                         mem.guest_phys_addr,
 195                         mem.memory_size,
 196                         mem.flags);
 197                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 198                 if (r == -1)
 199                         fprintf(stderr, "%s: %m\n", __FUNCTION__);
 200         }
 201         return r;
 202 }
 203
 204 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
 205                                           int (*change)(kvm_context_t kvm,
 206                                                         uint64_t start,
 207                                                         uint64_t len))
 208 {
 209         int i, r;
 210
 211         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 212                 if (slots[i].len)
 213                         r = change(kvm, slots[i].phys_addr, slots[i].len);
 214         }
 215         return r;
 216 }
 217
 218 int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm,
 219                                     uint64_t phys_addr,
 220                                     uint64_t len)
 221 {
 222         int slot = get_slot(phys_addr);
 223
 224         DPRINTF("start %"PRIx64" len %"PRIx64"\n", phys_addr, len);
 225         if (slot == -1) {
 226                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 227                 return -EINVAL;
 228         }
 229
 230         if (slots[slot].logging_count++)
 231                 return 0;
 232
 233         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 234                                           KVM_MEM_LOG_DIRTY_PAGES,
 235                                           KVM_MEM_LOG_DIRTY_PAGES);
 236 }
 237
 238 int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm,
 239                                      uint64_t phys_addr,
 240                                      uint64_t len)
 241 {
 242         int slot = get_slot(phys_addr);
 243
 244         if (slot == -1) {
 245                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 246                 return -EINVAL;
 247         }
 248
 249         if (--slots[slot].logging_count)
 250                 return 0;
 251
 252         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 253                                           0,
 254                                           KVM_MEM_LOG_DIRTY_PAGES);
 255 }
 256
 257 /**
 258  * Enable dirty page logging for all memory regions
 259  */
 260 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 261 {
 262         if (kvm->dirty_pages_log_all)
 263                 return 0;
 264         kvm->dirty_pages_log_all = 1;
 265         return kvm_dirty_pages_log_change_all(kvm,
 266                                               kvm_dirty_pages_log_enable_slot);
 267 }
 268
 269 /**
 270  * Enable dirty page logging only for memory regions that were created with
 271  *     dirty logging enabled (disable for all other memory regions).
 272  */
 273 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 274 {
 275         if (!kvm->dirty_pages_log_all)
 276                 return 0;
 277         kvm->dirty_pages_log_all = 0;
 278         return kvm_dirty_pages_log_change_all(kvm,
 279                                               kvm_dirty_pages_log_disable_slot);
 280 }
 281
 282
 283 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 284                        void *opaque)
 285 {
 286         int fd;
 287         kvm_context_t kvm;
 288         int r;
 289
 290         fd = open("/dev/kvm", O_RDWR);
 291         if (fd == -1) {
 292                 perror("open /dev/kvm");
 293                 return NULL;
 294         }
 295         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 296         if (r == -1) {
 297             fprintf(stderr, "kvm kernel version too old: "
 298                     "KVM_GET_API_VERSION ioctl not supported\n");
 299             goto out_close;
 300         }
 301         if (r < EXPECTED_KVM_API_VERSION) {
 302                 fprintf(stderr, "kvm kernel version too old: "
 303                         "We expect API version %d or newer, but got "
 304                         "version %d\n",
 305                         EXPECTED_KVM_API_VERSION, r);
 306             goto out_close;
 307         }
 308         if (r > EXPECTED_KVM_API_VERSION) {
 309             fprintf(stderr, "kvm userspace version too old\n");
 310             goto out_close;
 311         }
 312         kvm_abi = r;
 313         kvm_page_size = getpagesize();
 314         kvm = malloc(sizeof(*kvm));
 315         if (kvm == NULL)
 316                 goto out_close;
 317         memset(kvm, 0, sizeof(*kvm));
 318         kvm->fd = fd;
 319         kvm->vm_fd = -1;
 320         kvm->callbacks = callbacks;
 321         kvm->opaque = opaque;
 322         kvm->dirty_pages_log_all = 0;
 323         kvm->no_irqchip_creation = 0;
 324         kvm->no_pit_creation = 0;
 325
 326         return kvm;
 327  out_close:
 328         close(fd);
 329         return NULL;
 330 }
 331
 332 void kvm_finalize(kvm_context_t kvm)
 333 {
 334         if (kvm->vcpu_fd[0] != -1)
 335                 close(kvm->vcpu_fd[0]);
 336         if (kvm->vm_fd != -1)
 337                 close(kvm->vm_fd);
 338         close(kvm->fd);
 339         free(kvm);
 340 }
 341
 342 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 343 {
 344         kvm->no_irqchip_creation = 1;
 345 }
 346
 347 void kvm_disable_pit_creation(kvm_context_t kvm)
 348 {
 349         kvm->no_pit_creation = 1;
 350 }
 351
 352 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 353 {
 354         long mmap_size;
 355         int r;
 356
 357         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 358         if (r == -1) {
 359                 r = -errno;
 360                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 361                 return r;
 362         }
 363         kvm->vcpu_fd[slot] = r;
 364         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 365         if (mmap_size == -1) {
 366                 r = -errno;
 367                 fprintf(stderr, "get vcpu mmap size: %m\n");
 368                 return r;
 369         }
 370         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 371                               kvm->vcpu_fd[slot], 0);
 372         if (kvm->run[slot] == MAP_FAILED) {
 373                 r = -errno;
 374                 fprintf(stderr, "mmap vcpu area: %m\n");
 375                 return r;
 376         }
 377         return 0;
 378 }
 379
 380 int kvm_create_vm(kvm_context_t kvm)
 381 {
 382         int fd = kvm->fd;
 383
 384         kvm->vcpu_fd[0] = -1;
 385
 386         fd = ioctl(fd, KVM_CREATE_VM, 0);
 387         if (fd == -1) {
 388                 fprintf(stderr, "kvm_create_vm: %m\n");
 389                 return -1;
 390         }
 391         kvm->vm_fd = fd;
 392         return 0;
 393 }
 394
 395 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 396                                        unsigned long phys_mem_bytes,
 397                                        void **vm_mem)
 398 {
 399 #ifdef KVM_CAP_USER_MEMORY
 400         int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 401         if (r > 0)
 402                 return 0;
 403         fprintf(stderr, "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
 404 #else
 405 #error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
 406 #endif
 407         return -1;
 408 }
 409
 410 int kvm_check_extension(kvm_context_t kvm, int ext)
 411 {
 412         int ret;
 413
 414         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 415         if (ret > 0)
 416                 return 1;
 417         return 0;
 418 }
 419
 420 void kvm_create_irqchip(kvm_context_t kvm)
 421 {
 422         int r;
 423
 424         kvm->irqchip_in_kernel = 0;
 425 #ifdef KVM_CAP_IRQCHIP
 426         if (!kvm->no_irqchip_creation) {
 427                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 428                 if (r > 0) {    /* kernel irqchip supported */
 429                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 430                         if (r >= 0)
 431                                 kvm->irqchip_in_kernel = 1;
 432                         else
 433                                 fprintf(stderr, "Create kernel PIC irqchip failed\n");
 434                 }
 435         }
 436 #endif
 437 }
 438
 439 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 440 {
 441         int r;
 442
 443         r = kvm_create_vm(kvm);
 444         if (r < 0)
 445                 return r;
 446         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 447         if (r < 0)
 448                 return r;
 449         init_slots();
 450         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 451         if (r < 0)
 452                 return r;
 453         kvm_create_irqchip(kvm);
 454
 455         return 0;
 456 }
 457
 458
 459 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 460                           unsigned long len, int log, int writable)
 461 {
 462         int r;
 463         int prot = PROT_READ;
 464         void *ptr;
 465         struct kvm_userspace_memory_region memory = {
 466                 .memory_size = len,
 467                 .guest_phys_addr = phys_start,
 468                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 469         };
 470
 471         if (writable)
 472                 prot |= PROT_WRITE;
 473
 474 #if !defined(__s390__)
 475         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 476 #else
 477         ptr = mmap(LIBKVM_S390_ORIGIN, len, prot | PROT_EXEC,
 478                 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 479 #endif
 480         if (ptr == MAP_FAILED) {
 481                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 482                 return 0;
 483         }
 484
 485         memset(ptr, 0, len);
 486
 487         memory.userspace_addr = (unsigned long)ptr;
 488         memory.slot = get_free_slot(kvm);
 489         DPRINTF("slot %d start %llx len %llx flags %x\n",
 490                 memory.slot,
 491                 memory.guest_phys_addr,
 492                 memory.memory_size,
 493                 memory.flags);
 494         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 495         if (r == -1) {
 496                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 497                 return 0;
 498         }
 499         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 500                       memory.userspace_addr, memory.flags);
 501
 502         return ptr;
 503 }
 504
 505 int kvm_register_phys_mem(kvm_context_t kvm,
 506                           unsigned long phys_start, void *userspace_addr,
 507                           unsigned long len, int log)
 508 {
 509
 510         struct kvm_userspace_memory_region memory = {
 511                 .memory_size = len,
 512                 .guest_phys_addr = phys_start,
 513                 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
 514                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 515         };
 516         int r;
 517
 518         memory.slot = get_free_slot(kvm);
 519         DPRINTF("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %lx\n",
 520                 memory.guest_phys_addr, memory.memory_size,
 521                 memory.userspace_addr, memory.slot, memory.flags);
 522         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 523         if (r == -1) {
 524                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 525                 return -1;
 526         }
 527         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 528                       memory.userspace_addr, memory.flags);
 529         return 0;
 530 }
 531
 532
 533 /* destroy/free a whole slot.
 534  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 535  */
 536 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 537                           unsigned long len)
 538 {
 539         int slot;
 540         int r;
 541         struct kvm_userspace_memory_region memory = {
 542                 .memory_size = 0,
 543                 .guest_phys_addr = phys_start,
 544                 .userspace_addr = 0,
 545                 .flags = 0,
 546         };
 547
 548         slot = get_slot(phys_start);
 549
 550         if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
 551                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 552                         __FUNCTION__, slot);
 553                 return;
 554         }
 555         if (phys_start != slots[slot].phys_addr) {
 556                 fprintf(stderr,
 557                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 558                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 559                 phys_start = slots[slot].phys_addr;
 560         }
 561
 562         memory.slot = slot;
 563         DPRINTF("slot %d start %llx len %llx flags %x\n",
 564                 memory.slot,
 565                 memory.guest_phys_addr,
 566                 memory.memory_size,
 567                 memory.flags);
 568         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 569         if (r == -1) {
 570                 fprintf(stderr, "destroy_userspace_phys_mem: %s",
 571                         strerror(errno));
 572                 return;
 573         }
 574
 575         free_slot(memory.slot);
 576 }
 577
 578 void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr, unsigned long size)
 579 {
 580
 581         int slot = get_container_slot(phys_addr, size);
 582
 583         if (slot != -1) {
 584                 DPRINTF("Unregistering memory region %llx (%lx)\n", phys_addr, size);
 585                 kvm_destroy_phys_mem(kvm, phys_addr, size);
 586                 return;
 587         }
 588 }
 589
 590 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 591 {
 592         int r;
 593         struct kvm_dirty_log log = {
 594                 .slot = slot,
 595         };
 596
 597         log.dirty_bitmap = buf;
 598
 599         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 600         if (r == -1)
 601                 return -errno;
 602         return 0;
 603 }
 604
 605 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 606 {
 607         int slot;
 608
 609         slot = get_slot(phys_addr);
 610         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 611 }
 612
 613 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 614 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 615
 616 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 617                               unsigned long len, void *buf, void *opaque,
 618                               int (*cb)(unsigned long start, unsigned long len,
 619                                         void*bitmap, void *opaque))
 620 {
 621         int i;
 622         int r;
 623         unsigned long end_addr = phys_addr + len;
 624
 625         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 626                 if ((slots[i].len && (uint64_t)slots[i].phys_addr >= phys_addr)
 627                     && ((uint64_t)slots[i].phys_addr + slots[i].len  <= end_addr)) {
 628                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 629                         if (r)
 630                                 return r;
 631                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 632                         if (r)
 633                                 return r;
 634                 }
 635         }
 636         return 0;
 637 }
 638
 639 #ifdef KVM_CAP_IRQCHIP
 640
 641 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level)
 642 {
 643         struct kvm_irq_level event;
 644         int r;
 645
 646         if (!kvm->irqchip_in_kernel)
 647                 return 0;
 648         event.level = level;
 649         event.irq = irq;
 650         r = ioctl(kvm->vm_fd, KVM_IRQ_LINE, &event);
 651         if (r == -1)
 652                 perror("kvm_set_irq_level");
 653         return 1;
 654 }
 655
 656 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 657 {
 658         int r;
 659
 660         if (!kvm->irqchip_in_kernel)
 661                 return 0;
 662         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 663         if (r == -1) {
 664                 r = -errno;
 665                 perror("kvm_get_irqchip\n");
 666         }
 667         return r;
 668 }
 669
 670 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 671 {
 672         int r;
 673
 674         if (!kvm->irqchip_in_kernel)
 675                 return 0;
 676         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 677         if (r == -1) {
 678                 r = -errno;
 679                 perror("kvm_set_irqchip\n");
 680         }
 681         return r;
 682 }
 683
 684 #endif
 685
 686 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 687 {
 688         uint16_t addr = run->io.port;
 689         int r;
 690         int i;
 691         void *p = (void *)run + run->io.data_offset;
 692
 693         for (i = 0; i < run->io.count; ++i) {
 694                 switch (run->io.direction) {
 695                 case KVM_EXIT_IO_IN:
 696                         switch (run->io.size) {
 697                         case 1:
 698                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 699                                 break;
 700                         case 2:
 701                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 702                                 break;
 703                         case 4:
 704                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 705                                 break;
 706                         default:
 707                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 708                                 return -EMSGSIZE;
 709                         }
 710                         break;
 711                 case KVM_EXIT_IO_OUT:
 712                         switch (run->io.size) {
 713                         case 1:
 714                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 715                                                      *(uint8_t *)p);
 716                                 break;
 717                         case 2:
 718                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 719                                                      *(uint16_t *)p);
 720                                 break;
 721                         case 4:
 722                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 723                                                      *(uint32_t *)p);
 724                                 break;
 725                         default:
 726                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 727                                 return -EMSGSIZE;
 728                         }
 729                         break;
 730                 default:
 731                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 732                         return -EPROTO;
 733                 }
 734
 735                 p += run->io.size;
 736         }
 737
 738         return 0;
 739 }
 740
 741 int handle_debug(kvm_context_t kvm, int vcpu, void *env)
 742 {
 743 #ifdef KVM_CAP_SET_GUEST_DEBUG
 744     struct kvm_run *run = kvm->run[vcpu];
 745
 746     return kvm->callbacks->debug(kvm->opaque, env, &run->debug.arch);
 747 #else
 748     return 0;
 749 #endif
 750 }
 751
 752 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 753 {
 754     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 755 }
 756
 757 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 758 {
 759     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 760 }
 761
 762 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 763 {
 764     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 765 }
 766
 767 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 768 {
 769     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 770 }
 771
 772 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 773 {
 774     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 775 }
 776
 777 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 778 {
 779     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 780 }
 781
 782 #ifdef KVM_CAP_MP_STATE
 783 int kvm_get_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 784 {
 785     int r;
 786
 787     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 788     if (r > 0)
 789         return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_MP_STATE, mp_state);
 790     return -ENOSYS;
 791 }
 792
 793 int kvm_set_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 794 {
 795     int r;
 796
 797     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 798     if (r > 0)
 799         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_MP_STATE, mp_state);
 800     return -ENOSYS;
 801 }
 802 #endif
 803
 804 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 805 {
 806         unsigned long addr = kvm_run->mmio.phys_addr;
 807         void *data = kvm_run->mmio.data;
 808
 809         /* hack: Red Hat 7.1 generates these weird accesses. */
 810         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 811             return 0;
 812
 813         if (kvm_run->mmio.is_write)
 814                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 815                                         kvm_run->mmio.len);
 816         else
 817                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 818                                         kvm_run->mmio.len);
 819 }
 820
 821 int handle_io_window(kvm_context_t kvm)
 822 {
 823         return kvm->callbacks->io_window(kvm->opaque);
 824 }
 825
 826 int handle_halt(kvm_context_t kvm, int vcpu)
 827 {
 828         return kvm->callbacks->halt(kvm->opaque, vcpu);
 829 }
 830
 831 int handle_shutdown(kvm_context_t kvm, void *env)
 832 {
 833         return kvm->callbacks->shutdown(kvm->opaque, env);
 834 }
 835
 836 int try_push_interrupts(kvm_context_t kvm)
 837 {
 838         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 839 }
 840
 841 static inline void push_nmi(kvm_context_t kvm)
 842 {
 843 #ifdef KVM_CAP_USER_NMI
 844         kvm->callbacks->push_nmi(kvm->opaque);
 845 #endif /* KVM_CAP_USER_NMI */
 846 }
 847
 848 void post_kvm_run(kvm_context_t kvm, void *env)
 849 {
 850         kvm->callbacks->post_kvm_run(kvm->opaque, env);
 851 }
 852
 853 int pre_kvm_run(kvm_context_t kvm, void *env)
 854 {
 855         return kvm->callbacks->pre_kvm_run(kvm->opaque, env);
 856 }
 857
 858 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 859 {
 860         struct kvm_run *run = kvm->run[vcpu];
 861
 862         return run->if_flag;
 863 }
 864
 865 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 866 {
 867         struct kvm_run *run = kvm->run[vcpu];
 868
 869         return run->ready_for_interrupt_injection;
 870 }
 871
 872 int kvm_run(kvm_context_t kvm, int vcpu, void *env)
 873 {
 874         int r;
 875         int fd = kvm->vcpu_fd[vcpu];
 876         struct kvm_run *run = kvm->run[vcpu];
 877
 878 again:
 879         push_nmi(kvm);
 880 #if !defined(__s390__)
 881         if (!kvm->irqchip_in_kernel)
 882                 run->request_interrupt_window = try_push_interrupts(kvm);
 883 #endif
 884         r = pre_kvm_run(kvm, env);
 885         if (r)
 886             return r;
 887         r = ioctl(fd, KVM_RUN, 0);
 888
 889         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 890                 r = -errno;
 891                 post_kvm_run(kvm, env);
 892                 fprintf(stderr, "kvm_run: %s\n", strerror(-r));
 893                 return r;
 894         }
 895
 896         post_kvm_run(kvm, env);
 897
 898 #if defined(KVM_CAP_COALESCED_MMIO)
 899         if (kvm->coalesced_mmio) {
 900                 struct kvm_coalesced_mmio_ring *ring = (void *)run +
 901                                                 kvm->coalesced_mmio * PAGE_SIZE;
 902                 while (ring->first != ring->last) {
 903                         kvm->callbacks->mmio_write(kvm->opaque,
 904                                  ring->coalesced_mmio[ring->first].phys_addr,
 905                                 &ring->coalesced_mmio[ring->first].data[0],
 906                                  ring->coalesced_mmio[ring->first].len);
 907                         smp_wmb();
 908                         ring->first = (ring->first + 1) %
 909                                                         KVM_COALESCED_MMIO_MAX;
 910                 }
 911         }
 912 #endif
 913
 914 #if !defined(__s390__)
 915         if (r == -1) {
 916                 r = handle_io_window(kvm);
 917                 goto more;
 918         }
 919 #endif
 920         if (1) {
 921                 switch (run->exit_reason) {
 922                 case KVM_EXIT_UNKNOWN:
 923                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 924                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 925                         kvm_show_regs(kvm, vcpu);
 926                         abort();
 927                         break;
 928                 case KVM_EXIT_FAIL_ENTRY:
 929                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 930                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 931                         kvm_show_regs(kvm, vcpu);
 932                         return -ENOEXEC;
 933                         break;
 934                 case KVM_EXIT_EXCEPTION:
 935                         fprintf(stderr, "exception %d (%x)\n",
 936                                run->ex.exception,
 937                                run->ex.error_code);
 938                         kvm_show_regs(kvm, vcpu);
 939                         kvm_show_code(kvm, vcpu);
 940                         abort();
 941                         break;
 942                 case KVM_EXIT_IO:
 943                         r = handle_io(kvm, run, vcpu);
 944                         break;
 945                 case KVM_EXIT_DEBUG:
 946                         r = handle_debug(kvm, vcpu, env);
 947                         break;
 948                 case KVM_EXIT_MMIO:
 949                         r = handle_mmio(kvm, run);
 950                         break;
 951                 case KVM_EXIT_HLT:
 952                         r = handle_halt(kvm, vcpu);
 953                         break;
 954                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 955                         break;
 956                 case KVM_EXIT_SHUTDOWN:
 957                         r = handle_shutdown(kvm, env);
 958                         break;
 959 #if defined(__s390__)
 960                 case KVM_EXIT_S390_SIEIC:
 961                         r = kvm->callbacks->s390_handle_intercept(kvm, vcpu,
 962                                 run);
 963                         break;
 964                 case KVM_EXIT_S390_RESET:
 965                         r = kvm->callbacks->s390_handle_reset(kvm, vcpu, run);
 966                         break;
 967 #endif
 968                 default:
 969                         if (kvm_arch_run(run, kvm, vcpu)) {
 970                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 971                                                         run->exit_reason);
 972                                 kvm_show_regs(kvm, vcpu);
 973                                 abort();
 974                         }
 975                         break;
 976                 }
 977         }
 978 more:
 979         if (!r)
 980                 goto again;
 981         return r;
 982 }
 983
 984 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 985 {
 986         struct kvm_interrupt intr;
 987
 988         intr.irq = irq;
 989         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 990 }
 991
 992 #ifdef KVM_CAP_SET_GUEST_DEBUG
 993 int kvm_set_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_guest_debug *dbg)
 994 {
 995         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_GUEST_DEBUG, dbg);
 996 }
 997 #endif
 998
 999 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
1000 {
1001         struct kvm_signal_mask *sigmask;
1002         int r;
1003
1004         if (!sigset) {
1005                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
1006                 if (r == -1)
1007                         r = -errno;
1008                 return r;
1009         }
1010         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
1011         if (!sigmask)
1012                 return -ENOMEM;
1013
1014         sigmask->len = 8;
1015         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1016         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
1017         if (r == -1)
1018                 r = -errno;
1019         free(sigmask);
1020         return r;
1021 }
1022
1023 int kvm_irqchip_in_kernel(kvm_context_t kvm)
1024 {
1025         return kvm->irqchip_in_kernel;
1026 }
1027
1028 int kvm_pit_in_kernel(kvm_context_t kvm)
1029 {
1030         return kvm->pit_in_kernel;
1031 }
1032
1033 int kvm_has_sync_mmu(kvm_context_t kvm)
1034 {
1035         int r = 0;
1036 #ifdef KVM_CAP_SYNC_MMU
1037         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
1038 #endif
1039         return r;
1040 }
1041
1042 int kvm_inject_nmi(kvm_context_t kvm, int vcpu)
1043 {
1044 #ifdef KVM_CAP_USER_NMI
1045         return ioctl(kvm->vcpu_fd[vcpu], KVM_NMI);
1046 #else
1047         return -ENOSYS;
1048 #endif
1049 }
1050
1051 int kvm_init_coalesced_mmio(kvm_context_t kvm)
1052 {
1053         int r = 0;
1054         kvm->coalesced_mmio = 0;
1055 #ifdef KVM_CAP_COALESCED_MMIO
1056         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
1057         if (r > 0) {
1058                 kvm->coalesced_mmio = r;
1059                 return 0;
1060         }
1061 #endif
1062         return r;
1063 }
1064
1065 int kvm_register_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1066 {
1067 #ifdef KVM_CAP_COALESCED_MMIO
1068         struct kvm_coalesced_mmio_zone zone;
1069         int r;
1070
1071         if (kvm->coalesced_mmio) {
1072
1073                 zone.addr = addr;
1074                 zone.size = size;
1075
1076                 r = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
1077                 if (r == -1) {
1078                         perror("kvm_register_coalesced_mmio_zone");
1079                         return -errno;
1080                 }
1081                 return 0;
1082         }
1083 #endif
1084         return -ENOSYS;
1085 }
1086
1087 int kvm_unregister_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1088 {
1089 #ifdef KVM_CAP_COALESCED_MMIO
1090         struct kvm_coalesced_mmio_zone zone;
1091         int r;
1092
1093         if (kvm->coalesced_mmio) {
1094
1095                 zone.addr = addr;
1096                 zone.size = size;
1097
1098                 r = ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1099                 if (r == -1) {
1100                         perror("kvm_unregister_coalesced_mmio_zone");
1101                         return -errno;
1102                 }
1103                 DPRINTF("Unregistered coalesced mmio region for %llx (%lx)\n", addr, size);
1104                 return 0;
1105         }
1106 #endif
1107         return -ENOSYS;
1108 }
1109
1110 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1111 int kvm_assign_pci_device(kvm_context_t kvm,
1112                           struct kvm_assigned_pci_dev *assigned_dev)
1113 {
1114         int ret;
1115
1116         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
1117         if (ret < 0)
1118                 return -errno;
1119
1120         return ret;
1121 }
1122
1123 int kvm_assign_irq(kvm_context_t kvm,
1124                    struct kvm_assigned_irq *assigned_irq)
1125 {
1126         int ret;
1127
1128         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_IRQ, assigned_irq);
1129         if (ret < 0)
1130                 return -errno;
1131
1132         return ret;
1133 }
1134 #endif
1135
1136 int kvm_destroy_memory_region_works(kvm_context_t kvm)
1137 {
1138         int ret = 0;
1139
1140 #ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
1141         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
1142                     KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
1143         if (ret <= 0)
1144                 ret = 0;
1145 #endif
1146         return ret;
1147 }