?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "support colon in filenames"
[qemu-kvm/fedora.git] / qemu-kvm.c
blob982ad334ddfa93019283e3c6ba0b7a977048a9db
1 /*
2 * qemu/kvm integration
3 *
4 * Copyright (C) 2006-2008 Qumranet Technologies
5 *
6 * Licensed under the terms of the GNU GPL version 2 or higher.
7 */
8 #include "config.h"
9 #include "config-host.h"
10
11 #include <assert.h>
12 #include <string.h>
13 #include "hw/hw.h"
14 #include "sysemu.h"
15 #include "qemu-common.h"
16 #include "console.h"
17 #include "block.h"
18 #include "compatfd.h"
19 #include "gdbstub.h"
20
21 #include "qemu-kvm.h"
22 #include "libkvm-all.h"
23 #include "libkvm.h"
24
25 #include <pthread.h>
26 #include <sys/utsname.h>
27 #include <sys/syscall.h>
28 #include <sys/mman.h>
29 #include <sys/ioctl.h>
30 #include <signal.h>
31
32 #define false 0
33 #define true 1
34
35 #define EXPECTED_KVM_API_VERSION 12
36
37 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
38 #error libkvm: userspace and kernel version mismatch
39 #endif
40
41 int kvm_allowed = 1;
42 int kvm_irqchip = 1;
43 int kvm_pit = 1;
44 int kvm_pit_reinject = 1;
45 int kvm_nested = 0;
46 kvm_context_t kvm_context;
47
48 pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
49 pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
50 pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
51 pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
52 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
53 __thread CPUState *current_env;
54
55 static int qemu_system_ready;
56
57 #define SIG_IPI (SIGRTMIN+4)
58
59 pthread_t io_thread;
60 static int io_thread_fd = -1;
61 static int io_thread_sigfd = -1;
62
63 static CPUState *kvm_debug_cpu_requested;
64
65 static uint64_t phys_ram_size;
66
67 /* The list of ioperm_data */
68 static LIST_HEAD(, ioperm_data) ioperm_head;
69
70 //#define DEBUG_MEMREG
71 #ifdef DEBUG_MEMREG
72 #define DPRINTF(fmt, args...) \
73 do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
74 #else
75 #define DPRINTF(fmt, args...) do {} while (0)
76 #endif
77
78 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
79
80 int kvm_abi = EXPECTED_KVM_API_VERSION;
81 int kvm_page_size;
82
83 #ifdef KVM_CAP_SET_GUEST_DEBUG
84 static int kvm_debug(void *opaque, void *data,
85 struct kvm_debug_exit_arch *arch_info)
86 {
87 int handle = kvm_arch_debug(arch_info);
88 CPUState *env = data;
89
90 if (handle) {
91 kvm_debug_cpu_requested = env;
92 env->kvm_cpu_state.stopped = 1;
93 }
94 return handle;
95 }
96 #endif
97
98 static int kvm_inb(void *opaque, uint16_t addr, uint8_t *data)
99 {
100 *data = cpu_inb(0, addr);
101 return 0;
102 }
103
104 static int kvm_inw(void *opaque, uint16_t addr, uint16_t *data)
105 {
106 *data = cpu_inw(0, addr);
107 return 0;
108 }
109
110 static int kvm_inl(void *opaque, uint16_t addr, uint32_t *data)
111 {
112 *data = cpu_inl(0, addr);
113 return 0;
114 }
115
116 #define PM_IO_BASE 0xb000
117
118 static int kvm_outb(void *opaque, uint16_t addr, uint8_t data)
119 {
120 if (addr == 0xb2) {
121 switch (data) {
122 case 0: {
123 cpu_outb(0, 0xb3, 0);
124 break;
125 }
126 case 0xf0: {
127 unsigned x;
128
129 /* enable acpi */
130 x = cpu_inw(0, PM_IO_BASE + 4);
131 x &= ~1;
132 cpu_outw(0, PM_IO_BASE + 4, x);
133 break;
134 }
135 case 0xf1: {
136 unsigned x;
137
138 /* enable acpi */
139 x = cpu_inw(0, PM_IO_BASE + 4);
140 x |= 1;
141 cpu_outw(0, PM_IO_BASE + 4, x);
142 break;
143 }
144 default:
145 break;
146 }
147 return 0;
148 }
149 cpu_outb(0, addr, data);
150 return 0;
151 }
152
153 static int kvm_outw(void *opaque, uint16_t addr, uint16_t data)
154 {
155 cpu_outw(0, addr, data);
156 return 0;
157 }
158
159 static int kvm_outl(void *opaque, uint16_t addr, uint32_t data)
160 {
161 cpu_outl(0, addr, data);
162 return 0;
163 }
164
165 int kvm_mmio_read(void *opaque, uint64_t addr, uint8_t *data, int len)
166 {
167 cpu_physical_memory_rw(addr, data, len, 0);
168 return 0;
169 }
170
171 int kvm_mmio_write(void *opaque, uint64_t addr, uint8_t *data, int len)
172 {
173 cpu_physical_memory_rw(addr, data, len, 1);
174 return 0;
175 }
176
177 static int handle_unhandled(kvm_context_t kvm, kvm_vcpu_context_t vcpu,
178 uint64_t reason)
179 {
180 fprintf(stderr, "kvm: unhandled exit %"PRIx64"\n", reason);
181 return -EINVAL;
182 }
183
184
185 static inline void set_gsi(kvm_context_t kvm, unsigned int gsi)
186 {
187 uint32_t *bitmap = kvm->used_gsi_bitmap;
188
189 if (gsi < kvm->max_gsi)
190 bitmap[gsi / 32] |= 1U << (gsi % 32);
191 else
192 DPRINTF("Invalid GSI %d\n");
193 }
194
195 static inline void clear_gsi(kvm_context_t kvm, unsigned int gsi)
196 {
197 uint32_t *bitmap = kvm->used_gsi_bitmap;
198
199 if (gsi < kvm->max_gsi)
200 bitmap[gsi / 32] &= ~(1U << (gsi % 32));
201 else
202 DPRINTF("Invalid GSI %d\n");
203 }
204
205 struct slot_info {
206 unsigned long phys_addr;
207 unsigned long len;
208 unsigned long userspace_addr;
209 unsigned flags;
210 int logging_count;
211 };
212
213 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
214
215 static void init_slots(void)
216 {
217 int i;
218
219 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
220 slots[i].len = 0;
221 }
222
223 static int get_free_slot(kvm_context_t kvm)
224 {
225 int i;
226 int tss_ext;
227
228 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
229 tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
230 #else
231 tss_ext = 0;
232 #endif
233
234 /*
235 * on older kernels where the set tss ioctl is not supprted we must save
236 * slot 0 to hold the extended memory, as the vmx will use the last 3
237 * pages of this slot.
238 */
239 if (tss_ext > 0)
240 i = 0;
241 else
242 i = 1;
243
244 for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
245 if (!slots[i].len)
246 return i;
247 return -1;
248 }
249
250 static void register_slot(int slot, unsigned long phys_addr, unsigned long len,
251 unsigned long userspace_addr, unsigned flags)
252 {
253 slots[slot].phys_addr = phys_addr;
254 slots[slot].len = len;
255 slots[slot].userspace_addr = userspace_addr;
256 slots[slot].flags = flags;
257 }
258
259 static void free_slot(int slot)
260 {
261 slots[slot].len = 0;
262 slots[slot].logging_count = 0;
263 }
264
265 static int get_slot(unsigned long phys_addr)
266 {
267 int i;
268
269 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
270 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
271 (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
272 return i;
273 }
274 return -1;
275 }
276
277 /* Returns -1 if this slot is not totally contained on any other,
278 * and the number of the slot otherwise */
279 static int get_container_slot(uint64_t phys_addr, unsigned long size)
280 {
281 int i;
282
283 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
284 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
285 (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
286 return i;
287 return -1;
288 }
289
290 int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr, unsigned long size)
291 {
292 int slot = get_container_slot(phys_addr, size);
293 if (slot == -1)
294 return 0;
295 return 1;
296 }
297
298 /*
299 * dirty pages logging control
300 */
301 static int kvm_dirty_pages_log_change(kvm_context_t kvm,
302 unsigned long phys_addr,
303 unsigned flags,
304 unsigned mask)
305 {
306 int r = -1;
307 int slot = get_slot(phys_addr);
308
309 if (slot == -1) {
310 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
311 return 1;
312 }
313
314 flags = (slots[slot].flags & ~mask) | flags;
315 if (flags == slots[slot].flags)
316 return 0;
317 slots[slot].flags = flags;
318
319 {
320 struct kvm_userspace_memory_region mem = {
321 .slot = slot,
322 .memory_size = slots[slot].len,
323 .guest_phys_addr = slots[slot].phys_addr,
324 .userspace_addr = slots[slot].userspace_addr,
325 .flags = slots[slot].flags,
326 };
327
328
329 DPRINTF("slot %d start %llx len %llx flags %x\n",
330 mem.slot,
331 mem.guest_phys_addr,
332 mem.memory_size,
333 mem.flags);
334 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
335 if (r == -1)
336 fprintf(stderr, "%s: %m\n", __FUNCTION__);
337 }
338 return r;
339 }
340
341 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
342 int (*change)(kvm_context_t kvm,
343 uint64_t start,
344 uint64_t len))
345 {
346 int i, r;
347
348 for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
349 if (slots[i].len)
350 r = change(kvm, slots[i].phys_addr, slots[i].len);
351 }
352 return r;
353 }
354
355 int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm,
356 uint64_t phys_addr,
357 uint64_t len)
358 {
359 int slot = get_slot(phys_addr);
360
361 DPRINTF("start %"PRIx64" len %"PRIx64"\n", phys_addr, len);
362 if (slot == -1) {
363 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
364 return -EINVAL;
365 }
366
367 if (slots[slot].logging_count++)
368 return 0;
369
370 return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
371 KVM_MEM_LOG_DIRTY_PAGES,
372 KVM_MEM_LOG_DIRTY_PAGES);
373 }
374
375 int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm,
376 uint64_t phys_addr,
377 uint64_t len)
378 {
379 int slot = get_slot(phys_addr);
380
381 if (slot == -1) {
382 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
383 return -EINVAL;
384 }
385
386 if (--slots[slot].logging_count)
387 return 0;
388
389 return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
390 0,
391 KVM_MEM_LOG_DIRTY_PAGES);
392 }
393
394 /**
395 * Enable dirty page logging for all memory regions
396 */
397 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
398 {
399 if (kvm->dirty_pages_log_all)
400 return 0;
401 kvm->dirty_pages_log_all = 1;
402 return kvm_dirty_pages_log_change_all(kvm,
403 kvm_dirty_pages_log_enable_slot);
404 }
405
406 /**
407 * Enable dirty page logging only for memory regions that were created with
408 * dirty logging enabled (disable for all other memory regions).
409 */
410 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
411 {
412 if (!kvm->dirty_pages_log_all)
413 return 0;
414 kvm->dirty_pages_log_all = 0;
415 return kvm_dirty_pages_log_change_all(kvm,
416 kvm_dirty_pages_log_disable_slot);
417 }
418
419
420 kvm_context_t kvm_init(void *opaque)
421 {
422 int fd;
423 kvm_context_t kvm;
424 int r, gsi_count;
425
426 fd = open("/dev/kvm", O_RDWR);
427 if (fd == -1) {
428 perror("open /dev/kvm");
429 return NULL;
430 }
431 r = ioctl(fd, KVM_GET_API_VERSION, 0);
432 if (r == -1) {
433 fprintf(stderr, "kvm kernel version too old: "
434 "KVM_GET_API_VERSION ioctl not supported\n");
435 goto out_close;
436 }
437 if (r < EXPECTED_KVM_API_VERSION) {
438 fprintf(stderr, "kvm kernel version too old: "
439 "We expect API version %d or newer, but got "
440 "version %d\n",
441 EXPECTED_KVM_API_VERSION, r);
442 goto out_close;
443 }
444 if (r > EXPECTED_KVM_API_VERSION) {
445 fprintf(stderr, "kvm userspace version too old\n");
446 goto out_close;
447 }
448 kvm_abi = r;
449 kvm_page_size = getpagesize();
450 kvm = malloc(sizeof(*kvm));
451 if (kvm == NULL)
452 goto out_close;
453 memset(kvm, 0, sizeof(*kvm));
454 kvm->fd = fd;
455 kvm->vm_fd = -1;
456 kvm->opaque = opaque;
457 kvm->dirty_pages_log_all = 0;
458 kvm->no_irqchip_creation = 0;
459 kvm->no_pit_creation = 0;
460
461 gsi_count = kvm_get_gsi_count(kvm);
462 if (gsi_count > 0) {
463 int gsi_bits, i;
464
465 /* Round up so we can search ints using ffs */
466 gsi_bits = ALIGN(gsi_count, 32);
467 kvm->used_gsi_bitmap = malloc(gsi_bits / 8);
468 if (!kvm->used_gsi_bitmap)
469 goto out_close;
470 memset(kvm->used_gsi_bitmap, 0, gsi_bits / 8);
471 kvm->max_gsi = gsi_bits;
472
473 /* Mark any over-allocated bits as already in use */
474 for (i = gsi_count; i < gsi_bits; i++)
475 set_gsi(kvm, i);
476 }
477
478 return kvm;
479 out_close:
480 close(fd);
481 return NULL;
482 }
483
484 void kvm_finalize(kvm_context_t kvm)
485 {
486 /* FIXME
487 if (kvm->vcpu_fd[0] != -1)
488 close(kvm->vcpu_fd[0]);
489 if (kvm->vm_fd != -1)
490 close(kvm->vm_fd);
491 */
492 close(kvm->fd);
493 free(kvm);
494 }
495
496 void kvm_disable_irqchip_creation(kvm_context_t kvm)
497 {
498 kvm->no_irqchip_creation = 1;
499 }
500
501 void kvm_disable_pit_creation(kvm_context_t kvm)
502 {
503 kvm->no_pit_creation = 1;
504 }
505
506 kvm_vcpu_context_t kvm_create_vcpu(kvm_context_t kvm, int id)
507 {
508 long mmap_size;
509 int r;
510 kvm_vcpu_context_t vcpu_ctx = malloc(sizeof(struct kvm_vcpu_context));
511
512 if (!vcpu_ctx) {
513 errno = ENOMEM;
514 return NULL;
515 }
516
517 vcpu_ctx->kvm = kvm;
518 vcpu_ctx->id = id;
519
520 r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, id);
521 if (r == -1) {
522 fprintf(stderr, "kvm_create_vcpu: %m\n");
523 goto err;
524 }
525 vcpu_ctx->fd = r;
526 mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
527 if (mmap_size == -1) {
528 fprintf(stderr, "get vcpu mmap size: %m\n");
529 goto err_fd;
530 }
531 vcpu_ctx->run = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
532 vcpu_ctx->fd, 0);
533 if (vcpu_ctx->run == MAP_FAILED) {
534 fprintf(stderr, "mmap vcpu area: %m\n");
535 goto err_fd;
536 }
537 return vcpu_ctx;
538 err_fd:
539 close(vcpu_ctx->fd);
540 err:
541 free(vcpu_ctx);
542 return NULL;
543 }
544
545 static int kvm_set_boot_vcpu_id(kvm_context_t kvm, uint32_t id)
546 {
547 #ifdef KVM_CAP_SET_BOOT_CPU_ID
548 int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_BOOT_CPU_ID);
549 if (r > 0)
550 return ioctl(kvm->vm_fd, KVM_SET_BOOT_CPU_ID, id);
551 return -ENOSYS;
552 #else
553 return -ENOSYS;
554 #endif
555 }
556
557 int kvm_create_vm(kvm_context_t kvm)
558 {
559 int fd = kvm->fd;
560
561 #ifdef KVM_CAP_IRQ_ROUTING
562 kvm->irq_routes = malloc(sizeof(*kvm->irq_routes));
563 if (!kvm->irq_routes)
564 return -ENOMEM;
565 memset(kvm->irq_routes, 0, sizeof(*kvm->irq_routes));
566 kvm->nr_allocated_irq_routes = 0;
567 #endif
568
569 fd = ioctl(fd, KVM_CREATE_VM, 0);
570 if (fd == -1) {
571 fprintf(stderr, "kvm_create_vm: %m\n");
572 return -1;
573 }
574 kvm->vm_fd = fd;
575 return 0;
576 }
577
578 static int kvm_create_default_phys_mem(kvm_context_t kvm,
579 unsigned long phys_mem_bytes,
580 void **vm_mem)
581 {
582 #ifdef KVM_CAP_USER_MEMORY
583 int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
584 if (r > 0)
585 return 0;
586 fprintf(stderr, "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
587 #else
588 #error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
589 #endif
590 return -1;
591 }
592
593 int kvm_check_extension(kvm_context_t kvm, int ext)
594 {
595 int ret;
596
597 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
598 if (ret > 0)
599 return ret;
600 return 0;
601 }
602
603 void kvm_create_irqchip(kvm_context_t kvm)
604 {
605 int r;
606
607 kvm->irqchip_in_kernel = 0;
608 #ifdef KVM_CAP_IRQCHIP
609 if (!kvm->no_irqchip_creation) {
610 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
611 if (r > 0) { /* kernel irqchip supported */
612 r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
613 if (r >= 0) {
614 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE;
615 #if defined(KVM_CAP_IRQ_INJECT_STATUS) && defined(KVM_IRQ_LINE_STATUS)
616 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
617 KVM_CAP_IRQ_INJECT_STATUS);
618 if (r > 0)
619 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
620 #endif
621 kvm->irqchip_in_kernel = 1;
622 }
623 else
624 fprintf(stderr, "Create kernel PIC irqchip failed\n");
625 }
626 }
627 #endif
628 }
629
630 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
631 {
632 int r;
633
634 r = kvm_create_vm(kvm);
635 if (r < 0)
636 return r;
637 r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
638 if (r < 0)
639 return r;
640 init_slots();
641 r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
642 if (r < 0)
643 return r;
644 kvm_create_irqchip(kvm);
645
646 return 0;
647 }
648
649
650 int kvm_register_phys_mem(kvm_context_t kvm,
651 unsigned long phys_start, void *userspace_addr,
652 unsigned long len, int log)
653 {
654
655 struct kvm_userspace_memory_region memory = {
656 .memory_size = len,
657 .guest_phys_addr = phys_start,
658 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
659 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
660 };
661 int r;
662
663 memory.slot = get_free_slot(kvm);
664 DPRINTF("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %lx\n",
665 memory.guest_phys_addr, memory.memory_size,
666 memory.userspace_addr, memory.slot, memory.flags);
667 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
668 if (r == -1) {
669 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
670 return -1;
671 }
672 register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
673 memory.userspace_addr, memory.flags);
674 return 0;
675 }
676
677
678 /* destroy/free a whole slot.
679 * phys_start, len and slot are the params passed to kvm_create_phys_mem()
680 */
681 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
682 unsigned long len)
683 {
684 int slot;
685 int r;
686 struct kvm_userspace_memory_region memory = {
687 .memory_size = 0,
688 .guest_phys_addr = phys_start,
689 .userspace_addr = 0,
690 .flags = 0,
691 };
692
693 slot = get_slot(phys_start);
694
695 if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
696 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
697 __FUNCTION__, slot);
698 return;
699 }
700 if (phys_start != slots[slot].phys_addr) {
701 fprintf(stderr,
702 "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
703 __FUNCTION__, phys_start, slots[slot].phys_addr);
704 phys_start = slots[slot].phys_addr;
705 }
706
707 memory.slot = slot;
708 DPRINTF("slot %d start %llx len %llx flags %x\n",
709 memory.slot,
710 memory.guest_phys_addr,
711 memory.memory_size,
712 memory.flags);
713 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
714 if (r == -1) {
715 fprintf(stderr, "destroy_userspace_phys_mem: %s",
716 strerror(errno));
717 return;
718 }
719
720 free_slot(memory.slot);
721 }
722
723 void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr, unsigned long size)
724 {
725
726 int slot = get_container_slot(phys_addr, size);
727
728 if (slot != -1) {
729 DPRINTF("Unregistering memory region %llx (%lx)\n", phys_addr, size);
730 kvm_destroy_phys_mem(kvm, phys_addr, size);
731 return;
732 }
733 }
734
735 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
736 {
737 int r;
738 struct kvm_dirty_log log = {
739 .slot = slot,
740 };
741
742 log.dirty_bitmap = buf;
743
744 r = ioctl(kvm->vm_fd, ioctl_num, &log);
745 if (r == -1)
746 return -errno;
747 return 0;
748 }
749
750 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
751 {
752 int slot;
753
754 slot = get_slot(phys_addr);
755 return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
756 }
757
758 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
759 unsigned long len, void *opaque,
760 int (*cb)(unsigned long start, unsigned long len,
761 void*bitmap, void *opaque))
762 {
763 int i;
764 int r;
765 unsigned long end_addr = phys_addr + len;
766 void *buf;
767
768 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
769 if ((slots[i].len && (uint64_t)slots[i].phys_addr >= phys_addr)
770 && ((uint64_t)slots[i].phys_addr + slots[i].len <= end_addr)) {
771 buf = qemu_malloc((slots[i].len / 4096 + 7) / 8 + 2);
772 r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
773 if (r) {
774 qemu_free(buf);
775 return r;
776 }
777 r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
778 qemu_free(buf);
779 if (r)
780 return r;
781 }
782 }
783 return 0;
784 }
785
786 #ifdef KVM_CAP_IRQCHIP
787
788 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level, int *status)
789 {
790 struct kvm_irq_level event;
791 int r;
792
793 if (!kvm->irqchip_in_kernel)
794 return 0;
795 event.level = level;
796 event.irq = irq;
797 r = ioctl(kvm->vm_fd, kvm->irqchip_inject_ioctl, &event);
798 if (r == -1)
799 perror("kvm_set_irq_level");
800
801 if (status) {
802 #ifdef KVM_CAP_IRQ_INJECT_STATUS
803 *status = (kvm->irqchip_inject_ioctl == KVM_IRQ_LINE) ?
804 1 : event.status;
805 #else
806 *status = 1;
807 #endif
808 }
809
810 return 1;
811 }
812
813 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
814 {
815 int r;
816
817 if (!kvm->irqchip_in_kernel)
818 return 0;
819 r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
820 if (r == -1) {
821 r = -errno;
822 perror("kvm_get_irqchip\n");
823 }
824 return r;
825 }
826
827 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
828 {
829 int r;
830
831 if (!kvm->irqchip_in_kernel)
832 return 0;
833 r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
834 if (r == -1) {
835 r = -errno;
836 perror("kvm_set_irqchip\n");
837 }
838 return r;
839 }
840
841 #endif
842
843 static int handle_io(kvm_vcpu_context_t vcpu)
844 {
845 struct kvm_run *run = vcpu->run;
846 kvm_context_t kvm = vcpu->kvm;
847 uint16_t addr = run->io.port;
848 int r;
849 int i;
850 void *p = (void *)run + run->io.data_offset;
851
852 for (i = 0; i < run->io.count; ++i) {
853 switch (run->io.direction) {
854 case KVM_EXIT_IO_IN:
855 switch (run->io.size) {
856 case 1:
857 r = kvm_inb(kvm->opaque, addr, p);
858 break;
859 case 2:
860 r = kvm_inw(kvm->opaque, addr, p);
861 break;
862 case 4:
863 r = kvm_inl(kvm->opaque, addr, p);
864 break;
865 default:
866 fprintf(stderr, "bad I/O size %d\n", run->io.size);
867 return -EMSGSIZE;
868 }
869 break;
870 case KVM_EXIT_IO_OUT:
871 switch (run->io.size) {
872 case 1:
873 r = kvm_outb(kvm->opaque, addr,
874 *(uint8_t *)p);
875 break;
876 case 2:
877 r = kvm_outw(kvm->opaque, addr,
878 *(uint16_t *)p);
879 break;
880 case 4:
881 r = kvm_outl(kvm->opaque, addr,
882 *(uint32_t *)p);
883 break;
884 default:
885 fprintf(stderr, "bad I/O size %d\n", run->io.size);
886 return -EMSGSIZE;
887 }
888 break;
889 default:
890 fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
891 return -EPROTO;
892 }
893
894 p += run->io.size;
895 }
896
897 return 0;
898 }
899
900 int handle_debug(kvm_vcpu_context_t vcpu, void *env)
901 {
902 #ifdef KVM_CAP_SET_GUEST_DEBUG
903 struct kvm_run *run = vcpu->run;
904 kvm_context_t kvm = vcpu->kvm;
905
906 return kvm_debug(kvm->opaque, env, &run->debug.arch);
907 #else
908 return 0;
909 #endif
910 }
911
912 int kvm_get_regs(kvm_vcpu_context_t vcpu, struct kvm_regs *regs)
913 {
914 return ioctl(vcpu->fd, KVM_GET_REGS, regs);
915 }
916
917 int kvm_set_regs(kvm_vcpu_context_t vcpu, struct kvm_regs *regs)
918 {
919 return ioctl(vcpu->fd, KVM_SET_REGS, regs);
920 }
921
922 int kvm_get_fpu(kvm_vcpu_context_t vcpu, struct kvm_fpu *fpu)
923 {
924 return ioctl(vcpu->fd, KVM_GET_FPU, fpu);
925 }
926
927 int kvm_set_fpu(kvm_vcpu_context_t vcpu, struct kvm_fpu *fpu)
928 {
929 return ioctl(vcpu->fd, KVM_SET_FPU, fpu);
930 }
931
932 int kvm_get_sregs(kvm_vcpu_context_t vcpu, struct kvm_sregs *sregs)
933 {
934 return ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
935 }
936
937 int kvm_set_sregs(kvm_vcpu_context_t vcpu, struct kvm_sregs *sregs)
938 {
939 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
940 }
941
942 #ifdef KVM_CAP_MP_STATE
943 int kvm_get_mpstate(kvm_vcpu_context_t vcpu, struct kvm_mp_state *mp_state)
944 {
945 int r;
946
947 r = ioctl(vcpu->kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
948 if (r > 0)
949 return ioctl(vcpu->fd, KVM_GET_MP_STATE, mp_state);
950 return -ENOSYS;
951 }
952
953 int kvm_set_mpstate(kvm_vcpu_context_t vcpu, struct kvm_mp_state *mp_state)
954 {
955 int r;
956
957 r = ioctl(vcpu->kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
958 if (r > 0)
959 return ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
960 return -ENOSYS;
961 }
962 #endif
963
964 static int handle_mmio(kvm_vcpu_context_t vcpu)
965 {
966 unsigned long addr = vcpu->run->mmio.phys_addr;
967 kvm_context_t kvm = vcpu->kvm;
968 struct kvm_run *kvm_run = vcpu->run;
969 void *data = kvm_run->mmio.data;
970
971 /* hack: Red Hat 7.1 generates these weird accesses. */
972 if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
973 return 0;
974
975 if (kvm_run->mmio.is_write)
976 return kvm_mmio_write(kvm->opaque, addr, data,
977 kvm_run->mmio.len);
978 else
979 return kvm_mmio_read(kvm->opaque, addr, data,
980 kvm_run->mmio.len);
981 }
982
983 int handle_io_window(kvm_context_t kvm)
984 {
985 return 1;
986 }
987
988 int handle_halt(kvm_vcpu_context_t vcpu)
989 {
990 return kvm_arch_halt(vcpu->kvm->opaque, vcpu);
991 }
992
993 int handle_shutdown(kvm_context_t kvm, CPUState *env)
994 {
995 /* stop the current vcpu from going back to guest mode */
996 env->kvm_cpu_state.stopped = 1;
997
998 qemu_system_reset_request();
999 return 1;
1000 }
1001
1002 static inline void push_nmi(kvm_context_t kvm)
1003 {
1004 #ifdef KVM_CAP_USER_NMI
1005 kvm_arch_push_nmi(kvm->opaque);
1006 #endif /* KVM_CAP_USER_NMI */
1007 }
1008
1009 void post_kvm_run(kvm_context_t kvm, CPUState *env)
1010 {
1011 pthread_mutex_lock(&qemu_mutex);
1012 kvm_arch_post_kvm_run(kvm->opaque, env);
1013 }
1014
1015 int pre_kvm_run(kvm_context_t kvm, CPUState *env)
1016 {
1017 kvm_arch_pre_kvm_run(kvm->opaque, env);
1018
1019 if (env->exit_request)
1020 return 1;
1021 pthread_mutex_unlock(&qemu_mutex);
1022 return 0;
1023 }
1024
1025 int kvm_get_interrupt_flag(kvm_vcpu_context_t vcpu)
1026 {
1027 return vcpu->run->if_flag;
1028 }
1029
1030 int kvm_is_ready_for_interrupt_injection(kvm_vcpu_context_t vcpu)
1031 {
1032 return vcpu->run->ready_for_interrupt_injection;
1033 }
1034
1035 int kvm_run(kvm_vcpu_context_t vcpu, void *env)
1036 {
1037 int r;
1038 int fd = vcpu->fd;
1039 struct kvm_run *run = vcpu->run;
1040 kvm_context_t kvm = vcpu->kvm;
1041
1042 again:
1043 push_nmi(kvm);
1044 #if !defined(__s390__)
1045 if (!kvm->irqchip_in_kernel)
1046 run->request_interrupt_window = kvm_arch_try_push_interrupts(env);
1047 #endif
1048 r = pre_kvm_run(kvm, env);
1049 if (r)
1050 return r;
1051 r = ioctl(fd, KVM_RUN, 0);
1052
1053 if (r == -1 && errno != EINTR && errno != EAGAIN) {
1054 r = -errno;
1055 post_kvm_run(kvm, env);
1056 fprintf(stderr, "kvm_run: %s\n", strerror(-r));
1057 return r;
1058 }
1059
1060 post_kvm_run(kvm, env);
1061
1062 #if defined(KVM_CAP_COALESCED_MMIO)
1063 if (kvm->coalesced_mmio) {
1064 struct kvm_coalesced_mmio_ring *ring = (void *)run +
1065 kvm->coalesced_mmio * PAGE_SIZE;
1066 while (ring->first != ring->last) {
1067 kvm_mmio_write(kvm->opaque,
1068 ring->coalesced_mmio[ring->first].phys_addr,
1069 &ring->coalesced_mmio[ring->first].data[0],
1070 ring->coalesced_mmio[ring->first].len);
1071 smp_wmb();
1072 ring->first = (ring->first + 1) %
1073 KVM_COALESCED_MMIO_MAX;
1074 }
1075 }
1076 #endif
1077
1078 #if !defined(__s390__)
1079 if (r == -1) {
1080 r = handle_io_window(kvm);
1081 goto more;
1082 }
1083 #endif
1084 if (1) {
1085 switch (run->exit_reason) {
1086 case KVM_EXIT_UNKNOWN:
1087 r = handle_unhandled(kvm, vcpu,
1088 run->hw.hardware_exit_reason);
1089 break;
1090 case KVM_EXIT_FAIL_ENTRY:
1091 r = handle_unhandled(kvm, vcpu,
1092 run->fail_entry.hardware_entry_failure_reason);
1093 break;
1094 case KVM_EXIT_EXCEPTION:
1095 fprintf(stderr, "exception %d (%x)\n",
1096 run->ex.exception,
1097 run->ex.error_code);
1098 kvm_show_regs(vcpu);
1099 kvm_show_code(vcpu);
1100 abort();
1101 break;
1102 case KVM_EXIT_IO:
1103 r = handle_io(vcpu);
1104 break;
1105 case KVM_EXIT_DEBUG:
1106 r = handle_debug(vcpu, env);
1107 break;
1108 case KVM_EXIT_MMIO:
1109 r = handle_mmio(vcpu);
1110 break;
1111 case KVM_EXIT_HLT:
1112 r = handle_halt(vcpu);
1113 break;
1114 case KVM_EXIT_IRQ_WINDOW_OPEN:
1115 break;
1116 case KVM_EXIT_SHUTDOWN:
1117 r = handle_shutdown(kvm, env);
1118 break;
1119 #if defined(__s390__)
1120 case KVM_EXIT_S390_SIEIC:
1121 r = kvm_s390_handle_intercept(kvm, vcpu,
1122 run);
1123 break;
1124 case KVM_EXIT_S390_RESET:
1125 r = kvm_s390_handle_reset(kvm, vcpu, run);
1126 break;
1127 #endif
1128 default:
1129 if (kvm_arch_run(vcpu)) {
1130 fprintf(stderr, "unhandled vm exit: 0x%x\n",
1131 run->exit_reason);
1132 kvm_show_regs(vcpu);
1133 abort();
1134 }
1135 break;
1136 }
1137 }
1138 more:
1139 if (!r)
1140 goto again;
1141 return r;
1142 }
1143
1144 int kvm_inject_irq(kvm_vcpu_context_t vcpu, unsigned irq)
1145 {
1146 struct kvm_interrupt intr;
1147
1148 intr.irq = irq;
1149 return ioctl(vcpu->fd, KVM_INTERRUPT, &intr);
1150 }
1151
1152 #ifdef KVM_CAP_SET_GUEST_DEBUG
1153 int kvm_set_guest_debug(kvm_vcpu_context_t vcpu, struct kvm_guest_debug *dbg)
1154 {
1155 return ioctl(vcpu->fd, KVM_SET_GUEST_DEBUG, dbg);
1156 }
1157 #endif
1158
1159 int kvm_set_signal_mask(kvm_vcpu_context_t vcpu, const sigset_t *sigset)
1160 {
1161 struct kvm_signal_mask *sigmask;
1162 int r;
1163
1164 if (!sigset) {
1165 r = ioctl(vcpu->fd, KVM_SET_SIGNAL_MASK, NULL);
1166 if (r == -1)
1167 r = -errno;
1168 return r;
1169 }
1170 sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
1171 if (!sigmask)
1172 return -ENOMEM;
1173
1174 sigmask->len = 8;
1175 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1176 r = ioctl(vcpu->fd, KVM_SET_SIGNAL_MASK, sigmask);
1177 if (r == -1)
1178 r = -errno;
1179 free(sigmask);
1180 return r;
1181 }
1182
1183 int kvm_irqchip_in_kernel(kvm_context_t kvm)
1184 {
1185 return kvm->irqchip_in_kernel;
1186 }
1187
1188 int kvm_pit_in_kernel(kvm_context_t kvm)
1189 {
1190 return kvm->pit_in_kernel;
1191 }
1192
1193 int kvm_has_sync_mmu(void)
1194 {
1195 int r = 0;
1196 #ifdef KVM_CAP_SYNC_MMU
1197 r = ioctl(kvm_context->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
1198 #endif
1199 return r;
1200 }
1201
1202 int kvm_inject_nmi(kvm_vcpu_context_t vcpu)
1203 {
1204 #ifdef KVM_CAP_USER_NMI
1205 return ioctl(vcpu->fd, KVM_NMI);
1206 #else
1207 return -ENOSYS;
1208 #endif
1209 }
1210
1211 int kvm_init_coalesced_mmio(kvm_context_t kvm)
1212 {
1213 int r = 0;
1214 kvm->coalesced_mmio = 0;
1215 #ifdef KVM_CAP_COALESCED_MMIO
1216 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
1217 if (r > 0) {
1218 kvm->coalesced_mmio = r;
1219 return 0;
1220 }
1221 #endif
1222 return r;
1223 }
1224
1225 int kvm_coalesce_mmio_region(target_phys_addr_t addr, ram_addr_t size)
1226 {
1227 #ifdef KVM_CAP_COALESCED_MMIO
1228 kvm_context_t kvm = kvm_context;
1229 struct kvm_coalesced_mmio_zone zone;
1230 int r;
1231
1232 if (kvm->coalesced_mmio) {
1233
1234 zone.addr = addr;
1235 zone.size = size;
1236
1237 r = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
1238 if (r == -1) {
1239 perror("kvm_register_coalesced_mmio_zone");
1240 return -errno;
1241 }
1242 return 0;
1243 }
1244 #endif
1245 return -ENOSYS;
1246 }
1247
1248 int kvm_uncoalesce_mmio_region(target_phys_addr_t addr, ram_addr_t size)
1249 {
1250 #ifdef KVM_CAP_COALESCED_MMIO
1251 kvm_context_t kvm = kvm_context;
1252 struct kvm_coalesced_mmio_zone zone;
1253 int r;
1254
1255 if (kvm->coalesced_mmio) {
1256
1257 zone.addr = addr;
1258 zone.size = size;
1259
1260 r = ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1261 if (r == -1) {
1262 perror("kvm_unregister_coalesced_mmio_zone");
1263 return -errno;
1264 }
1265 DPRINTF("Unregistered coalesced mmio region for %llx (%lx)\n", addr, size);
1266 return 0;
1267 }
1268 #endif
1269 return -ENOSYS;
1270 }
1271
1272 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1273 int kvm_assign_pci_device(kvm_context_t kvm,
1274 struct kvm_assigned_pci_dev *assigned_dev)
1275 {
1276 int ret;
1277
1278 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
1279 if (ret < 0)
1280 return -errno;
1281
1282 return ret;
1283 }
1284
1285 static int kvm_old_assign_irq(kvm_context_t kvm,
1286 struct kvm_assigned_irq *assigned_irq)
1287 {
1288 int ret;
1289
1290 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_IRQ, assigned_irq);
1291 if (ret < 0)
1292 return -errno;
1293
1294 return ret;
1295 }
1296
1297 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
1298 int kvm_assign_irq(kvm_context_t kvm,
1299 struct kvm_assigned_irq *assigned_irq)
1300 {
1301 int ret;
1302
1303 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_ASSIGN_DEV_IRQ);
1304 if (ret > 0) {
1305 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_DEV_IRQ, assigned_irq);
1306 if (ret < 0)
1307 return -errno;
1308 return ret;
1309 }
1310
1311 return kvm_old_assign_irq(kvm, assigned_irq);
1312 }
1313
1314 int kvm_deassign_irq(kvm_context_t kvm,
1315 struct kvm_assigned_irq *assigned_irq)
1316 {
1317 int ret;
1318
1319 ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_DEV_IRQ, assigned_irq);
1320 if (ret < 0)
1321 return -errno;
1322
1323 return ret;
1324 }
1325 #else
1326 int kvm_assign_irq(kvm_context_t kvm,
1327 struct kvm_assigned_irq *assigned_irq)
1328 {
1329 return kvm_old_assign_irq(kvm, assigned_irq);
1330 }
1331 #endif
1332 #endif
1333
1334 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
1335 int kvm_deassign_pci_device(kvm_context_t kvm,
1336 struct kvm_assigned_pci_dev *assigned_dev)
1337 {
1338 int ret;
1339
1340 ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_PCI_DEVICE, assigned_dev);
1341 if (ret < 0)
1342 return -errno;
1343
1344 return ret;
1345 }
1346 #endif
1347
1348 int kvm_destroy_memory_region_works(kvm_context_t kvm)
1349 {
1350 int ret = 0;
1351
1352 #ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
1353 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
1354 KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
1355 if (ret <= 0)
1356 ret = 0;
1357 #endif
1358 return ret;
1359 }
1360
1361 int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
1362 {
1363 #ifdef KVM_CAP_REINJECT_CONTROL
1364 int r;
1365 struct kvm_reinject_control control;
1366
1367 control.pit_reinject = pit_reinject;
1368
1369 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_REINJECT_CONTROL);
1370 if (r > 0) {
1371 r = ioctl(kvm->vm_fd, KVM_REINJECT_CONTROL, &control);
1372 if (r == -1)
1373 return -errno;
1374 return r;
1375 }
1376 #endif
1377 return -ENOSYS;
1378 }
1379
1380 int kvm_has_gsi_routing(kvm_context_t kvm)
1381 {
1382 int r = 0;
1383
1384 #ifdef KVM_CAP_IRQ_ROUTING
1385 r = kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1386 #endif
1387 return r;
1388 }
1389
1390 int kvm_get_gsi_count(kvm_context_t kvm)
1391 {
1392 #ifdef KVM_CAP_IRQ_ROUTING
1393 return kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1394 #else
1395 return -EINVAL;
1396 #endif
1397 }
1398
1399 int kvm_clear_gsi_routes(kvm_context_t kvm)
1400 {
1401 #ifdef KVM_CAP_IRQ_ROUTING
1402 kvm->irq_routes->nr = 0;
1403 return 0;
1404 #else
1405 return -EINVAL;
1406 #endif
1407 }
1408
1409 int kvm_add_routing_entry(kvm_context_t kvm,
1410 struct kvm_irq_routing_entry* entry)
1411 {
1412 #ifdef KVM_CAP_IRQ_ROUTING
1413 struct kvm_irq_routing *z;
1414 struct kvm_irq_routing_entry *new;
1415 int n, size;
1416
1417 if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
1418 n = kvm->nr_allocated_irq_routes * 2;
1419 if (n < 64)
1420 n = 64;
1421 size = sizeof(struct kvm_irq_routing);
1422 size += n * sizeof(*new);
1423 z = realloc(kvm->irq_routes, size);
1424 if (!z)
1425 return -ENOMEM;
1426 kvm->nr_allocated_irq_routes = n;
1427 kvm->irq_routes = z;
1428 }
1429 n = kvm->irq_routes->nr++;
1430 new = &kvm->irq_routes->entries[n];
1431 memset(new, 0, sizeof(*new));
1432 new->gsi = entry->gsi;
1433 new->type = entry->type;
1434 new->flags = entry->flags;
1435 new->u = entry->u;
1436
1437 set_gsi(kvm, entry->gsi);
1438
1439 return 0;
1440 #else
1441 return -ENOSYS;
1442 #endif
1443 }
1444
1445 int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1446 {
1447 #ifdef KVM_CAP_IRQ_ROUTING
1448 struct kvm_irq_routing_entry e;
1449
1450 e.gsi = gsi;
1451 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1452 e.flags = 0;
1453 e.u.irqchip.irqchip = irqchip;
1454 e.u.irqchip.pin = pin;
1455 return kvm_add_routing_entry(kvm, &e);
1456 #else
1457 return -ENOSYS;
1458 #endif
1459 }
1460
1461 int kvm_del_routing_entry(kvm_context_t kvm,
1462 struct kvm_irq_routing_entry* entry)
1463 {
1464 #ifdef KVM_CAP_IRQ_ROUTING
1465 struct kvm_irq_routing_entry *e, *p;
1466 int i, gsi, found = 0;
1467
1468 gsi = entry->gsi;
1469
1470 for (i = 0; i < kvm->irq_routes->nr; ++i) {
1471 e = &kvm->irq_routes->entries[i];
1472 if (e->type == entry->type
1473 && e->gsi == gsi) {
1474 switch (e->type)
1475 {
1476 case KVM_IRQ_ROUTING_IRQCHIP: {
1477 if (e->u.irqchip.irqchip ==
1478 entry->u.irqchip.irqchip
1479 && e->u.irqchip.pin ==
1480 entry->u.irqchip.pin) {
1481 p = &kvm->irq_routes->
1482 entries[--kvm->irq_routes->nr];
1483 *e = *p;
1484 found = 1;
1485 }
1486 break;
1487 }
1488 case KVM_IRQ_ROUTING_MSI: {
1489 if (e->u.msi.address_lo ==
1490 entry->u.msi.address_lo
1491 && e->u.msi.address_hi ==
1492 entry->u.msi.address_hi
1493 && e->u.msi.data == entry->u.msi.data) {
1494 p = &kvm->irq_routes->
1495 entries[--kvm->irq_routes->nr];
1496 *e = *p;
1497 found = 1;
1498 }
1499 break;
1500 }
1501 default:
1502 break;
1503 }
1504 if (found) {
1505 /* If there are no other users of this GSI
1506 * mark it available in the bitmap */
1507 for (i = 0; i < kvm->irq_routes->nr; i++) {
1508 e = &kvm->irq_routes->entries[i];
1509 if (e->gsi == gsi)
1510 break;
1511 }
1512 if (i == kvm->irq_routes->nr)
1513 clear_gsi(kvm, gsi);
1514
1515 return 0;
1516 }
1517 }
1518 }
1519 return -ESRCH;
1520 #else
1521 return -ENOSYS;
1522 #endif
1523 }
1524
1525 int kvm_update_routing_entry(kvm_context_t kvm,
1526 struct kvm_irq_routing_entry* entry,
1527 struct kvm_irq_routing_entry* newentry)
1528 {
1529 #ifdef KVM_CAP_IRQ_ROUTING
1530 struct kvm_irq_routing_entry *e;
1531 int i;
1532
1533 if (entry->gsi != newentry->gsi ||
1534 entry->type != newentry->type) {
1535 return -EINVAL;
1536 }
1537
1538 for (i = 0; i < kvm->irq_routes->nr; ++i) {
1539 e = &kvm->irq_routes->entries[i];
1540 if (e->type != entry->type || e->gsi != entry->gsi) {
1541 continue;
1542 }
1543 switch (e->type) {
1544 case KVM_IRQ_ROUTING_IRQCHIP:
1545 if (e->u.irqchip.irqchip == entry->u.irqchip.irqchip &&
1546 e->u.irqchip.pin == entry->u.irqchip.pin) {
1547 memcpy(&e->u.irqchip, &entry->u.irqchip, sizeof e->u.irqchip);
1548 return 0;
1549 }
1550 break;
1551 case KVM_IRQ_ROUTING_MSI:
1552 if (e->u.msi.address_lo == entry->u.msi.address_lo &&
1553 e->u.msi.address_hi == entry->u.msi.address_hi &&
1554 e->u.msi.data == entry->u.msi.data) {
1555 memcpy(&e->u.msi, &entry->u.msi, sizeof e->u.msi);
1556 return 0;
1557 }
1558 break;
1559 default:
1560 break;
1561 }
1562 }
1563 return -ESRCH;
1564 #else
1565 return -ENOSYS;
1566 #endif
1567 }
1568
1569 int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1570 {
1571 #ifdef KVM_CAP_IRQ_ROUTING
1572 struct kvm_irq_routing_entry e;
1573
1574 e.gsi = gsi;
1575 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1576 e.flags = 0;
1577 e.u.irqchip.irqchip = irqchip;
1578 e.u.irqchip.pin = pin;
1579 return kvm_del_routing_entry(kvm, &e);
1580 #else
1581 return -ENOSYS;
1582 #endif
1583 }
1584
1585 int kvm_commit_irq_routes(kvm_context_t kvm)
1586 {
1587 #ifdef KVM_CAP_IRQ_ROUTING
1588 int r;
1589
1590 kvm->irq_routes->flags = 0;
1591 r = ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, kvm->irq_routes);
1592 if (r == -1)
1593 r = -errno;
1594 return r;
1595 #else
1596 return -ENOSYS;
1597 #endif
1598 }
1599
1600 int kvm_get_irq_route_gsi(kvm_context_t kvm)
1601 {
1602 int i, bit;
1603 uint32_t *buf = kvm->used_gsi_bitmap;
1604
1605 /* Return the lowest unused GSI in the bitmap */
1606 for (i = 0; i < kvm->max_gsi / 32; i++) {
1607 bit = ffs(~buf[i]);
1608 if (!bit)
1609 continue;
1610
1611 return bit - 1 + i * 32;
1612 }
1613
1614 return -ENOSPC;
1615 }
1616
1617 #ifdef KVM_CAP_DEVICE_MSIX
1618 int kvm_assign_set_msix_nr(kvm_context_t kvm,
1619 struct kvm_assigned_msix_nr *msix_nr)
1620 {
1621 int ret;
1622
1623 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_NR, msix_nr);
1624 if (ret < 0)
1625 return -errno;
1626
1627 return ret;
1628 }
1629
1630 int kvm_assign_set_msix_entry(kvm_context_t kvm,
1631 struct kvm_assigned_msix_entry *entry)
1632 {
1633 int ret;
1634
1635 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_ENTRY, entry);
1636 if (ret < 0)
1637 return -errno;
1638
1639 return ret;
1640 }
1641 #endif
1642
1643 #if defined(KVM_CAP_IRQFD) && defined(CONFIG_eventfd)
1644
1645 #include <sys/eventfd.h>
1646
1647 static int _kvm_irqfd(kvm_context_t kvm, int fd, int gsi, int flags)
1648 {
1649 int r;
1650 struct kvm_irqfd data = {
1651 .fd = fd,
1652 .gsi = gsi,
1653 .flags = flags,
1654 };
1655
1656 r = ioctl(kvm->vm_fd, KVM_IRQFD, &data);
1657 if (r == -1)
1658 r = -errno;
1659 return r;
1660 }
1661
1662 int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
1663 {
1664 int r;
1665 int fd;
1666
1667 if (!kvm_check_extension(kvm, KVM_CAP_IRQFD))
1668 return -ENOENT;
1669
1670 fd = eventfd(0, 0);
1671 if (fd < 0)
1672 return -errno;
1673
1674 r = _kvm_irqfd(kvm, fd, gsi, 0);
1675 if (r < 0) {
1676 close(fd);
1677 return -errno;
1678 }
1679
1680 return fd;
1681 }
1682
1683 #else /* KVM_CAP_IRQFD */
1684
1685 int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
1686 {
1687 return -ENOSYS;
1688 }
1689
1690 #endif /* KVM_CAP_IRQFD */
1691 static inline unsigned long kvm_get_thread_id(void)
1692 {
1693 return syscall(SYS_gettid);
1694 }
1695
1696 static void qemu_cond_wait(pthread_cond_t *cond)
1697 {
1698 CPUState *env = cpu_single_env;
1699 static const struct timespec ts = {
1700 .tv_sec = 0,
1701 .tv_nsec = 100000,
1702 };
1703
1704 pthread_cond_timedwait(cond, &qemu_mutex, &ts);
1705 cpu_single_env = env;
1706 }
1707
1708 static void sig_ipi_handler(int n)
1709 {
1710 }
1711
1712 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
1713 {
1714 struct qemu_work_item wi;
1715
1716 if (env == current_env) {
1717 func(data);
1718 return;
1719 }
1720
1721 wi.func = func;
1722 wi.data = data;
1723 if (!env->kvm_cpu_state.queued_work_first)
1724 env->kvm_cpu_state.queued_work_first = &wi;
1725 else
1726 env->kvm_cpu_state.queued_work_last->next = &wi;
1727 env->kvm_cpu_state.queued_work_last = &wi;
1728 wi.next = NULL;
1729 wi.done = false;
1730
1731 pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
1732 while (!wi.done)
1733 qemu_cond_wait(&qemu_work_cond);
1734 }
1735
1736 static void inject_interrupt(void *data)
1737 {
1738 cpu_interrupt(current_env, (long)data);
1739 }
1740
1741 void kvm_inject_interrupt(CPUState *env, int mask)
1742 {
1743 on_vcpu(env, inject_interrupt, (void *)(long)mask);
1744 }
1745
1746 void kvm_update_interrupt_request(CPUState *env)
1747 {
1748 int signal = 0;
1749
1750 if (env) {
1751 if (!current_env || !current_env->kvm_cpu_state.created)
1752 signal = 1;
1753 /*
1754 * Testing for created here is really redundant
1755 */
1756 if (current_env && current_env->kvm_cpu_state.created &&
1757 env != current_env && !env->kvm_cpu_state.signalled)
1758 signal = 1;
1759
1760 if (signal) {
1761 env->kvm_cpu_state.signalled = 1;
1762 if (env->kvm_cpu_state.thread)
1763 pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
1764 }
1765 }
1766 }
1767
1768 static void kvm_do_load_registers(void *_env)
1769 {
1770 CPUState *env = _env;
1771
1772 kvm_arch_load_regs(env);
1773 }
1774
1775 void kvm_load_registers(CPUState *env)
1776 {
1777 if (kvm_enabled() && qemu_system_ready)
1778 on_vcpu(env, kvm_do_load_registers, env);
1779 }
1780
1781 static void kvm_do_save_registers(void *_env)
1782 {
1783 CPUState *env = _env;
1784
1785 kvm_arch_save_regs(env);
1786 }
1787
1788 void kvm_save_registers(CPUState *env)
1789 {
1790 if (kvm_enabled())
1791 on_vcpu(env, kvm_do_save_registers, env);
1792 }
1793
1794 static void kvm_do_load_mpstate(void *_env)
1795 {
1796 CPUState *env = _env;
1797
1798 kvm_arch_load_mpstate(env);
1799 }
1800
1801 void kvm_load_mpstate(CPUState *env)
1802 {
1803 if (kvm_enabled() && qemu_system_ready)
1804 on_vcpu(env, kvm_do_load_mpstate, env);
1805 }
1806
1807 static void kvm_do_save_mpstate(void *_env)
1808 {
1809 CPUState *env = _env;
1810
1811 kvm_arch_save_mpstate(env);
1812 env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
1813 }
1814
1815 void kvm_save_mpstate(CPUState *env)
1816 {
1817 if (kvm_enabled())
1818 on_vcpu(env, kvm_do_save_mpstate, env);
1819 }
1820
1821 int kvm_cpu_exec(CPUState *env)
1822 {
1823 int r;
1824
1825 r = kvm_run(env->kvm_cpu_state.vcpu_ctx, env);
1826 if (r < 0) {
1827 printf("kvm_run returned %d\n", r);
1828 vm_stop(0);
1829 }
1830
1831 return 0;
1832 }
1833
1834 static int is_cpu_stopped(CPUState *env)
1835 {
1836 return !vm_running || env->kvm_cpu_state.stopped;
1837 }
1838
1839 static void flush_queued_work(CPUState *env)
1840 {
1841 struct qemu_work_item *wi;
1842
1843 if (!env->kvm_cpu_state.queued_work_first)
1844 return;
1845
1846 while ((wi = env->kvm_cpu_state.queued_work_first)) {
1847 env->kvm_cpu_state.queued_work_first = wi->next;
1848 wi->func(wi->data);
1849 wi->done = true;
1850 }
1851 env->kvm_cpu_state.queued_work_last = NULL;
1852 pthread_cond_broadcast(&qemu_work_cond);
1853 }
1854
1855 static void kvm_main_loop_wait(CPUState *env, int timeout)
1856 {
1857 struct timespec ts;
1858 int r, e;
1859 siginfo_t siginfo;
1860 sigset_t waitset;
1861
1862 pthread_mutex_unlock(&qemu_mutex);
1863
1864 ts.tv_sec = timeout / 1000;
1865 ts.tv_nsec = (timeout % 1000) * 1000000;
1866 sigemptyset(&waitset);
1867 sigaddset(&waitset, SIG_IPI);
1868
1869 r = sigtimedwait(&waitset, &siginfo, &ts);
1870 e = errno;
1871
1872 pthread_mutex_lock(&qemu_mutex);
1873
1874 if (r == -1 && !(e == EAGAIN || e == EINTR)) {
1875 printf("sigtimedwait: %s\n", strerror(e));
1876 exit(1);
1877 }
1878
1879 cpu_single_env = env;
1880 flush_queued_work(env);
1881
1882 if (env->kvm_cpu_state.stop) {
1883 env->kvm_cpu_state.stop = 0;
1884 env->kvm_cpu_state.stopped = 1;
1885 pthread_cond_signal(&qemu_pause_cond);
1886 }
1887
1888 env->kvm_cpu_state.signalled = 0;
1889 }
1890
1891 static int all_threads_paused(void)
1892 {
1893 CPUState *penv = first_cpu;
1894
1895 while (penv) {
1896 if (penv->kvm_cpu_state.stop)
1897 return 0;
1898 penv = (CPUState *)penv->next_cpu;
1899 }
1900
1901 return 1;
1902 }
1903
1904 static void pause_all_threads(void)
1905 {
1906 CPUState *penv = first_cpu;
1907
1908 while (penv) {
1909 if (penv != cpu_single_env) {
1910 penv->kvm_cpu_state.stop = 1;
1911 pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
1912 } else {
1913 penv->kvm_cpu_state.stop = 0;
1914 penv->kvm_cpu_state.stopped = 1;
1915 cpu_exit(penv);
1916 }
1917 penv = (CPUState *)penv->next_cpu;
1918 }
1919
1920 while (!all_threads_paused())
1921 qemu_cond_wait(&qemu_pause_cond);
1922 }
1923
1924 static void resume_all_threads(void)
1925 {
1926 CPUState *penv = first_cpu;
1927
1928 assert(!cpu_single_env);
1929
1930 while (penv) {
1931 penv->kvm_cpu_state.stop = 0;
1932 penv->kvm_cpu_state.stopped = 0;
1933 pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
1934 penv = (CPUState *)penv->next_cpu;
1935 }
1936 }
1937
1938 static void kvm_vm_state_change_handler(void *context, int running, int reason)
1939 {
1940 if (running)
1941 resume_all_threads();
1942 else
1943 pause_all_threads();
1944 }
1945
1946 static void setup_kernel_sigmask(CPUState *env)
1947 {
1948 sigset_t set;
1949
1950 sigemptyset(&set);
1951 sigaddset(&set, SIGUSR2);
1952 sigaddset(&set, SIGIO);
1953 sigaddset(&set, SIGALRM);
1954 sigprocmask(SIG_BLOCK, &set, NULL);
1955
1956 sigprocmask(SIG_BLOCK, NULL, &set);
1957 sigdelset(&set, SIG_IPI);
1958
1959 kvm_set_signal_mask(env->kvm_cpu_state.vcpu_ctx, &set);
1960 }
1961
1962 static void qemu_kvm_system_reset(void)
1963 {
1964 CPUState *penv = first_cpu;
1965
1966 pause_all_threads();
1967
1968 qemu_system_reset();
1969
1970 while (penv) {
1971 kvm_arch_cpu_reset(penv);
1972 penv = (CPUState *)penv->next_cpu;
1973 }
1974
1975 resume_all_threads();
1976 }
1977
1978 static void process_irqchip_events(CPUState *env)
1979 {
1980 kvm_arch_process_irqchip_events(env);
1981 if (kvm_arch_has_work(env))
1982 env->halted = 0;
1983 }
1984
1985 static int kvm_main_loop_cpu(CPUState *env)
1986 {
1987 setup_kernel_sigmask(env);
1988
1989 pthread_mutex_lock(&qemu_mutex);
1990
1991 kvm_qemu_init_env(env);
1992 #ifdef TARGET_I386
1993 kvm_tpr_vcpu_start(env);
1994 #endif
1995
1996 cpu_single_env = env;
1997 kvm_arch_load_regs(env);
1998
1999 while (1) {
2000 int run_cpu = !is_cpu_stopped(env);
2001 if (run_cpu && !kvm_irqchip_in_kernel(kvm_context)) {
2002 process_irqchip_events(env);
2003 run_cpu = !env->halted;
2004 }
2005 if (run_cpu) {
2006 kvm_main_loop_wait(env, 0);
2007 kvm_cpu_exec(env);
2008 } else {
2009 kvm_main_loop_wait(env, 1000);
2010 }
2011 }
2012 pthread_mutex_unlock(&qemu_mutex);
2013 return 0;
2014 }
2015
2016 static void *ap_main_loop(void *_env)
2017 {
2018 CPUState *env = _env;
2019 sigset_t signals;
2020 struct ioperm_data *data = NULL;
2021
2022 current_env = env;
2023 env->thread_id = kvm_get_thread_id();
2024 sigfillset(&signals);
2025 sigprocmask(SIG_BLOCK, &signals, NULL);
2026 env->kvm_cpu_state.vcpu_ctx = kvm_create_vcpu(kvm_context, env->cpu_index);
2027
2028 #ifdef USE_KVM_DEVICE_ASSIGNMENT
2029 /* do ioperm for io ports of assigned devices */
2030 LIST_FOREACH(data, &ioperm_head, entries)
2031 on_vcpu(env, kvm_arch_do_ioperm, data);
2032 #endif
2033
2034 /* signal VCPU creation */
2035 pthread_mutex_lock(&qemu_mutex);
2036 current_env->kvm_cpu_state.created = 1;
2037 pthread_cond_signal(&qemu_vcpu_cond);
2038
2039 /* and wait for machine initialization */
2040 while (!qemu_system_ready)
2041 qemu_cond_wait(&qemu_system_cond);
2042 pthread_mutex_unlock(&qemu_mutex);
2043
2044 kvm_main_loop_cpu(env);
2045 return NULL;
2046 }
2047
2048 void kvm_init_vcpu(CPUState *env)
2049 {
2050 pthread_create(&env->kvm_cpu_state.thread, NULL, ap_main_loop, env);
2051
2052 while (env->kvm_cpu_state.created == 0)
2053 qemu_cond_wait(&qemu_vcpu_cond);
2054 }
2055
2056 int kvm_vcpu_inited(CPUState *env)
2057 {
2058 return env->kvm_cpu_state.created;
2059 }
2060
2061 int kvm_init_ap(void)
2062 {
2063 #ifdef TARGET_I386
2064 kvm_tpr_opt_setup();
2065 #endif
2066 qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);
2067
2068 signal(SIG_IPI, sig_ipi_handler);
2069 return 0;
2070 }
2071
2072 void qemu_kvm_notify_work(void)
2073 {
2074 uint64_t value = 1;
2075 char buffer[8];
2076 size_t offset = 0;
2077
2078 if (io_thread_fd == -1)
2079 return;
2080
2081 memcpy(buffer, &value, sizeof(value));
2082
2083 while (offset < 8) {
2084 ssize_t len;
2085
2086 len = write(io_thread_fd, buffer + offset, 8 - offset);
2087 if (len == -1 && errno == EINTR)
2088 continue;
2089
2090 if (len <= 0)
2091 break;
2092
2093 offset += len;
2094 }
2095
2096 if (offset != 8)
2097 fprintf(stderr, "failed to notify io thread\n");
2098 }
2099
2100 /* If we have signalfd, we mask out the signals we want to handle and then
2101 * use signalfd to listen for them. We rely on whatever the current signal
2102 * handler is to dispatch the signals when we receive them.
2103 */
2104
2105 static void sigfd_handler(void *opaque)
2106 {
2107 int fd = (unsigned long)opaque;
2108 struct qemu_signalfd_siginfo info;
2109 struct sigaction action;
2110 ssize_t len;
2111
2112 while (1) {
2113 do {
2114 len = read(fd, &info, sizeof(info));
2115 } while (len == -1 && errno == EINTR);
2116
2117 if (len == -1 && errno == EAGAIN)
2118 break;
2119
2120 if (len != sizeof(info)) {
2121 printf("read from sigfd returned %zd: %m\n", len);
2122 return;
2123 }
2124
2125 sigaction(info.ssi_signo, NULL, &action);
2126 if (action.sa_handler)
2127 action.sa_handler(info.ssi_signo);
2128
2129 }
2130 }
2131
2132 /* Used to break IO thread out of select */
2133 static void io_thread_wakeup(void *opaque)
2134 {
2135 int fd = (unsigned long)opaque;
2136 char buffer[8];
2137 size_t offset = 0;
2138
2139 while (offset < 8) {
2140 ssize_t len;
2141
2142 len = read(fd, buffer + offset, 8 - offset);
2143 if (len == -1 && errno == EINTR)
2144 continue;
2145
2146 if (len <= 0)
2147 break;
2148
2149 offset += len;
2150 }
2151 }
2152
2153 int kvm_main_loop(void)
2154 {
2155 int fds[2];
2156 sigset_t mask;
2157 int sigfd;
2158
2159 io_thread = pthread_self();
2160 qemu_system_ready = 1;
2161
2162 if (qemu_eventfd(fds) == -1) {
2163 fprintf(stderr, "failed to create eventfd\n");
2164 return -errno;
2165 }
2166
2167 qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
2168 (void *)(unsigned long)fds[0]);
2169
2170 io_thread_fd = fds[1];
2171
2172 sigemptyset(&mask);
2173 sigaddset(&mask, SIGIO);
2174 sigaddset(&mask, SIGALRM);
2175 sigprocmask(SIG_BLOCK, &mask, NULL);
2176
2177 sigfd = qemu_signalfd(&mask);
2178 if (sigfd == -1) {
2179 fprintf(stderr, "failed to create signalfd\n");
2180 return -errno;
2181 }
2182
2183 fcntl(sigfd, F_SETFL, O_NONBLOCK);
2184
2185 qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
2186 (void *)(unsigned long)sigfd);
2187
2188 pthread_cond_broadcast(&qemu_system_cond);
2189
2190 io_thread_sigfd = sigfd;
2191 cpu_single_env = NULL;
2192
2193 while (1) {
2194 main_loop_wait(1000);
2195 if (qemu_shutdown_requested()) {
2196 if (qemu_no_shutdown()) {
2197 vm_stop(0);
2198 } else
2199 break;
2200 } else if (qemu_powerdown_requested())
2201 qemu_system_powerdown();
2202 else if (qemu_reset_requested())
2203 qemu_kvm_system_reset();
2204 else if (kvm_debug_cpu_requested) {
2205 gdb_set_stop_cpu(kvm_debug_cpu_requested);
2206 vm_stop(EXCP_DEBUG);
2207 kvm_debug_cpu_requested = NULL;
2208 }
2209 }
2210
2211 pause_all_threads();
2212 pthread_mutex_unlock(&qemu_mutex);
2213
2214 return 0;
2215 }
2216
2217 int kvm_qemu_init()
2218 {
2219 /* Try to initialize kvm */
2220 kvm_context = kvm_init(cpu_single_env);
2221 if (!kvm_context) {
2222 return -1;
2223 }
2224 pthread_mutex_lock(&qemu_mutex);
2225
2226 return 0;
2227 }
2228
2229 #ifdef TARGET_I386
2230 static int destroy_region_works = 0;
2231 #endif
2232
2233
2234 #if !defined(TARGET_I386)
2235 int kvm_arch_init_irq_routing(void)
2236 {
2237 return 0;
2238 }
2239 #endif
2240
2241 int kvm_qemu_create_context(void)
2242 {
2243 int r;
2244
2245 if (!kvm_irqchip) {
2246 kvm_disable_irqchip_creation(kvm_context);
2247 }
2248 if (!kvm_pit) {
2249 kvm_disable_pit_creation(kvm_context);
2250 }
2251 if (kvm_create(kvm_context, 0, NULL) < 0) {
2252 kvm_finalize(kvm_context);
2253 return -1;
2254 }
2255 r = kvm_arch_qemu_create_context();
2256 if(r <0)
2257 kvm_finalize(kvm_context);
2258 if (kvm_pit && !kvm_pit_reinject) {
2259 if (kvm_reinject_control(kvm_context, 0)) {
2260 fprintf(stderr, "failure to disable in-kernel PIT reinjection\n");
2261 return -1;
2262 }
2263 }
2264 #ifdef TARGET_I386
2265 destroy_region_works = kvm_destroy_memory_region_works(kvm_context);
2266 #endif
2267
2268 r = kvm_arch_init_irq_routing();
2269 if (r < 0) {
2270 return r;
2271 }
2272
2273 return 0;
2274 }
2275
2276 #ifdef TARGET_I386
2277 static int must_use_aliases_source(target_phys_addr_t addr)
2278 {
2279 if (destroy_region_works)
2280 return false;
2281 if (addr == 0xa0000 || addr == 0xa8000)
2282 return true;
2283 return false;
2284 }
2285
2286 static int must_use_aliases_target(target_phys_addr_t addr)
2287 {
2288 if (destroy_region_works)
2289 return false;
2290 if (addr >= 0xe0000000 && addr < 0x100000000ull)
2291 return true;
2292 return false;
2293 }
2294
2295 static struct mapping {
2296 target_phys_addr_t phys;
2297 ram_addr_t ram;
2298 ram_addr_t len;
2299 } mappings[50];
2300 static int nr_mappings;
2301
2302 static struct mapping *find_ram_mapping(ram_addr_t ram_addr)
2303 {
2304 struct mapping *p;
2305
2306 for (p = mappings; p < mappings + nr_mappings; ++p) {
2307 if (p->ram <= ram_addr && ram_addr < p->ram + p->len) {
2308 return p;
2309 }
2310 }
2311 return NULL;
2312 }
2313
2314 static struct mapping *find_mapping(target_phys_addr_t start_addr)
2315 {
2316 struct mapping *p;
2317
2318 for (p = mappings; p < mappings + nr_mappings; ++p) {
2319 if (p->phys <= start_addr && start_addr < p->phys + p->len) {
2320 return p;
2321 }
2322 }
2323 return NULL;
2324 }
2325
2326 static void drop_mapping(target_phys_addr_t start_addr)
2327 {
2328 struct mapping *p = find_mapping(start_addr);
2329
2330 if (p)
2331 *p = mappings[--nr_mappings];
2332 }
2333 #endif
2334
2335 void kvm_set_phys_mem(target_phys_addr_t start_addr, ram_addr_t size,
2336 ram_addr_t phys_offset)
2337 {
2338 int r = 0;
2339 unsigned long area_flags;
2340 #ifdef TARGET_I386
2341 struct mapping *p;
2342 #endif
2343
2344 if (start_addr + size > phys_ram_size) {
2345 phys_ram_size = start_addr + size;
2346 }
2347
2348 phys_offset &= ~IO_MEM_ROM;
2349 area_flags = phys_offset & ~TARGET_PAGE_MASK;
2350
2351 if (area_flags != IO_MEM_RAM) {
2352 #ifdef TARGET_I386
2353 if (must_use_aliases_source(start_addr)) {
2354 kvm_destroy_memory_alias(kvm_context, start_addr);
2355 return;
2356 }
2357 if (must_use_aliases_target(start_addr))
2358 return;
2359 #endif
2360 while (size > 0) {
2361 p = find_mapping(start_addr);
2362 if (p) {
2363 kvm_unregister_memory_area(kvm_context, p->phys, p->len);
2364 drop_mapping(p->phys);
2365 }
2366 start_addr += TARGET_PAGE_SIZE;
2367 if (size > TARGET_PAGE_SIZE) {
2368 size -= TARGET_PAGE_SIZE;
2369 } else {
2370 size = 0;
2371 }
2372 }
2373 return;
2374 }
2375
2376 r = kvm_is_containing_region(kvm_context, start_addr, size);
2377 if (r)
2378 return;
2379
2380 if (area_flags >= TLB_MMIO)
2381 return;
2382
2383 #ifdef TARGET_I386
2384 if (must_use_aliases_source(start_addr)) {
2385 p = find_ram_mapping(phys_offset);
2386 if (p) {
2387 kvm_create_memory_alias(kvm_context, start_addr, size,
2388 p->phys + (phys_offset - p->ram));
2389 }
2390 return;
2391 }
2392 #endif
2393
2394 r = kvm_register_phys_mem(kvm_context, start_addr,
2395 qemu_get_ram_ptr(phys_offset),
2396 size, 0);
2397 if (r < 0) {
2398 printf("kvm_cpu_register_physical_memory: failed\n");
2399 exit(1);
2400 }
2401
2402 #ifdef TARGET_I386
2403 drop_mapping(start_addr);
2404 p = &mappings[nr_mappings++];
2405 p->phys = start_addr;
2406 p->ram = phys_offset;
2407 p->len = size;
2408 #endif
2409
2410 return;
2411 }
2412
2413 int kvm_setup_guest_memory(void *area, unsigned long size)
2414 {
2415 int ret = 0;
2416
2417 #ifdef MADV_DONTFORK
2418 if (kvm_enabled() && !kvm_has_sync_mmu())
2419 ret = madvise(area, size, MADV_DONTFORK);
2420 #endif
2421
2422 if (ret)
2423 perror ("madvise");
2424
2425 return ret;
2426 }
2427
2428 int kvm_qemu_check_extension(int ext)
2429 {
2430 return kvm_check_extension(kvm_context, ext);
2431 }
2432
2433 int kvm_qemu_init_env(CPUState *cenv)
2434 {
2435 return kvm_arch_qemu_init_env(cenv);
2436 }
2437
2438 #ifdef KVM_CAP_SET_GUEST_DEBUG
2439 struct kvm_sw_breakpoint_head kvm_sw_breakpoints =
2440 TAILQ_HEAD_INITIALIZER(kvm_sw_breakpoints);
2441
2442 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(target_ulong pc)
2443 {
2444 struct kvm_sw_breakpoint *bp;
2445
2446 TAILQ_FOREACH(bp, &kvm_sw_breakpoints, entry) {
2447 if (bp->pc == pc)
2448 return bp;
2449 }
2450 return NULL;
2451 }
2452
2453 struct kvm_set_guest_debug_data {
2454 struct kvm_guest_debug dbg;
2455 int err;
2456 };
2457
2458 static void kvm_invoke_set_guest_debug(void *data)
2459 {
2460 struct kvm_set_guest_debug_data *dbg_data = data;
2461
2462 dbg_data->err = kvm_set_guest_debug(cpu_single_env->kvm_cpu_state.vcpu_ctx,
2463 &dbg_data->dbg);
2464 }
2465
2466 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
2467 {
2468 struct kvm_set_guest_debug_data data;
2469
2470 data.dbg.control = 0;
2471 if (env->singlestep_enabled)
2472 data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2473
2474 kvm_arch_update_guest_debug(env, &data.dbg);
2475 data.dbg.control |= reinject_trap;
2476
2477 on_vcpu(env, kvm_invoke_set_guest_debug, &data);
2478 return data.err;
2479 }
2480
2481 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
2482 target_ulong len, int type)
2483 {
2484 struct kvm_sw_breakpoint *bp;
2485 CPUState *env;
2486 int err;
2487
2488 if (type == GDB_BREAKPOINT_SW) {
2489 bp = kvm_find_sw_breakpoint(addr);
2490 if (bp) {
2491 bp->use_count++;
2492 return 0;
2493 }
2494
2495 bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
2496 if (!bp)
2497 return -ENOMEM;
2498
2499 bp->pc = addr;
2500 bp->use_count = 1;
2501 err = kvm_arch_insert_sw_breakpoint(current_env, bp);
2502 if (err) {
2503 free(bp);
2504 return err;
2505 }
2506
2507 TAILQ_INSERT_HEAD(&kvm_sw_breakpoints, bp, entry);
2508 } else {
2509 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
2510 if (err)
2511 return err;
2512 }
2513
2514 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2515 err = kvm_update_guest_debug(env, 0);
2516 if (err)
2517 return err;
2518 }
2519 return 0;
2520 }
2521
2522 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
2523 target_ulong len, int type)
2524 {
2525 struct kvm_sw_breakpoint *bp;
2526 CPUState *env;
2527 int err;
2528
2529 if (type == GDB_BREAKPOINT_SW) {
2530 bp = kvm_find_sw_breakpoint(addr);
2531 if (!bp)
2532 return -ENOENT;
2533
2534 if (bp->use_count > 1) {
2535 bp->use_count--;
2536 return 0;
2537 }
2538
2539 err = kvm_arch_remove_sw_breakpoint(current_env, bp);
2540 if (err)
2541 return err;
2542
2543 TAILQ_REMOVE(&kvm_sw_breakpoints, bp, entry);
2544 qemu_free(bp);
2545 } else {
2546 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
2547 if (err)
2548 return err;
2549 }
2550
2551 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2552 err = kvm_update_guest_debug(env, 0);
2553 if (err)
2554 return err;
2555 }
2556 return 0;
2557 }
2558
2559 void kvm_remove_all_breakpoints(CPUState *current_env)
2560 {
2561 struct kvm_sw_breakpoint *bp, *next;
2562 CPUState *env;
2563
2564 TAILQ_FOREACH_SAFE(bp, &kvm_sw_breakpoints, entry, next) {
2565 if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
2566 /* Try harder to find a CPU that currently sees the breakpoint. */
2567 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2568 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
2569 break;
2570 }
2571 }
2572 }
2573 kvm_arch_remove_all_hw_breakpoints();
2574
2575 for (env = first_cpu; env != NULL; env = env->next_cpu)
2576 kvm_update_guest_debug(env, 0);
2577 }
2578
2579 #else /* !KVM_CAP_SET_GUEST_DEBUG */
2580
2581 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
2582 {
2583 return -EINVAL;
2584 }
2585
2586 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
2587 target_ulong len, int type)
2588 {
2589 return -EINVAL;
2590 }
2591
2592 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
2593 target_ulong len, int type)
2594 {
2595 return -EINVAL;
2596 }
2597
2598 void kvm_remove_all_breakpoints(CPUState *current_env)
2599 {
2600 }
2601 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
2602
2603 /*
2604 * dirty pages logging
2605 */
2606 /* FIXME: use unsigned long pointer instead of unsigned char */
2607 unsigned char *kvm_dirty_bitmap = NULL;
2608 int kvm_physical_memory_set_dirty_tracking(int enable)
2609 {
2610 int r = 0;
2611
2612 if (!kvm_enabled())
2613 return 0;
2614
2615 if (enable) {
2616 if (!kvm_dirty_bitmap) {
2617 unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
2618 kvm_dirty_bitmap = qemu_malloc(bitmap_size);
2619 if (kvm_dirty_bitmap == NULL) {
2620 perror("Failed to allocate dirty pages bitmap");
2621 r=-1;
2622 }
2623 else {
2624 r = kvm_dirty_pages_log_enable_all(kvm_context);
2625 }
2626 }
2627 }
2628 else {
2629 if (kvm_dirty_bitmap) {
2630 r = kvm_dirty_pages_log_reset(kvm_context);
2631 qemu_free(kvm_dirty_bitmap);
2632 kvm_dirty_bitmap = NULL;
2633 }
2634 }
2635 return r;
2636 }
2637
2638 /* get kvm's dirty pages bitmap and update qemu's */
2639 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
2640 unsigned char *bitmap,
2641 unsigned long offset,
2642 unsigned long mem_size)
2643 {
2644 unsigned int i, j, n=0;
2645 unsigned char c;
2646 unsigned long page_number, addr, addr1;
2647 ram_addr_t ram_addr;
2648 unsigned int len = ((mem_size/TARGET_PAGE_SIZE) + 7) / 8;
2649
2650 /*
2651 * bitmap-traveling is faster than memory-traveling (for addr...)
2652 * especially when most of the memory is not dirty.
2653 */
2654 for (i=0; i<len; i++) {
2655 c = bitmap[i];
2656 while (c>0) {
2657 j = ffsl(c) - 1;
2658 c &= ~(1u<<j);
2659 page_number = i * 8 + j;
2660 addr1 = page_number * TARGET_PAGE_SIZE;
2661 addr = offset + addr1;
2662 ram_addr = cpu_get_physical_page_desc(addr);
2663 cpu_physical_memory_set_dirty(ram_addr);
2664 n++;
2665 }
2666 }
2667 return 0;
2668 }
2669 static int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
2670 void *bitmap, void *opaque)
2671 {
2672 return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
2673 }
2674
2675 /*
2676 * get kvm's dirty pages bitmap and update qemu's
2677 * we only care about physical ram, which resides in slots 0 and 3
2678 */
2679 int kvm_update_dirty_pages_log(void)
2680 {
2681 int r = 0;
2682
2683
2684 r = kvm_get_dirty_pages_range(kvm_context, 0, -1UL,
2685 NULL,
2686 kvm_get_dirty_bitmap_cb);
2687 return r;
2688 }
2689
2690 void kvm_qemu_log_memory(target_phys_addr_t start, target_phys_addr_t size,
2691 int log)
2692 {
2693 if (log)
2694 kvm_dirty_pages_log_enable_slot(kvm_context, start, size);
2695 else {
2696 #ifdef TARGET_I386
2697 if (must_use_aliases_target(start))
2698 return;
2699 #endif
2700 kvm_dirty_pages_log_disable_slot(kvm_context, start, size);
2701 }
2702 }
2703
2704 int kvm_get_phys_ram_page_bitmap(unsigned char *bitmap)
2705 {
2706 unsigned int bsize = BITMAP_SIZE(phys_ram_size);
2707 unsigned int brsize = BITMAP_SIZE(ram_size);
2708 unsigned int extra_pages = (phys_ram_size - ram_size) / TARGET_PAGE_SIZE;
2709 unsigned int extra_bytes = (extra_pages +7)/8;
2710 unsigned int hole_start = BITMAP_SIZE(0xa0000);
2711 unsigned int hole_end = BITMAP_SIZE(0xc0000);
2712
2713 memset(bitmap, 0xFF, brsize + extra_bytes);
2714 memset(bitmap + hole_start, 0, hole_end - hole_start);
2715 memset(bitmap + brsize + extra_bytes, 0, bsize - brsize - extra_bytes);
2716
2717 return 0;
2718 }
2719
2720 #ifdef KVM_CAP_IRQCHIP
2721
2722 int kvm_set_irq(int irq, int level, int *status)
2723 {
2724 return kvm_set_irq_level(kvm_context, irq, level, status);
2725 }
2726
2727 #endif
2728
2729 int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
2730 {
2731 return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
2732 }
2733
2734 void kvm_mutex_unlock(void)
2735 {
2736 assert(!cpu_single_env);
2737 pthread_mutex_unlock(&qemu_mutex);
2738 }
2739
2740 void kvm_mutex_lock(void)
2741 {
2742 pthread_mutex_lock(&qemu_mutex);
2743 cpu_single_env = NULL;
2744 }
2745
2746 #ifdef USE_KVM_DEVICE_ASSIGNMENT
2747 void kvm_add_ioperm_data(struct ioperm_data *data)
2748 {
2749 LIST_INSERT_HEAD(&ioperm_head, data, entries);
2750 }
2751
2752 void kvm_remove_ioperm_data(unsigned long start_port, unsigned long num)
2753 {
2754 struct ioperm_data *data;
2755
2756 data = LIST_FIRST(&ioperm_head);
2757 while (data) {
2758 struct ioperm_data *next = LIST_NEXT(data, entries);
2759
2760 if (data->start_port == start_port && data->num == num) {
2761 LIST_REMOVE(data, entries);
2762 qemu_free(data);
2763 }
2764
2765 data = next;
2766 }
2767 }
2768
2769 void kvm_ioperm(CPUState *env, void *data)
2770 {
2771 if (kvm_enabled() && qemu_system_ready)
2772 on_vcpu(env, kvm_arch_do_ioperm, data);
2773 }
2774
2775 #endif
2776
2777 int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
2778 {
2779 #ifndef TARGET_IA64
2780
2781 #ifdef TARGET_I386
2782 if (must_use_aliases_source(start_addr))
2783 return 0;
2784 #endif
2785
2786 kvm_get_dirty_pages_range(kvm_context, start_addr, end_addr - start_addr,
2787 NULL, kvm_get_dirty_bitmap_cb);
2788 #endif
2789 return 0;
2790 }
2791
2792 int kvm_log_start(target_phys_addr_t phys_addr, target_phys_addr_t len)
2793 {
2794 #ifdef TARGET_I386
2795 if (must_use_aliases_source(phys_addr))
2796 return 0;
2797 #endif
2798
2799 #ifndef TARGET_IA64
2800 kvm_qemu_log_memory(phys_addr, len, 1);
2801 #endif
2802 return 0;
2803 }
2804
2805 int kvm_log_stop(target_phys_addr_t phys_addr, target_phys_addr_t len)
2806 {
2807 #ifdef TARGET_I386
2808 if (must_use_aliases_source(phys_addr))
2809 return 0;
2810 #endif
2811
2812 #ifndef TARGET_IA64
2813 kvm_qemu_log_memory(phys_addr, len, 0);
2814 #endif
2815 return 0;
2816 }
2817
2818 void qemu_kvm_cpu_stop(CPUState *env)
2819 {
2820 if (kvm_enabled())
2821 env->kvm_cpu_state.stopped = 1;
2822 }
2823
2824 int kvm_set_boot_cpu_id(uint32_t id)
2825 {
2826 return kvm_set_boot_vcpu_id(kvm_context, id);
2827 }
Patches shipped by Fedora