2 * kvm_ia64.c: Basic KVM suppport On Itanium series processors
5 * Copyright (C) 2007, Intel Corporation.
6 * Xiantao Zhang (xiantao.zhang@intel.com)
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 * Place - Suite 330, Boston, MA 02111-1307 USA.
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/smp.h>
29 #include <linux/kvm_host.h>
30 #include <linux/kvm.h>
31 #include <linux/bitops.h>
32 #include <linux/hrtimer.h>
33 #include <linux/uaccess.h>
34 #include <linux/iommu.h>
35 #include <linux/intel-iommu.h>
37 #include <asm/pgtable.h>
38 #include <asm/gcc_intrin.h>
40 #include <asm/cacheflush.h>
41 #include <asm/div64.h>
44 #include <asm/sn/addrs.h>
45 #include <asm/sn/clksupport.h>
46 #include <asm/sn/shub_mmr.h>
55 static unsigned long kvm_vmm_base
;
56 static unsigned long kvm_vsa_base
;
57 static unsigned long kvm_vm_buffer
;
58 static unsigned long kvm_vm_buffer_size
;
59 unsigned long kvm_vmm_gp
;
61 static long vp_env_info
;
63 static struct kvm_vmm_info
*kvm_vmm_info
;
65 static DEFINE_PER_CPU(struct kvm_vcpu
*, last_vcpu
);
67 struct kvm_stats_debugfs_item debugfs_entries
[] = {
71 static unsigned long kvm_get_itc(struct kvm_vcpu
*vcpu
)
73 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
74 if (vcpu
->kvm
->arch
.is_sn2
)
78 return ia64_getreg(_IA64_REG_AR_ITC
);
81 static void kvm_flush_icache(unsigned long start
, unsigned long len
)
85 for (l
= 0; l
< (len
+ 32); l
+= 32)
86 ia64_fc((void *)(start
+ l
));
92 static void kvm_flush_tlb_all(void)
94 unsigned long i
, j
, count0
, count1
, stride0
, stride1
, addr
;
97 addr
= local_cpu_data
->ptce_base
;
98 count0
= local_cpu_data
->ptce_count
[0];
99 count1
= local_cpu_data
->ptce_count
[1];
100 stride0
= local_cpu_data
->ptce_stride
[0];
101 stride1
= local_cpu_data
->ptce_stride
[1];
103 local_irq_save(flags
);
104 for (i
= 0; i
< count0
; ++i
) {
105 for (j
= 0; j
< count1
; ++j
) {
111 local_irq_restore(flags
);
112 ia64_srlz_i(); /* srlz.i implies srlz.d */
115 long ia64_pal_vp_create(u64
*vpd
, u64
*host_iva
, u64
*opt_handler
)
117 struct ia64_pal_retval iprv
;
119 PAL_CALL_STK(iprv
, PAL_VP_CREATE
, (u64
)vpd
, (u64
)host_iva
,
125 static DEFINE_SPINLOCK(vp_lock
);
127 int kvm_arch_hardware_enable(void *garbage
)
132 unsigned long saved_psr
;
135 pte
= pte_val(mk_pte_phys(__pa(kvm_vmm_base
), PAGE_KERNEL
));
136 local_irq_save(saved_psr
);
137 slot
= ia64_itr_entry(0x3, KVM_VMM_BASE
, pte
, KVM_VMM_SHIFT
);
138 local_irq_restore(saved_psr
);
143 status
= ia64_pal_vp_init_env(kvm_vsa_base
?
144 VP_INIT_ENV
: VP_INIT_ENV_INITALIZE
,
145 __pa(kvm_vm_buffer
), KVM_VM_BUFFER_BASE
, &tmp_base
);
147 spin_unlock(&vp_lock
);
148 printk(KERN_WARNING
"kvm: Failed to Enable VT Support!!!!\n");
153 kvm_vsa_base
= tmp_base
;
154 printk(KERN_INFO
"kvm: kvm_vsa_base:0x%lx\n", kvm_vsa_base
);
156 spin_unlock(&vp_lock
);
157 ia64_ptr_entry(0x3, slot
);
162 void kvm_arch_hardware_disable(void *garbage
)
168 unsigned long saved_psr
;
169 unsigned long host_iva
= ia64_getreg(_IA64_REG_CR_IVA
);
171 pte
= pte_val(mk_pte_phys(__pa(kvm_vmm_base
),
174 local_irq_save(saved_psr
);
175 slot
= ia64_itr_entry(0x3, KVM_VMM_BASE
, pte
, KVM_VMM_SHIFT
);
176 local_irq_restore(saved_psr
);
180 status
= ia64_pal_vp_exit_env(host_iva
);
182 printk(KERN_DEBUG
"kvm: Failed to disable VT support! :%ld\n",
184 ia64_ptr_entry(0x3, slot
);
187 void kvm_arch_check_processor_compat(void *rtn
)
192 int kvm_dev_ioctl_check_extension(long ext
)
198 case KVM_CAP_IRQCHIP
:
199 case KVM_CAP_MP_STATE
:
200 case KVM_CAP_IRQ_INJECT_STATUS
:
203 case KVM_CAP_COALESCED_MMIO
:
204 r
= KVM_COALESCED_MMIO_PAGE_OFFSET
;
216 static int handle_vm_error(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
218 kvm_run
->exit_reason
= KVM_EXIT_UNKNOWN
;
219 kvm_run
->hw
.hardware_exit_reason
= 1;
223 static int handle_mmio(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
225 struct kvm_mmio_req
*p
;
226 struct kvm_io_device
*mmio_dev
;
229 p
= kvm_get_vcpu_ioreq(vcpu
);
231 if ((p
->addr
& PAGE_MASK
) == IOAPIC_DEFAULT_BASE_ADDRESS
)
233 vcpu
->mmio_needed
= 1;
234 vcpu
->mmio_phys_addr
= kvm_run
->mmio
.phys_addr
= p
->addr
;
235 vcpu
->mmio_size
= kvm_run
->mmio
.len
= p
->size
;
236 vcpu
->mmio_is_write
= kvm_run
->mmio
.is_write
= !p
->dir
;
238 if (vcpu
->mmio_is_write
)
239 memcpy(vcpu
->mmio_data
, &p
->data
, p
->size
);
240 memcpy(kvm_run
->mmio
.data
, &p
->data
, p
->size
);
241 kvm_run
->exit_reason
= KVM_EXIT_MMIO
;
245 r
= kvm_io_bus_read(vcpu
->kvm
, KVM_MMIO_BUS
, p
->addr
,
248 r
= kvm_io_bus_write(vcpu
->kvm
, KVM_MMIO_BUS
, p
->addr
,
251 printk(KERN_ERR
"kvm: No iodevice found! addr:%lx\n", p
->addr
);
252 p
->state
= STATE_IORESP_READY
;
257 static int handle_pal_call(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
259 struct exit_ctl_data
*p
;
261 p
= kvm_get_exit_data(vcpu
);
263 if (p
->exit_reason
== EXIT_REASON_PAL_CALL
)
264 return kvm_pal_emul(vcpu
, kvm_run
);
266 kvm_run
->exit_reason
= KVM_EXIT_UNKNOWN
;
267 kvm_run
->hw
.hardware_exit_reason
= 2;
272 static int handle_sal_call(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
274 struct exit_ctl_data
*p
;
276 p
= kvm_get_exit_data(vcpu
);
278 if (p
->exit_reason
== EXIT_REASON_SAL_CALL
) {
282 kvm_run
->exit_reason
= KVM_EXIT_UNKNOWN
;
283 kvm_run
->hw
.hardware_exit_reason
= 3;
289 static int __apic_accept_irq(struct kvm_vcpu
*vcpu
, uint64_t vector
)
291 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
293 if (!test_and_set_bit(vector
, &vpd
->irr
[0])) {
294 vcpu
->arch
.irq_new_pending
= 1;
302 * offset: address offset to IPI space.
303 * value: deliver value.
305 static void vcpu_deliver_ipi(struct kvm_vcpu
*vcpu
, uint64_t dm
,
320 printk(KERN_ERR
"kvm: Unimplemented Deliver reserved IPI!\n");
323 __apic_accept_irq(vcpu
, vector
);
326 static struct kvm_vcpu
*lid_to_vcpu(struct kvm
*kvm
, unsigned long id
,
331 struct kvm_vcpu
*vcpu
;
333 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
334 lid
.val
= VCPU_LID(vcpu
);
335 if (lid
.id
== id
&& lid
.eid
== eid
)
342 static int handle_ipi(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
344 struct exit_ctl_data
*p
= kvm_get_exit_data(vcpu
);
345 struct kvm_vcpu
*target_vcpu
;
346 struct kvm_pt_regs
*regs
;
347 union ia64_ipi_a addr
= p
->u
.ipi_data
.addr
;
348 union ia64_ipi_d data
= p
->u
.ipi_data
.data
;
350 target_vcpu
= lid_to_vcpu(vcpu
->kvm
, addr
.id
, addr
.eid
);
352 return handle_vm_error(vcpu
, kvm_run
);
354 if (!target_vcpu
->arch
.launched
) {
355 regs
= vcpu_regs(target_vcpu
);
357 regs
->cr_iip
= vcpu
->kvm
->arch
.rdv_sal_data
.boot_ip
;
358 regs
->r1
= vcpu
->kvm
->arch
.rdv_sal_data
.boot_gp
;
360 target_vcpu
->arch
.mp_state
= KVM_MP_STATE_RUNNABLE
;
361 if (waitqueue_active(&target_vcpu
->wq
))
362 wake_up_interruptible(&target_vcpu
->wq
);
364 vcpu_deliver_ipi(target_vcpu
, data
.dm
, data
.vector
);
365 if (target_vcpu
!= vcpu
)
366 kvm_vcpu_kick(target_vcpu
);
373 struct kvm_ptc_g ptc_g_data
;
374 struct kvm_vcpu
*vcpu
;
377 static void vcpu_global_purge(void *info
)
379 struct call_data
*p
= (struct call_data
*)info
;
380 struct kvm_vcpu
*vcpu
= p
->vcpu
;
382 if (test_bit(KVM_REQ_TLB_FLUSH
, &vcpu
->requests
))
385 set_bit(KVM_REQ_PTC_G
, &vcpu
->requests
);
386 if (vcpu
->arch
.ptc_g_count
< MAX_PTC_G_NUM
) {
387 vcpu
->arch
.ptc_g_data
[vcpu
->arch
.ptc_g_count
++] =
390 clear_bit(KVM_REQ_PTC_G
, &vcpu
->requests
);
391 vcpu
->arch
.ptc_g_count
= 0;
392 set_bit(KVM_REQ_TLB_FLUSH
, &vcpu
->requests
);
396 static int handle_global_purge(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
398 struct exit_ctl_data
*p
= kvm_get_exit_data(vcpu
);
399 struct kvm
*kvm
= vcpu
->kvm
;
400 struct call_data call_data
;
402 struct kvm_vcpu
*vcpui
;
404 call_data
.ptc_g_data
= p
->u
.ptc_g_data
;
406 kvm_for_each_vcpu(i
, vcpui
, kvm
) {
407 if (vcpui
->arch
.mp_state
== KVM_MP_STATE_UNINITIALIZED
||
411 if (waitqueue_active(&vcpui
->wq
))
412 wake_up_interruptible(&vcpui
->wq
);
414 if (vcpui
->cpu
!= -1) {
415 call_data
.vcpu
= vcpui
;
416 smp_call_function_single(vcpui
->cpu
,
417 vcpu_global_purge
, &call_data
, 1);
419 printk(KERN_WARNING
"kvm: Uninit vcpu received ipi!\n");
425 static int handle_switch_rr6(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
430 static int kvm_sn2_setup_mappings(struct kvm_vcpu
*vcpu
)
432 unsigned long pte
, rtc_phys_addr
, map_addr
;
435 map_addr
= KVM_VMM_BASE
+ (1UL << KVM_VMM_SHIFT
);
436 rtc_phys_addr
= LOCAL_MMR_OFFSET
| SH_RTC
;
437 pte
= pte_val(mk_pte_phys(rtc_phys_addr
, PAGE_KERNEL_UC
));
438 slot
= ia64_itr_entry(0x3, map_addr
, pte
, PAGE_SHIFT
);
439 vcpu
->arch
.sn_rtc_tr_slot
= slot
;
441 printk(KERN_ERR
"Mayday mayday! RTC mapping failed!\n");
447 int kvm_emulate_halt(struct kvm_vcpu
*vcpu
)
452 unsigned long vcpu_now_itc
;
453 unsigned long expires
;
454 struct hrtimer
*p_ht
= &vcpu
->arch
.hlt_timer
;
455 unsigned long cyc_per_usec
= local_cpu_data
->cyc_per_usec
;
456 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
458 if (irqchip_in_kernel(vcpu
->kvm
)) {
460 vcpu_now_itc
= kvm_get_itc(vcpu
) + vcpu
->arch
.itc_offset
;
462 if (time_after(vcpu_now_itc
, vpd
->itm
)) {
463 vcpu
->arch
.timer_check
= 1;
466 itc_diff
= vpd
->itm
- vcpu_now_itc
;
468 itc_diff
= -itc_diff
;
470 expires
= div64_u64(itc_diff
, cyc_per_usec
);
471 kt
= ktime_set(0, 1000 * expires
);
473 vcpu
->arch
.ht_active
= 1;
474 hrtimer_start(p_ht
, kt
, HRTIMER_MODE_ABS
);
476 vcpu
->arch
.mp_state
= KVM_MP_STATE_HALTED
;
477 kvm_vcpu_block(vcpu
);
478 hrtimer_cancel(p_ht
);
479 vcpu
->arch
.ht_active
= 0;
481 if (test_and_clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
) ||
482 kvm_cpu_has_pending_timer(vcpu
))
483 if (vcpu
->arch
.mp_state
== KVM_MP_STATE_HALTED
)
484 vcpu
->arch
.mp_state
= KVM_MP_STATE_RUNNABLE
;
486 if (vcpu
->arch
.mp_state
!= KVM_MP_STATE_RUNNABLE
)
490 printk(KERN_ERR
"kvm: Unsupported userspace halt!");
495 static int handle_vm_shutdown(struct kvm_vcpu
*vcpu
,
496 struct kvm_run
*kvm_run
)
498 kvm_run
->exit_reason
= KVM_EXIT_SHUTDOWN
;
502 static int handle_external_interrupt(struct kvm_vcpu
*vcpu
,
503 struct kvm_run
*kvm_run
)
508 static int handle_vcpu_debug(struct kvm_vcpu
*vcpu
,
509 struct kvm_run
*kvm_run
)
511 printk("VMM: %s", vcpu
->arch
.log_buf
);
515 static int (*kvm_vti_exit_handlers
[])(struct kvm_vcpu
*vcpu
,
516 struct kvm_run
*kvm_run
) = {
517 [EXIT_REASON_VM_PANIC
] = handle_vm_error
,
518 [EXIT_REASON_MMIO_INSTRUCTION
] = handle_mmio
,
519 [EXIT_REASON_PAL_CALL
] = handle_pal_call
,
520 [EXIT_REASON_SAL_CALL
] = handle_sal_call
,
521 [EXIT_REASON_SWITCH_RR6
] = handle_switch_rr6
,
522 [EXIT_REASON_VM_DESTROY
] = handle_vm_shutdown
,
523 [EXIT_REASON_EXTERNAL_INTERRUPT
] = handle_external_interrupt
,
524 [EXIT_REASON_IPI
] = handle_ipi
,
525 [EXIT_REASON_PTC_G
] = handle_global_purge
,
526 [EXIT_REASON_DEBUG
] = handle_vcpu_debug
,
530 static const int kvm_vti_max_exit_handlers
=
531 sizeof(kvm_vti_exit_handlers
)/sizeof(*kvm_vti_exit_handlers
);
533 static uint32_t kvm_get_exit_reason(struct kvm_vcpu
*vcpu
)
535 struct exit_ctl_data
*p_exit_data
;
537 p_exit_data
= kvm_get_exit_data(vcpu
);
538 return p_exit_data
->exit_reason
;
542 * The guest has exited. See if we can fix it or if we need userspace
545 static int kvm_handle_exit(struct kvm_run
*kvm_run
, struct kvm_vcpu
*vcpu
)
547 u32 exit_reason
= kvm_get_exit_reason(vcpu
);
548 vcpu
->arch
.last_exit
= exit_reason
;
550 if (exit_reason
< kvm_vti_max_exit_handlers
551 && kvm_vti_exit_handlers
[exit_reason
])
552 return kvm_vti_exit_handlers
[exit_reason
](vcpu
, kvm_run
);
554 kvm_run
->exit_reason
= KVM_EXIT_UNKNOWN
;
555 kvm_run
->hw
.hardware_exit_reason
= exit_reason
;
560 static inline void vti_set_rr6(unsigned long rr6
)
562 ia64_set_rr(RR6
, rr6
);
566 static int kvm_insert_vmm_mapping(struct kvm_vcpu
*vcpu
)
569 struct kvm
*kvm
= vcpu
->kvm
;
572 /*Insert a pair of tr to map vmm*/
573 pte
= pte_val(mk_pte_phys(__pa(kvm_vmm_base
), PAGE_KERNEL
));
574 r
= ia64_itr_entry(0x3, KVM_VMM_BASE
, pte
, KVM_VMM_SHIFT
);
577 vcpu
->arch
.vmm_tr_slot
= r
;
578 /*Insert a pairt of tr to map data of vm*/
579 pte
= pte_val(mk_pte_phys(__pa(kvm
->arch
.vm_base
), PAGE_KERNEL
));
580 r
= ia64_itr_entry(0x3, KVM_VM_DATA_BASE
,
581 pte
, KVM_VM_DATA_SHIFT
);
584 vcpu
->arch
.vm_tr_slot
= r
;
586 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
587 if (kvm
->arch
.is_sn2
) {
588 r
= kvm_sn2_setup_mappings(vcpu
);
599 static void kvm_purge_vmm_mapping(struct kvm_vcpu
*vcpu
)
601 struct kvm
*kvm
= vcpu
->kvm
;
602 ia64_ptr_entry(0x3, vcpu
->arch
.vmm_tr_slot
);
603 ia64_ptr_entry(0x3, vcpu
->arch
.vm_tr_slot
);
604 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
605 if (kvm
->arch
.is_sn2
)
606 ia64_ptr_entry(0x3, vcpu
->arch
.sn_rtc_tr_slot
);
610 static int kvm_vcpu_pre_transition(struct kvm_vcpu
*vcpu
)
614 int cpu
= smp_processor_id();
616 if (vcpu
->arch
.last_run_cpu
!= cpu
||
617 per_cpu(last_vcpu
, cpu
) != vcpu
) {
618 per_cpu(last_vcpu
, cpu
) = vcpu
;
619 vcpu
->arch
.last_run_cpu
= cpu
;
623 vcpu
->arch
.host_rr6
= ia64_get_rr(RR6
);
624 vti_set_rr6(vcpu
->arch
.vmm_rr
);
626 r
= kvm_insert_vmm_mapping(vcpu
);
627 local_irq_restore(psr
);
631 static void kvm_vcpu_post_transition(struct kvm_vcpu
*vcpu
)
633 kvm_purge_vmm_mapping(vcpu
);
634 vti_set_rr6(vcpu
->arch
.host_rr6
);
637 static int __vcpu_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
639 union context
*host_ctx
, *guest_ctx
;
642 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
645 if (signal_pending(current
)) {
647 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
654 /*Get host and guest context with guest address space.*/
655 host_ctx
= kvm_get_host_context(vcpu
);
656 guest_ctx
= kvm_get_guest_context(vcpu
);
658 clear_bit(KVM_REQ_KICK
, &vcpu
->requests
);
660 r
= kvm_vcpu_pre_transition(vcpu
);
664 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
668 * Transition to the guest
670 kvm_vmm_info
->tramp_entry(host_ctx
, guest_ctx
);
672 kvm_vcpu_post_transition(vcpu
);
674 vcpu
->arch
.launched
= 1;
675 set_bit(KVM_REQ_KICK
, &vcpu
->requests
);
679 * We must have an instruction between local_irq_enable() and
680 * kvm_guest_exit(), so the timer interrupt isn't delayed by
681 * the interrupt shadow. The stat.exits increment will do nicely.
682 * But we need to prevent reordering, hence this barrier():
688 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
690 r
= kvm_handle_exit(kvm_run
, vcpu
);
698 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
701 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
710 kvm_run
->exit_reason
= KVM_EXIT_FAIL_ENTRY
;
714 static void kvm_set_mmio_data(struct kvm_vcpu
*vcpu
)
716 struct kvm_mmio_req
*p
= kvm_get_vcpu_ioreq(vcpu
);
718 if (!vcpu
->mmio_is_write
)
719 memcpy(&p
->data
, vcpu
->mmio_data
, 8);
720 p
->state
= STATE_IORESP_READY
;
723 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
730 if (vcpu
->sigset_active
)
731 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
733 if (unlikely(vcpu
->arch
.mp_state
== KVM_MP_STATE_UNINITIALIZED
)) {
734 kvm_vcpu_block(vcpu
);
735 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
740 if (vcpu
->mmio_needed
) {
741 memcpy(vcpu
->mmio_data
, kvm_run
->mmio
.data
, 8);
742 kvm_set_mmio_data(vcpu
);
743 vcpu
->mmio_read_completed
= 1;
744 vcpu
->mmio_needed
= 0;
746 r
= __vcpu_run(vcpu
, kvm_run
);
748 if (vcpu
->sigset_active
)
749 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
755 static struct kvm
*kvm_alloc_kvm(void)
761 BUG_ON(sizeof(struct kvm
) > KVM_VM_STRUCT_SIZE
);
763 vm_base
= __get_free_pages(GFP_KERNEL
, get_order(KVM_VM_DATA_SIZE
));
766 return ERR_PTR(-ENOMEM
);
768 memset((void *)vm_base
, 0, KVM_VM_DATA_SIZE
);
769 kvm
= (struct kvm
*)(vm_base
+
770 offsetof(struct kvm_vm_data
, kvm_vm_struct
));
771 kvm
->arch
.vm_base
= vm_base
;
772 printk(KERN_DEBUG
"kvm: vm's data area:0x%lx\n", vm_base
);
777 struct kvm_io_range
{
783 static const struct kvm_io_range io_ranges
[] = {
784 {VGA_IO_START
, VGA_IO_SIZE
, GPFN_FRAME_BUFFER
},
785 {MMIO_START
, MMIO_SIZE
, GPFN_LOW_MMIO
},
786 {LEGACY_IO_START
, LEGACY_IO_SIZE
, GPFN_LEGACY_IO
},
787 {IO_SAPIC_START
, IO_SAPIC_SIZE
, GPFN_IOSAPIC
},
788 {PIB_START
, PIB_SIZE
, GPFN_PIB
},
791 static void kvm_build_io_pmt(struct kvm
*kvm
)
795 /* Mark I/O ranges */
796 for (i
= 0; i
< (sizeof(io_ranges
) / sizeof(struct kvm_io_range
));
798 for (j
= io_ranges
[i
].start
;
799 j
< io_ranges
[i
].start
+ io_ranges
[i
].size
;
801 kvm_set_pmt_entry(kvm
, j
>> PAGE_SHIFT
,
802 io_ranges
[i
].type
, 0);
807 /*Use unused rids to virtualize guest rid.*/
808 #define GUEST_PHYSICAL_RR0 0x1739
809 #define GUEST_PHYSICAL_RR4 0x2739
810 #define VMM_INIT_RR 0x1660
812 static void kvm_init_vm(struct kvm
*kvm
)
816 kvm
->arch
.metaphysical_rr0
= GUEST_PHYSICAL_RR0
;
817 kvm
->arch
.metaphysical_rr4
= GUEST_PHYSICAL_RR4
;
818 kvm
->arch
.vmm_init_rr
= VMM_INIT_RR
;
821 *Fill P2M entries for MMIO/IO ranges
823 kvm_build_io_pmt(kvm
);
825 INIT_LIST_HEAD(&kvm
->arch
.assigned_dev_head
);
827 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
828 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID
, &kvm
->arch
.irq_sources_bitmap
);
831 struct kvm
*kvm_arch_create_vm(void)
833 struct kvm
*kvm
= kvm_alloc_kvm();
836 return ERR_PTR(-ENOMEM
);
838 kvm
->arch
.is_sn2
= ia64_platform_is("sn2");
846 static int kvm_vm_ioctl_get_irqchip(struct kvm
*kvm
,
847 struct kvm_irqchip
*chip
)
852 switch (chip
->chip_id
) {
853 case KVM_IRQCHIP_IOAPIC
:
854 r
= kvm_get_ioapic(kvm
, &chip
->chip
.ioapic
);
863 static int kvm_vm_ioctl_set_irqchip(struct kvm
*kvm
, struct kvm_irqchip
*chip
)
868 switch (chip
->chip_id
) {
869 case KVM_IRQCHIP_IOAPIC
:
870 r
= kvm_set_ioapic(kvm
, &chip
->chip
.ioapic
);
879 #define RESTORE_REGS(_x) vcpu->arch._x = regs->_x
881 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
883 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
888 for (i
= 0; i
< 16; i
++) {
889 vpd
->vgr
[i
] = regs
->vpd
.vgr
[i
];
890 vpd
->vbgr
[i
] = regs
->vpd
.vbgr
[i
];
892 for (i
= 0; i
< 128; i
++)
893 vpd
->vcr
[i
] = regs
->vpd
.vcr
[i
];
894 vpd
->vhpi
= regs
->vpd
.vhpi
;
895 vpd
->vnat
= regs
->vpd
.vnat
;
896 vpd
->vbnat
= regs
->vpd
.vbnat
;
897 vpd
->vpsr
= regs
->vpd
.vpsr
;
899 vpd
->vpr
= regs
->vpd
.vpr
;
901 memcpy(&vcpu
->arch
.guest
, ®s
->saved_guest
, sizeof(union context
));
903 RESTORE_REGS(mp_state
);
904 RESTORE_REGS(vmm_rr
);
905 memcpy(vcpu
->arch
.itrs
, regs
->itrs
, sizeof(struct thash_data
) * NITRS
);
906 memcpy(vcpu
->arch
.dtrs
, regs
->dtrs
, sizeof(struct thash_data
) * NDTRS
);
907 RESTORE_REGS(itr_regions
);
908 RESTORE_REGS(dtr_regions
);
909 RESTORE_REGS(tc_regions
);
910 RESTORE_REGS(irq_check
);
911 RESTORE_REGS(itc_check
);
912 RESTORE_REGS(timer_check
);
913 RESTORE_REGS(timer_pending
);
914 RESTORE_REGS(last_itc
);
915 for (i
= 0; i
< 8; i
++) {
916 vcpu
->arch
.vrr
[i
] = regs
->vrr
[i
];
917 vcpu
->arch
.ibr
[i
] = regs
->ibr
[i
];
918 vcpu
->arch
.dbr
[i
] = regs
->dbr
[i
];
920 for (i
= 0; i
< 4; i
++)
921 vcpu
->arch
.insvc
[i
] = regs
->insvc
[i
];
923 RESTORE_REGS(metaphysical_rr0
);
924 RESTORE_REGS(metaphysical_rr4
);
925 RESTORE_REGS(metaphysical_saved_rr0
);
926 RESTORE_REGS(metaphysical_saved_rr4
);
927 RESTORE_REGS(fp_psr
);
928 RESTORE_REGS(saved_gp
);
930 vcpu
->arch
.irq_new_pending
= 1;
931 vcpu
->arch
.itc_offset
= regs
->saved_itc
- kvm_get_itc(vcpu
);
932 set_bit(KVM_REQ_RESUME
, &vcpu
->requests
);
939 long kvm_arch_vm_ioctl(struct file
*filp
,
940 unsigned int ioctl
, unsigned long arg
)
942 struct kvm
*kvm
= filp
->private_data
;
943 void __user
*argp
= (void __user
*)arg
;
947 case KVM_SET_MEMORY_REGION
: {
948 struct kvm_memory_region kvm_mem
;
949 struct kvm_userspace_memory_region kvm_userspace_mem
;
952 if (copy_from_user(&kvm_mem
, argp
, sizeof kvm_mem
))
954 kvm_userspace_mem
.slot
= kvm_mem
.slot
;
955 kvm_userspace_mem
.flags
= kvm_mem
.flags
;
956 kvm_userspace_mem
.guest_phys_addr
=
957 kvm_mem
.guest_phys_addr
;
958 kvm_userspace_mem
.memory_size
= kvm_mem
.memory_size
;
959 r
= kvm_vm_ioctl_set_memory_region(kvm
,
960 &kvm_userspace_mem
, 0);
965 case KVM_CREATE_IRQCHIP
:
967 r
= kvm_ioapic_init(kvm
);
970 r
= kvm_setup_default_irq_routing(kvm
);
972 kvm_ioapic_destroy(kvm
);
976 case KVM_IRQ_LINE_STATUS
:
978 struct kvm_irq_level irq_event
;
981 if (copy_from_user(&irq_event
, argp
, sizeof irq_event
))
984 if (irqchip_in_kernel(kvm
)) {
986 status
= kvm_set_irq(kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
,
987 irq_event
.irq
, irq_event
.level
);
988 if (ioctl
== KVM_IRQ_LINE_STATUS
) {
990 irq_event
.status
= status
;
991 if (copy_to_user(argp
, &irq_event
,
999 case KVM_GET_IRQCHIP
: {
1000 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1001 struct kvm_irqchip chip
;
1004 if (copy_from_user(&chip
, argp
, sizeof chip
))
1007 if (!irqchip_in_kernel(kvm
))
1009 r
= kvm_vm_ioctl_get_irqchip(kvm
, &chip
);
1013 if (copy_to_user(argp
, &chip
, sizeof chip
))
1018 case KVM_SET_IRQCHIP
: {
1019 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1020 struct kvm_irqchip chip
;
1023 if (copy_from_user(&chip
, argp
, sizeof chip
))
1026 if (!irqchip_in_kernel(kvm
))
1028 r
= kvm_vm_ioctl_set_irqchip(kvm
, &chip
);
1041 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
1042 struct kvm_sregs
*sregs
)
1047 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
1048 struct kvm_sregs
*sregs
)
1053 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
1054 struct kvm_translation
*tr
)
1060 static int kvm_alloc_vmm_area(void)
1062 if (!kvm_vmm_base
&& (kvm_vm_buffer_size
< KVM_VM_BUFFER_SIZE
)) {
1063 kvm_vmm_base
= __get_free_pages(GFP_KERNEL
,
1064 get_order(KVM_VMM_SIZE
));
1068 memset((void *)kvm_vmm_base
, 0, KVM_VMM_SIZE
);
1069 kvm_vm_buffer
= kvm_vmm_base
+ VMM_SIZE
;
1071 printk(KERN_DEBUG
"kvm:VMM's Base Addr:0x%lx, vm_buffer:0x%lx\n",
1072 kvm_vmm_base
, kvm_vm_buffer
);
1078 static void kvm_free_vmm_area(void)
1081 /*Zero this area before free to avoid bits leak!!*/
1082 memset((void *)kvm_vmm_base
, 0, KVM_VMM_SIZE
);
1083 free_pages(kvm_vmm_base
, get_order(KVM_VMM_SIZE
));
1090 static int vti_init_vpd(struct kvm_vcpu
*vcpu
)
1093 union cpuid3_t cpuid3
;
1094 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
1097 return PTR_ERR(vpd
);
1100 for (i
= 0; i
< 5; i
++)
1101 vpd
->vcpuid
[i
] = ia64_get_cpuid(i
);
1103 /* Limit the CPUID number to 5 */
1104 cpuid3
.value
= vpd
->vcpuid
[3];
1105 cpuid3
.number
= 4; /* 5 - 1 */
1106 vpd
->vcpuid
[3] = cpuid3
.value
;
1108 /*Set vac and vdc fields*/
1109 vpd
->vac
.a_from_int_cr
= 1;
1110 vpd
->vac
.a_to_int_cr
= 1;
1111 vpd
->vac
.a_from_psr
= 1;
1112 vpd
->vac
.a_from_cpuid
= 1;
1113 vpd
->vac
.a_cover
= 1;
1116 vpd
->vdc
.d_vmsw
= 1;
1118 /*Set virtual buffer*/
1119 vpd
->virt_env_vaddr
= KVM_VM_BUFFER_BASE
;
1124 static int vti_create_vp(struct kvm_vcpu
*vcpu
)
1127 struct vpd
*vpd
= vcpu
->arch
.vpd
;
1128 unsigned long vmm_ivt
;
1130 vmm_ivt
= kvm_vmm_info
->vmm_ivt
;
1132 printk(KERN_DEBUG
"kvm: vcpu:%p,ivt: 0x%lx\n", vcpu
, vmm_ivt
);
1134 ret
= ia64_pal_vp_create((u64
*)vpd
, (u64
*)vmm_ivt
, 0);
1137 printk(KERN_ERR
"kvm: ia64_pal_vp_create failed!\n");
1143 static void init_ptce_info(struct kvm_vcpu
*vcpu
)
1145 ia64_ptce_info_t ptce
= {0};
1147 ia64_get_ptce(&ptce
);
1148 vcpu
->arch
.ptce_base
= ptce
.base
;
1149 vcpu
->arch
.ptce_count
[0] = ptce
.count
[0];
1150 vcpu
->arch
.ptce_count
[1] = ptce
.count
[1];
1151 vcpu
->arch
.ptce_stride
[0] = ptce
.stride
[0];
1152 vcpu
->arch
.ptce_stride
[1] = ptce
.stride
[1];
1155 static void kvm_migrate_hlt_timer(struct kvm_vcpu
*vcpu
)
1157 struct hrtimer
*p_ht
= &vcpu
->arch
.hlt_timer
;
1159 if (hrtimer_cancel(p_ht
))
1160 hrtimer_start_expires(p_ht
, HRTIMER_MODE_ABS
);
1163 static enum hrtimer_restart
hlt_timer_fn(struct hrtimer
*data
)
1165 struct kvm_vcpu
*vcpu
;
1166 wait_queue_head_t
*q
;
1168 vcpu
= container_of(data
, struct kvm_vcpu
, arch
.hlt_timer
);
1171 if (vcpu
->arch
.mp_state
!= KVM_MP_STATE_HALTED
)
1174 if (waitqueue_active(q
))
1175 wake_up_interruptible(q
);
1178 vcpu
->arch
.timer_fired
= 1;
1179 vcpu
->arch
.timer_check
= 1;
1180 return HRTIMER_NORESTART
;
1183 #define PALE_RESET_ENTRY 0x80000000ffffffb0UL
1185 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
1191 struct kvm
*kvm
= vcpu
->kvm
;
1192 struct kvm_pt_regs
*regs
= vcpu_regs(vcpu
);
1194 union context
*p_ctx
= &vcpu
->arch
.guest
;
1195 struct kvm_vcpu
*vmm_vcpu
= to_guest(vcpu
->kvm
, vcpu
);
1197 /*Init vcpu context for first run.*/
1198 if (IS_ERR(vmm_vcpu
))
1199 return PTR_ERR(vmm_vcpu
);
1201 if (kvm_vcpu_is_bsp(vcpu
)) {
1202 vcpu
->arch
.mp_state
= KVM_MP_STATE_RUNNABLE
;
1204 /*Set entry address for first run.*/
1205 regs
->cr_iip
= PALE_RESET_ENTRY
;
1207 /*Initialize itc offset for vcpus*/
1208 itc_offset
= 0UL - kvm_get_itc(vcpu
);
1209 for (i
= 0; i
< KVM_MAX_VCPUS
; i
++) {
1210 v
= (struct kvm_vcpu
*)((char *)vcpu
+
1211 sizeof(struct kvm_vcpu_data
) * i
);
1212 v
->arch
.itc_offset
= itc_offset
;
1213 v
->arch
.last_itc
= 0;
1216 vcpu
->arch
.mp_state
= KVM_MP_STATE_UNINITIALIZED
;
1219 vcpu
->arch
.apic
= kzalloc(sizeof(struct kvm_lapic
), GFP_KERNEL
);
1220 if (!vcpu
->arch
.apic
)
1222 vcpu
->arch
.apic
->vcpu
= vcpu
;
1225 p_ctx
->gr
[12] = (unsigned long)((char *)vmm_vcpu
+ KVM_STK_OFFSET
);
1226 p_ctx
->gr
[13] = (unsigned long)vmm_vcpu
;
1227 p_ctx
->psr
= 0x1008522000UL
;
1228 p_ctx
->ar
[40] = FPSR_DEFAULT
; /*fpsr*/
1229 p_ctx
->caller_unat
= 0;
1231 p_ctx
->ar
[36] = 0x0; /*unat*/
1232 p_ctx
->ar
[19] = 0x0; /*rnat*/
1233 p_ctx
->ar
[18] = (unsigned long)vmm_vcpu
+
1234 ((sizeof(struct kvm_vcpu
)+15) & ~15);
1235 p_ctx
->ar
[64] = 0x0; /*pfs*/
1236 p_ctx
->cr
[0] = 0x7e04UL
;
1237 p_ctx
->cr
[2] = (unsigned long)kvm_vmm_info
->vmm_ivt
;
1238 p_ctx
->cr
[8] = 0x3c;
1240 /*Initilize region register*/
1241 p_ctx
->rr
[0] = 0x30;
1242 p_ctx
->rr
[1] = 0x30;
1243 p_ctx
->rr
[2] = 0x30;
1244 p_ctx
->rr
[3] = 0x30;
1245 p_ctx
->rr
[4] = 0x30;
1246 p_ctx
->rr
[5] = 0x30;
1247 p_ctx
->rr
[7] = 0x30;
1249 /*Initilize branch register 0*/
1250 p_ctx
->br
[0] = *(unsigned long *)kvm_vmm_info
->vmm_entry
;
1252 vcpu
->arch
.vmm_rr
= kvm
->arch
.vmm_init_rr
;
1253 vcpu
->arch
.metaphysical_rr0
= kvm
->arch
.metaphysical_rr0
;
1254 vcpu
->arch
.metaphysical_rr4
= kvm
->arch
.metaphysical_rr4
;
1256 hrtimer_init(&vcpu
->arch
.hlt_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_ABS
);
1257 vcpu
->arch
.hlt_timer
.function
= hlt_timer_fn
;
1259 vcpu
->arch
.last_run_cpu
= -1;
1260 vcpu
->arch
.vpd
= (struct vpd
*)VPD_BASE(vcpu
->vcpu_id
);
1261 vcpu
->arch
.vsa_base
= kvm_vsa_base
;
1262 vcpu
->arch
.__gp
= kvm_vmm_gp
;
1263 vcpu
->arch
.dirty_log_lock_pa
= __pa(&kvm
->arch
.dirty_log_lock
);
1264 vcpu
->arch
.vhpt
.hash
= (struct thash_data
*)VHPT_BASE(vcpu
->vcpu_id
);
1265 vcpu
->arch
.vtlb
.hash
= (struct thash_data
*)VTLB_BASE(vcpu
->vcpu_id
);
1266 init_ptce_info(vcpu
);
1273 static int vti_vcpu_setup(struct kvm_vcpu
*vcpu
, int id
)
1278 local_irq_save(psr
);
1279 r
= kvm_insert_vmm_mapping(vcpu
);
1280 local_irq_restore(psr
);
1283 r
= kvm_vcpu_init(vcpu
, vcpu
->kvm
, id
);
1287 r
= vti_init_vpd(vcpu
);
1289 printk(KERN_DEBUG
"kvm: vpd init error!!\n");
1293 r
= vti_create_vp(vcpu
);
1297 kvm_purge_vmm_mapping(vcpu
);
1301 kvm_vcpu_uninit(vcpu
);
1306 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
1309 struct kvm_vcpu
*vcpu
;
1310 unsigned long vm_base
= kvm
->arch
.vm_base
;
1314 BUG_ON(sizeof(struct kvm_vcpu
) > VCPU_STRUCT_SIZE
/2);
1317 if (id
>= KVM_MAX_VCPUS
) {
1318 printk(KERN_ERR
"kvm: Can't configure vcpus > %ld",
1325 printk(KERN_ERR
"kvm: Create vcpu[%d] error!\n", id
);
1328 vcpu
= (struct kvm_vcpu
*)(vm_base
+ offsetof(struct kvm_vm_data
,
1329 vcpu_data
[id
].vcpu_struct
));
1333 r
= vti_vcpu_setup(vcpu
, id
);
1337 printk(KERN_DEBUG
"kvm: vcpu_setup error!!\n");
1346 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
1351 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1356 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1361 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
1362 struct kvm_guest_debug
*dbg
)
1367 static void free_kvm(struct kvm
*kvm
)
1369 unsigned long vm_base
= kvm
->arch
.vm_base
;
1372 memset((void *)vm_base
, 0, KVM_VM_DATA_SIZE
);
1373 free_pages(vm_base
, get_order(KVM_VM_DATA_SIZE
));
1378 static void kvm_release_vm_pages(struct kvm
*kvm
)
1380 struct kvm_memslots
*slots
;
1381 struct kvm_memory_slot
*memslot
;
1383 unsigned long base_gfn
;
1385 slots
= kvm_memslots(kvm
);
1386 for (i
= 0; i
< slots
->nmemslots
; i
++) {
1387 memslot
= &slots
->memslots
[i
];
1388 base_gfn
= memslot
->base_gfn
;
1390 for (j
= 0; j
< memslot
->npages
; j
++) {
1391 if (memslot
->rmap
[j
])
1392 put_page((struct page
*)memslot
->rmap
[j
]);
1397 void kvm_arch_sync_events(struct kvm
*kvm
)
1401 void kvm_arch_destroy_vm(struct kvm
*kvm
)
1403 kvm_iommu_unmap_guest(kvm
);
1404 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1405 kvm_free_all_assigned_devices(kvm
);
1407 kfree(kvm
->arch
.vioapic
);
1408 kvm_release_vm_pages(kvm
);
1409 kvm_free_physmem(kvm
);
1410 cleanup_srcu_struct(&kvm
->srcu
);
1414 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
1418 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1420 if (cpu
!= vcpu
->cpu
) {
1422 if (vcpu
->arch
.ht_active
)
1423 kvm_migrate_hlt_timer(vcpu
);
1427 #define SAVE_REGS(_x) regs->_x = vcpu->arch._x
1429 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
1431 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
1436 for (i
= 0; i
< 16; i
++) {
1437 regs
->vpd
.vgr
[i
] = vpd
->vgr
[i
];
1438 regs
->vpd
.vbgr
[i
] = vpd
->vbgr
[i
];
1440 for (i
= 0; i
< 128; i
++)
1441 regs
->vpd
.vcr
[i
] = vpd
->vcr
[i
];
1442 regs
->vpd
.vhpi
= vpd
->vhpi
;
1443 regs
->vpd
.vnat
= vpd
->vnat
;
1444 regs
->vpd
.vbnat
= vpd
->vbnat
;
1445 regs
->vpd
.vpsr
= vpd
->vpsr
;
1446 regs
->vpd
.vpr
= vpd
->vpr
;
1448 memcpy(®s
->saved_guest
, &vcpu
->arch
.guest
, sizeof(union context
));
1450 SAVE_REGS(mp_state
);
1452 memcpy(regs
->itrs
, vcpu
->arch
.itrs
, sizeof(struct thash_data
) * NITRS
);
1453 memcpy(regs
->dtrs
, vcpu
->arch
.dtrs
, sizeof(struct thash_data
) * NDTRS
);
1454 SAVE_REGS(itr_regions
);
1455 SAVE_REGS(dtr_regions
);
1456 SAVE_REGS(tc_regions
);
1457 SAVE_REGS(irq_check
);
1458 SAVE_REGS(itc_check
);
1459 SAVE_REGS(timer_check
);
1460 SAVE_REGS(timer_pending
);
1461 SAVE_REGS(last_itc
);
1462 for (i
= 0; i
< 8; i
++) {
1463 regs
->vrr
[i
] = vcpu
->arch
.vrr
[i
];
1464 regs
->ibr
[i
] = vcpu
->arch
.ibr
[i
];
1465 regs
->dbr
[i
] = vcpu
->arch
.dbr
[i
];
1467 for (i
= 0; i
< 4; i
++)
1468 regs
->insvc
[i
] = vcpu
->arch
.insvc
[i
];
1469 regs
->saved_itc
= vcpu
->arch
.itc_offset
+ kvm_get_itc(vcpu
);
1471 SAVE_REGS(metaphysical_rr0
);
1472 SAVE_REGS(metaphysical_rr4
);
1473 SAVE_REGS(metaphysical_saved_rr0
);
1474 SAVE_REGS(metaphysical_saved_rr4
);
1476 SAVE_REGS(saved_gp
);
1482 int kvm_arch_vcpu_ioctl_get_stack(struct kvm_vcpu
*vcpu
,
1483 struct kvm_ia64_vcpu_stack
*stack
)
1485 memcpy(stack
, vcpu
, sizeof(struct kvm_ia64_vcpu_stack
));
1489 int kvm_arch_vcpu_ioctl_set_stack(struct kvm_vcpu
*vcpu
,
1490 struct kvm_ia64_vcpu_stack
*stack
)
1492 memcpy(vcpu
+ 1, &stack
->stack
[0] + sizeof(struct kvm_vcpu
),
1493 sizeof(struct kvm_ia64_vcpu_stack
) - sizeof(struct kvm_vcpu
));
1495 vcpu
->arch
.exit_data
= ((struct kvm_vcpu
*)stack
)->arch
.exit_data
;
1499 void kvm_arch_vcpu_uninit(struct kvm_vcpu
*vcpu
)
1502 hrtimer_cancel(&vcpu
->arch
.hlt_timer
);
1503 kfree(vcpu
->arch
.apic
);
1507 long kvm_arch_vcpu_ioctl(struct file
*filp
,
1508 unsigned int ioctl
, unsigned long arg
)
1510 struct kvm_vcpu
*vcpu
= filp
->private_data
;
1511 void __user
*argp
= (void __user
*)arg
;
1512 struct kvm_ia64_vcpu_stack
*stack
= NULL
;
1516 case KVM_IA64_VCPU_GET_STACK
: {
1517 struct kvm_ia64_vcpu_stack __user
*user_stack
;
1518 void __user
*first_p
= argp
;
1521 if (copy_from_user(&user_stack
, first_p
, sizeof(void *)))
1524 if (!access_ok(VERIFY_WRITE
, user_stack
,
1525 sizeof(struct kvm_ia64_vcpu_stack
))) {
1526 printk(KERN_INFO
"KVM_IA64_VCPU_GET_STACK: "
1527 "Illegal user destination address for stack\n");
1530 stack
= kzalloc(sizeof(struct kvm_ia64_vcpu_stack
), GFP_KERNEL
);
1536 r
= kvm_arch_vcpu_ioctl_get_stack(vcpu
, stack
);
1540 if (copy_to_user(user_stack
, stack
,
1541 sizeof(struct kvm_ia64_vcpu_stack
))) {
1548 case KVM_IA64_VCPU_SET_STACK
: {
1549 struct kvm_ia64_vcpu_stack __user
*user_stack
;
1550 void __user
*first_p
= argp
;
1553 if (copy_from_user(&user_stack
, first_p
, sizeof(void *)))
1556 if (!access_ok(VERIFY_READ
, user_stack
,
1557 sizeof(struct kvm_ia64_vcpu_stack
))) {
1558 printk(KERN_INFO
"KVM_IA64_VCPU_SET_STACK: "
1559 "Illegal user address for stack\n");
1562 stack
= kmalloc(sizeof(struct kvm_ia64_vcpu_stack
), GFP_KERNEL
);
1567 if (copy_from_user(stack
, user_stack
,
1568 sizeof(struct kvm_ia64_vcpu_stack
)))
1571 r
= kvm_arch_vcpu_ioctl_set_stack(vcpu
, stack
);
1584 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
1585 struct kvm_memory_slot
*memslot
,
1586 struct kvm_memory_slot old
,
1587 struct kvm_userspace_memory_region
*mem
,
1592 int npages
= memslot
->npages
;
1593 unsigned long base_gfn
= memslot
->base_gfn
;
1595 if (base_gfn
+ npages
> (KVM_MAX_MEM_SIZE
>> PAGE_SHIFT
))
1598 for (i
= 0; i
< npages
; i
++) {
1599 pfn
= gfn_to_pfn(kvm
, base_gfn
+ i
);
1600 if (!kvm_is_mmio_pfn(pfn
)) {
1601 kvm_set_pmt_entry(kvm
, base_gfn
+ i
,
1603 _PAGE_AR_RWX
| _PAGE_MA_WB
);
1604 memslot
->rmap
[i
] = (unsigned long)pfn_to_page(pfn
);
1606 kvm_set_pmt_entry(kvm
, base_gfn
+ i
,
1607 GPFN_PHYS_MMIO
| (pfn
<< PAGE_SHIFT
),
1609 memslot
->rmap
[i
] = 0;
1616 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
1617 struct kvm_userspace_memory_region
*mem
,
1618 struct kvm_memory_slot old
,
1624 void kvm_arch_flush_shadow(struct kvm
*kvm
)
1626 kvm_flush_remote_tlbs(kvm
);
1629 long kvm_arch_dev_ioctl(struct file
*filp
,
1630 unsigned int ioctl
, unsigned long arg
)
1635 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
1637 kvm_vcpu_uninit(vcpu
);
1640 static int vti_cpu_has_kvm_support(void)
1642 long avail
= 1, status
= 1, control
= 1;
1645 ret
= ia64_pal_proc_get_features(&avail
, &status
, &control
, 0);
1649 if (!(avail
& PAL_PROC_VM_BIT
))
1652 printk(KERN_DEBUG
"kvm: Hardware Supports VT\n");
1654 ret
= ia64_pal_vp_env_info(&kvm_vm_buffer_size
, &vp_env_info
);
1657 printk(KERN_DEBUG
"kvm: VM Buffer Size:0x%lx\n", kvm_vm_buffer_size
);
1659 if (!(vp_env_info
& VP_OPCODE
)) {
1660 printk(KERN_WARNING
"kvm: No opcode ability on hardware, "
1661 "vm_env_info:0x%lx\n", vp_env_info
);
1671 * On SN2, the ITC isn't stable, so copy in fast path code to use the
1672 * SN2 RTC, replacing the ITC based default verion.
1674 static void kvm_patch_vmm(struct kvm_vmm_info
*vmm_info
,
1675 struct module
*module
)
1677 unsigned long new_ar
, new_ar_sn2
;
1678 unsigned long module_base
;
1680 if (!ia64_platform_is("sn2"))
1683 module_base
= (unsigned long)module
->module_core
;
1685 new_ar
= kvm_vmm_base
+ vmm_info
->patch_mov_ar
- module_base
;
1686 new_ar_sn2
= kvm_vmm_base
+ vmm_info
->patch_mov_ar_sn2
- module_base
;
1688 printk(KERN_INFO
"kvm: Patching ITC emulation to use SGI SN2 RTC "
1692 * Copy the SN2 version of mov_ar into place. They are both
1693 * the same size, so 6 bundles is sufficient (6 * 0x10).
1695 memcpy((void *)new_ar
, (void *)new_ar_sn2
, 0x60);
1698 static int kvm_relocate_vmm(struct kvm_vmm_info
*vmm_info
,
1699 struct module
*module
)
1701 unsigned long module_base
;
1702 unsigned long vmm_size
;
1704 unsigned long vmm_offset
, func_offset
, fdesc_offset
;
1705 struct fdesc
*p_fdesc
;
1709 if (!kvm_vmm_base
) {
1710 printk("kvm: kvm area hasn't been initilized yet!!\n");
1714 /*Calculate new position of relocated vmm module.*/
1715 module_base
= (unsigned long)module
->module_core
;
1716 vmm_size
= module
->core_size
;
1717 if (unlikely(vmm_size
> KVM_VMM_SIZE
))
1720 memcpy((void *)kvm_vmm_base
, (void *)module_base
, vmm_size
);
1721 kvm_patch_vmm(vmm_info
, module
);
1722 kvm_flush_icache(kvm_vmm_base
, vmm_size
);
1724 /*Recalculate kvm_vmm_info based on new VMM*/
1725 vmm_offset
= vmm_info
->vmm_ivt
- module_base
;
1726 kvm_vmm_info
->vmm_ivt
= KVM_VMM_BASE
+ vmm_offset
;
1727 printk(KERN_DEBUG
"kvm: Relocated VMM's IVT Base Addr:%lx\n",
1728 kvm_vmm_info
->vmm_ivt
);
1730 fdesc_offset
= (unsigned long)vmm_info
->vmm_entry
- module_base
;
1731 kvm_vmm_info
->vmm_entry
= (kvm_vmm_entry
*)(KVM_VMM_BASE
+
1733 func_offset
= *(unsigned long *)vmm_info
->vmm_entry
- module_base
;
1734 p_fdesc
= (struct fdesc
*)(kvm_vmm_base
+ fdesc_offset
);
1735 p_fdesc
->ip
= KVM_VMM_BASE
+ func_offset
;
1736 p_fdesc
->gp
= KVM_VMM_BASE
+(p_fdesc
->gp
- module_base
);
1738 printk(KERN_DEBUG
"kvm: Relocated VMM's Init Entry Addr:%lx\n",
1739 KVM_VMM_BASE
+func_offset
);
1741 fdesc_offset
= (unsigned long)vmm_info
->tramp_entry
- module_base
;
1742 kvm_vmm_info
->tramp_entry
= (kvm_tramp_entry
*)(KVM_VMM_BASE
+
1744 func_offset
= *(unsigned long *)vmm_info
->tramp_entry
- module_base
;
1745 p_fdesc
= (struct fdesc
*)(kvm_vmm_base
+ fdesc_offset
);
1746 p_fdesc
->ip
= KVM_VMM_BASE
+ func_offset
;
1747 p_fdesc
->gp
= KVM_VMM_BASE
+ (p_fdesc
->gp
- module_base
);
1749 kvm_vmm_gp
= p_fdesc
->gp
;
1751 printk(KERN_DEBUG
"kvm: Relocated VMM's Entry IP:%p\n",
1752 kvm_vmm_info
->vmm_entry
);
1753 printk(KERN_DEBUG
"kvm: Relocated VMM's Trampoline Entry IP:0x%lx\n",
1754 KVM_VMM_BASE
+ func_offset
);
1759 int kvm_arch_init(void *opaque
)
1762 struct kvm_vmm_info
*vmm_info
= (struct kvm_vmm_info
*)opaque
;
1764 if (!vti_cpu_has_kvm_support()) {
1765 printk(KERN_ERR
"kvm: No Hardware Virtualization Support!\n");
1771 printk(KERN_ERR
"kvm: Already loaded VMM module!\n");
1777 kvm_vmm_info
= kzalloc(sizeof(struct kvm_vmm_info
), GFP_KERNEL
);
1781 if (kvm_alloc_vmm_area())
1784 r
= kvm_relocate_vmm(vmm_info
, vmm_info
->module
);
1791 kvm_free_vmm_area();
1793 kfree(kvm_vmm_info
);
1798 void kvm_arch_exit(void)
1800 kvm_free_vmm_area();
1801 kfree(kvm_vmm_info
);
1802 kvm_vmm_info
= NULL
;
1805 static int kvm_ia64_sync_dirty_log(struct kvm
*kvm
,
1806 struct kvm_dirty_log
*log
)
1808 struct kvm_memory_slot
*memslot
;
1812 unsigned long *dirty_bitmap
= (unsigned long *)(kvm
->arch
.vm_base
+
1813 offsetof(struct kvm_vm_data
, kvm_mem_dirty_log
));
1816 if (log
->slot
>= KVM_MEMORY_SLOTS
)
1819 memslot
= &kvm
->memslots
->memslots
[log
->slot
];
1821 if (!memslot
->dirty_bitmap
)
1824 n
= kvm_dirty_bitmap_bytes(memslot
);
1825 base
= memslot
->base_gfn
/ BITS_PER_LONG
;
1827 for (i
= 0; i
< n
/sizeof(long); ++i
) {
1828 memslot
->dirty_bitmap
[i
] = dirty_bitmap
[base
+ i
];
1829 dirty_bitmap
[base
+ i
] = 0;
1836 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
1837 struct kvm_dirty_log
*log
)
1841 struct kvm_memory_slot
*memslot
;
1844 mutex_lock(&kvm
->slots_lock
);
1845 spin_lock(&kvm
->arch
.dirty_log_lock
);
1847 r
= kvm_ia64_sync_dirty_log(kvm
, log
);
1851 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
1855 /* If nothing is dirty, don't bother messing with page tables. */
1857 kvm_flush_remote_tlbs(kvm
);
1858 memslot
= &kvm
->memslots
->memslots
[log
->slot
];
1859 n
= kvm_dirty_bitmap_bytes(memslot
);
1860 memset(memslot
->dirty_bitmap
, 0, n
);
1864 mutex_unlock(&kvm
->slots_lock
);
1865 spin_unlock(&kvm
->arch
.dirty_log_lock
);
1869 int kvm_arch_hardware_setup(void)
1874 void kvm_arch_hardware_unsetup(void)
1878 void kvm_vcpu_kick(struct kvm_vcpu
*vcpu
)
1881 int cpu
= vcpu
->cpu
;
1883 if (waitqueue_active(&vcpu
->wq
))
1884 wake_up_interruptible(&vcpu
->wq
);
1887 if (cpu
!= me
&& (unsigned) cpu
< nr_cpu_ids
&& cpu_online(cpu
))
1888 if (!test_and_set_bit(KVM_REQ_KICK
, &vcpu
->requests
))
1889 smp_send_reschedule(cpu
);
1893 int kvm_apic_set_irq(struct kvm_vcpu
*vcpu
, struct kvm_lapic_irq
*irq
)
1895 return __apic_accept_irq(vcpu
, irq
->vector
);
1898 int kvm_apic_match_physical_addr(struct kvm_lapic
*apic
, u16 dest
)
1900 return apic
->vcpu
->vcpu_id
== dest
;
1903 int kvm_apic_match_logical_addr(struct kvm_lapic
*apic
, u8 mda
)
1908 int kvm_apic_compare_prio(struct kvm_vcpu
*vcpu1
, struct kvm_vcpu
*vcpu2
)
1910 return vcpu1
->arch
.xtp
- vcpu2
->arch
.xtp
;
1913 int kvm_apic_match_dest(struct kvm_vcpu
*vcpu
, struct kvm_lapic
*source
,
1914 int short_hand
, int dest
, int dest_mode
)
1916 struct kvm_lapic
*target
= vcpu
->arch
.apic
;
1917 return (dest_mode
== 0) ?
1918 kvm_apic_match_physical_addr(target
, dest
) :
1919 kvm_apic_match_logical_addr(target
, dest
);
1922 static int find_highest_bits(int *dat
)
1927 /* loop for all 256 bits */
1928 for (i
= 7; i
>= 0 ; i
--) {
1932 return i
* 32 + bitnum
- 1;
1939 int kvm_highest_pending_irq(struct kvm_vcpu
*vcpu
)
1941 struct vpd
*vpd
= to_host(vcpu
->kvm
, vcpu
->arch
.vpd
);
1943 if (vpd
->irr
[0] & (1UL << NMI_VECTOR
))
1945 if (vpd
->irr
[0] & (1UL << ExtINT_VECTOR
))
1946 return ExtINT_VECTOR
;
1948 return find_highest_bits((int *)&vpd
->irr
[0]);
1951 int kvm_cpu_has_pending_timer(struct kvm_vcpu
*vcpu
)
1953 return vcpu
->arch
.timer_fired
;
1956 gfn_t
unalias_gfn(struct kvm
*kvm
, gfn_t gfn
)
1961 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
1963 return (vcpu
->arch
.mp_state
== KVM_MP_STATE_RUNNABLE
) ||
1964 (kvm_highest_pending_irq(vcpu
) != -1);
1967 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
1968 struct kvm_mp_state
*mp_state
)
1971 mp_state
->mp_state
= vcpu
->arch
.mp_state
;
1976 static int vcpu_reset(struct kvm_vcpu
*vcpu
)
1980 local_irq_save(psr
);
1981 r
= kvm_insert_vmm_mapping(vcpu
);
1982 local_irq_restore(psr
);
1986 vcpu
->arch
.launched
= 0;
1987 kvm_arch_vcpu_uninit(vcpu
);
1988 r
= kvm_arch_vcpu_init(vcpu
);
1992 kvm_purge_vmm_mapping(vcpu
);
1998 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
1999 struct kvm_mp_state
*mp_state
)
2004 vcpu
->arch
.mp_state
= mp_state
->mp_state
;
2005 if (vcpu
->arch
.mp_state
== KVM_MP_STATE_UNINITIALIZED
)
2006 r
= vcpu_reset(vcpu
);