2 * Copyright (c) 2009, Microsoft Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
18 * Haiyang Zhang <haiyangz@microsoft.com>
19 * Hank Janssen <hjanssen@microsoft.com>
22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24 #include <linux/kernel.h>
26 #include <linux/slab.h>
27 #include <linux/vmalloc.h>
28 #include <linux/hyperv.h>
29 #include <linux/version.h>
30 #include <linux/interrupt.h>
31 #include <linux/clockchips.h>
32 #include <asm/hyperv.h>
33 #include <asm/mshyperv.h>
34 #include "hyperv_vmbus.h"
36 /* The one and only */
37 struct hv_context hv_context
= {
38 .synic_initialized
= false,
39 .hypercall_page
= NULL
,
42 #define HV_TIMER_FREQUENCY (10 * 1000 * 1000) /* 100ns period */
43 #define HV_MAX_MAX_DELTA_TICKS 0xffffffff
44 #define HV_MIN_DELTA_TICKS 1
47 * query_hypervisor_info - Get version info of the windows hypervisor
49 unsigned int host_info_eax
;
50 unsigned int host_info_ebx
;
51 unsigned int host_info_ecx
;
52 unsigned int host_info_edx
;
54 static int query_hypervisor_info(void)
60 unsigned int max_leaf
;
64 * Its assumed that this is called after confirming that Viridian
65 * is present. Query id and revision.
71 op
= HVCPUID_VENDOR_MAXFUNCTION
;
72 cpuid(op
, &eax
, &ebx
, &ecx
, &edx
);
76 if (max_leaf
>= HVCPUID_VERSION
) {
82 cpuid(op
, &eax
, &ebx
, &ecx
, &edx
);
92 * hv_do_hypercall- Invoke the specified hypercall
94 u64
hv_do_hypercall(u64 control
, void *input
, void *output
)
96 u64 input_address
= (input
) ? virt_to_phys(input
) : 0;
97 u64 output_address
= (output
) ? virt_to_phys(output
) : 0;
98 void *hypercall_page
= hv_context
.hypercall_page
;
103 return (u64
)ULLONG_MAX
;
105 __asm__
__volatile__("mov %0, %%r8" : : "r" (output_address
) : "r8");
106 __asm__
__volatile__("call *%3" : "=a" (hv_status
) :
107 "c" (control
), "d" (input_address
),
108 "m" (hypercall_page
));
114 u32 control_hi
= control
>> 32;
115 u32 control_lo
= control
& 0xFFFFFFFF;
116 u32 hv_status_hi
= 1;
117 u32 hv_status_lo
= 1;
118 u32 input_address_hi
= input_address
>> 32;
119 u32 input_address_lo
= input_address
& 0xFFFFFFFF;
120 u32 output_address_hi
= output_address
>> 32;
121 u32 output_address_lo
= output_address
& 0xFFFFFFFF;
124 return (u64
)ULLONG_MAX
;
126 __asm__
__volatile__ ("call *%8" : "=d"(hv_status_hi
),
127 "=a"(hv_status_lo
) : "d" (control_hi
),
128 "a" (control_lo
), "b" (input_address_hi
),
129 "c" (input_address_lo
), "D"(output_address_hi
),
130 "S"(output_address_lo
), "m" (hypercall_page
));
132 return hv_status_lo
| ((u64
)hv_status_hi
<< 32);
135 EXPORT_SYMBOL_GPL(hv_do_hypercall
);
138 static u64
read_hv_clock_tsc(struct clocksource
*arg
)
141 struct ms_hyperv_tsc_page
*tsc_pg
= hv_context
.tsc_page
;
143 if (tsc_pg
->tsc_sequence
!= 0) {
145 * Use the tsc page to compute the value.
150 u32 sequence
= tsc_pg
->tsc_sequence
;
152 u64 scale
= tsc_pg
->tsc_scale
;
153 s64 offset
= tsc_pg
->tsc_offset
;
156 /* current_tick = ((cur_tsc *scale) >> 64) + offset */
158 : "=d" (current_tick
), "=a" (tmp
)
159 : "a" (cur_tsc
), "r" (scale
));
161 current_tick
+= offset
;
162 if (tsc_pg
->tsc_sequence
== sequence
)
165 if (tsc_pg
->tsc_sequence
!= 0)
168 * Fallback using MSR method.
173 rdmsrl(HV_X64_MSR_TIME_REF_COUNT
, current_tick
);
177 static struct clocksource hyperv_cs_tsc
= {
178 .name
= "hyperv_clocksource_tsc_page",
180 .read
= read_hv_clock_tsc
,
181 .mask
= CLOCKSOURCE_MASK(64),
182 .flags
= CLOCK_SOURCE_IS_CONTINUOUS
,
188 * hv_init - Main initialization routine.
190 * This routine must be called before any other routines in here are called
195 union hv_x64_msr_hypercall_contents hypercall_msr
;
196 void *virtaddr
= NULL
;
198 memset(hv_context
.synic_event_page
, 0, sizeof(void *) * NR_CPUS
);
199 memset(hv_context
.synic_message_page
, 0,
200 sizeof(void *) * NR_CPUS
);
201 memset(hv_context
.post_msg_page
, 0,
202 sizeof(void *) * NR_CPUS
);
203 memset(hv_context
.vp_index
, 0,
204 sizeof(int) * NR_CPUS
);
205 memset(hv_context
.event_dpc
, 0,
206 sizeof(void *) * NR_CPUS
);
207 memset(hv_context
.msg_dpc
, 0,
208 sizeof(void *) * NR_CPUS
);
209 memset(hv_context
.clk_evt
, 0,
210 sizeof(void *) * NR_CPUS
);
212 max_leaf
= query_hypervisor_info();
217 hv_context
.guestid
= generate_guest_id(0, LINUX_VERSION_CODE
, 0);
218 wrmsrl(HV_X64_MSR_GUEST_OS_ID
, hv_context
.guestid
);
220 /* See if the hypercall page is already set */
221 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
223 virtaddr
= __vmalloc(PAGE_SIZE
, GFP_KERNEL
, PAGE_KERNEL_EXEC
);
228 hypercall_msr
.enable
= 1;
230 hypercall_msr
.guest_physical_address
= vmalloc_to_pfn(virtaddr
);
231 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
233 /* Confirm that hypercall page did get setup. */
234 hypercall_msr
.as_uint64
= 0;
235 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
237 if (!hypercall_msr
.enable
)
240 hv_context
.hypercall_page
= virtaddr
;
243 if (ms_hyperv
.features
& HV_X64_MSR_REFERENCE_TSC_AVAILABLE
) {
244 union hv_x64_msr_hypercall_contents tsc_msr
;
247 va_tsc
= __vmalloc(PAGE_SIZE
, GFP_KERNEL
, PAGE_KERNEL
);
250 hv_context
.tsc_page
= va_tsc
;
252 rdmsrl(HV_X64_MSR_REFERENCE_TSC
, tsc_msr
.as_uint64
);
255 tsc_msr
.guest_physical_address
= vmalloc_to_pfn(va_tsc
);
257 wrmsrl(HV_X64_MSR_REFERENCE_TSC
, tsc_msr
.as_uint64
);
258 clocksource_register_hz(&hyperv_cs_tsc
, NSEC_PER_SEC
/100);
265 if (hypercall_msr
.enable
) {
266 hypercall_msr
.as_uint64
= 0;
267 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
277 * hv_cleanup - Cleanup routine.
279 * This routine is called normally during driver unloading or exiting.
281 void hv_cleanup(bool crash
)
283 union hv_x64_msr_hypercall_contents hypercall_msr
;
285 /* Reset our OS id */
286 wrmsrl(HV_X64_MSR_GUEST_OS_ID
, 0);
288 if (hv_context
.hypercall_page
) {
289 hypercall_msr
.as_uint64
= 0;
290 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
292 vfree(hv_context
.hypercall_page
);
293 hv_context
.hypercall_page
= NULL
;
298 * Cleanup the TSC page based CS.
300 if (ms_hyperv
.features
& HV_X64_MSR_REFERENCE_TSC_AVAILABLE
) {
302 * Crash can happen in an interrupt context and unregistering
303 * a clocksource is impossible and redundant in this case.
305 if (!oops_in_progress
) {
306 clocksource_change_rating(&hyperv_cs_tsc
, 10);
307 clocksource_unregister(&hyperv_cs_tsc
);
310 hypercall_msr
.as_uint64
= 0;
311 wrmsrl(HV_X64_MSR_REFERENCE_TSC
, hypercall_msr
.as_uint64
);
313 vfree(hv_context
.tsc_page
);
314 hv_context
.tsc_page
= NULL
;
320 * hv_post_message - Post a message using the hypervisor message IPC.
322 * This involves a hypercall.
324 int hv_post_message(union hv_connection_id connection_id
,
325 enum hv_message_type message_type
,
326 void *payload
, size_t payload_size
)
329 struct hv_input_post_message
*aligned_msg
;
332 if (payload_size
> HV_MESSAGE_PAYLOAD_BYTE_COUNT
)
335 aligned_msg
= (struct hv_input_post_message
*)
336 hv_context
.post_msg_page
[get_cpu()];
338 aligned_msg
->connectionid
= connection_id
;
339 aligned_msg
->reserved
= 0;
340 aligned_msg
->message_type
= message_type
;
341 aligned_msg
->payload_size
= payload_size
;
342 memcpy((void *)aligned_msg
->payload
, payload
, payload_size
);
344 status
= hv_do_hypercall(HVCALL_POST_MESSAGE
, aligned_msg
, NULL
);
347 return status
& 0xFFFF;
350 static int hv_ce_set_next_event(unsigned long delta
,
351 struct clock_event_device
*evt
)
355 WARN_ON(!clockevent_state_oneshot(evt
));
357 rdmsrl(HV_X64_MSR_TIME_REF_COUNT
, current_tick
);
358 current_tick
+= delta
;
359 wrmsrl(HV_X64_MSR_STIMER0_COUNT
, current_tick
);
363 static int hv_ce_shutdown(struct clock_event_device
*evt
)
365 wrmsrl(HV_X64_MSR_STIMER0_COUNT
, 0);
366 wrmsrl(HV_X64_MSR_STIMER0_CONFIG
, 0);
371 static int hv_ce_set_oneshot(struct clock_event_device
*evt
)
373 union hv_timer_config timer_cfg
;
375 timer_cfg
.enable
= 1;
376 timer_cfg
.auto_enable
= 1;
377 timer_cfg
.sintx
= VMBUS_MESSAGE_SINT
;
378 wrmsrl(HV_X64_MSR_STIMER0_CONFIG
, timer_cfg
.as_uint64
);
383 static void hv_init_clockevent_device(struct clock_event_device
*dev
, int cpu
)
385 dev
->name
= "Hyper-V clockevent";
386 dev
->features
= CLOCK_EVT_FEAT_ONESHOT
;
387 dev
->cpumask
= cpumask_of(cpu
);
390 * Avoid settint dev->owner = THIS_MODULE deliberately as doing so will
391 * result in clockevents_config_and_register() taking additional
392 * references to the hv_vmbus module making it impossible to unload.
395 dev
->set_state_shutdown
= hv_ce_shutdown
;
396 dev
->set_state_oneshot
= hv_ce_set_oneshot
;
397 dev
->set_next_event
= hv_ce_set_next_event
;
401 int hv_synic_alloc(void)
403 size_t size
= sizeof(struct tasklet_struct
);
404 size_t ced_size
= sizeof(struct clock_event_device
);
407 hv_context
.hv_numa_map
= kzalloc(sizeof(struct cpumask
) * nr_node_ids
,
409 if (hv_context
.hv_numa_map
== NULL
) {
410 pr_err("Unable to allocate NUMA map\n");
414 for_each_online_cpu(cpu
) {
415 hv_context
.event_dpc
[cpu
] = kmalloc(size
, GFP_ATOMIC
);
416 if (hv_context
.event_dpc
[cpu
] == NULL
) {
417 pr_err("Unable to allocate event dpc\n");
420 tasklet_init(hv_context
.event_dpc
[cpu
], vmbus_on_event
, cpu
);
422 hv_context
.msg_dpc
[cpu
] = kmalloc(size
, GFP_ATOMIC
);
423 if (hv_context
.msg_dpc
[cpu
] == NULL
) {
424 pr_err("Unable to allocate event dpc\n");
427 tasklet_init(hv_context
.msg_dpc
[cpu
], vmbus_on_msg_dpc
, cpu
);
429 hv_context
.clk_evt
[cpu
] = kzalloc(ced_size
, GFP_ATOMIC
);
430 if (hv_context
.clk_evt
[cpu
] == NULL
) {
431 pr_err("Unable to allocate clock event device\n");
435 hv_init_clockevent_device(hv_context
.clk_evt
[cpu
], cpu
);
437 hv_context
.synic_message_page
[cpu
] =
438 (void *)get_zeroed_page(GFP_ATOMIC
);
440 if (hv_context
.synic_message_page
[cpu
] == NULL
) {
441 pr_err("Unable to allocate SYNIC message page\n");
445 hv_context
.synic_event_page
[cpu
] =
446 (void *)get_zeroed_page(GFP_ATOMIC
);
448 if (hv_context
.synic_event_page
[cpu
] == NULL
) {
449 pr_err("Unable to allocate SYNIC event page\n");
453 hv_context
.post_msg_page
[cpu
] =
454 (void *)get_zeroed_page(GFP_ATOMIC
);
456 if (hv_context
.post_msg_page
[cpu
] == NULL
) {
457 pr_err("Unable to allocate post msg page\n");
467 static void hv_synic_free_cpu(int cpu
)
469 kfree(hv_context
.event_dpc
[cpu
]);
470 kfree(hv_context
.msg_dpc
[cpu
]);
471 kfree(hv_context
.clk_evt
[cpu
]);
472 if (hv_context
.synic_event_page
[cpu
])
473 free_page((unsigned long)hv_context
.synic_event_page
[cpu
]);
474 if (hv_context
.synic_message_page
[cpu
])
475 free_page((unsigned long)hv_context
.synic_message_page
[cpu
]);
476 if (hv_context
.post_msg_page
[cpu
])
477 free_page((unsigned long)hv_context
.post_msg_page
[cpu
]);
480 void hv_synic_free(void)
484 kfree(hv_context
.hv_numa_map
);
485 for_each_online_cpu(cpu
)
486 hv_synic_free_cpu(cpu
);
490 * hv_synic_init - Initialize the Synthethic Interrupt Controller.
492 * If it is already initialized by another entity (ie x2v shim), we need to
493 * retrieve the initialized message and event pages. Otherwise, we create and
494 * initialize the message and event pages.
496 void hv_synic_init(void *arg
)
499 union hv_synic_simp simp
;
500 union hv_synic_siefp siefp
;
501 union hv_synic_sint shared_sint
;
502 union hv_synic_scontrol sctrl
;
505 int cpu
= smp_processor_id();
507 if (!hv_context
.hypercall_page
)
510 /* Check the version */
511 rdmsrl(HV_X64_MSR_SVERSION
, version
);
513 /* Setup the Synic's message page */
514 rdmsrl(HV_X64_MSR_SIMP
, simp
.as_uint64
);
515 simp
.simp_enabled
= 1;
516 simp
.base_simp_gpa
= virt_to_phys(hv_context
.synic_message_page
[cpu
])
519 wrmsrl(HV_X64_MSR_SIMP
, simp
.as_uint64
);
521 /* Setup the Synic's event page */
522 rdmsrl(HV_X64_MSR_SIEFP
, siefp
.as_uint64
);
523 siefp
.siefp_enabled
= 1;
524 siefp
.base_siefp_gpa
= virt_to_phys(hv_context
.synic_event_page
[cpu
])
527 wrmsrl(HV_X64_MSR_SIEFP
, siefp
.as_uint64
);
529 /* Setup the shared SINT. */
530 rdmsrl(HV_X64_MSR_SINT0
+ VMBUS_MESSAGE_SINT
, shared_sint
.as_uint64
);
532 shared_sint
.as_uint64
= 0;
533 shared_sint
.vector
= HYPERVISOR_CALLBACK_VECTOR
;
534 shared_sint
.masked
= false;
535 shared_sint
.auto_eoi
= true;
537 wrmsrl(HV_X64_MSR_SINT0
+ VMBUS_MESSAGE_SINT
, shared_sint
.as_uint64
);
539 /* Enable the global synic bit */
540 rdmsrl(HV_X64_MSR_SCONTROL
, sctrl
.as_uint64
);
543 wrmsrl(HV_X64_MSR_SCONTROL
, sctrl
.as_uint64
);
545 hv_context
.synic_initialized
= true;
548 * Setup the mapping between Hyper-V's notion
549 * of cpuid and Linux' notion of cpuid.
550 * This array will be indexed using Linux cpuid.
552 rdmsrl(HV_X64_MSR_VP_INDEX
, vp_index
);
553 hv_context
.vp_index
[cpu
] = (u32
)vp_index
;
555 INIT_LIST_HEAD(&hv_context
.percpu_list
[cpu
]);
558 * Register the per-cpu clockevent source.
560 if (ms_hyperv
.features
& HV_X64_MSR_SYNTIMER_AVAILABLE
)
561 clockevents_config_and_register(hv_context
.clk_evt
[cpu
],
564 HV_MAX_MAX_DELTA_TICKS
);
569 * hv_synic_clockevents_cleanup - Cleanup clockevent devices
571 void hv_synic_clockevents_cleanup(void)
575 if (!(ms_hyperv
.features
& HV_X64_MSR_SYNTIMER_AVAILABLE
))
578 for_each_present_cpu(cpu
)
579 clockevents_unbind_device(hv_context
.clk_evt
[cpu
], cpu
);
583 * hv_synic_cleanup - Cleanup routine for hv_synic_init().
585 void hv_synic_cleanup(void *arg
)
587 union hv_synic_sint shared_sint
;
588 union hv_synic_simp simp
;
589 union hv_synic_siefp siefp
;
590 union hv_synic_scontrol sctrl
;
591 int cpu
= smp_processor_id();
593 if (!hv_context
.synic_initialized
)
596 /* Turn off clockevent device */
597 if (ms_hyperv
.features
& HV_X64_MSR_SYNTIMER_AVAILABLE
) {
598 clockevents_unbind_device(hv_context
.clk_evt
[cpu
], cpu
);
599 hv_ce_shutdown(hv_context
.clk_evt
[cpu
]);
602 rdmsrl(HV_X64_MSR_SINT0
+ VMBUS_MESSAGE_SINT
, shared_sint
.as_uint64
);
604 shared_sint
.masked
= 1;
606 /* Need to correctly cleanup in the case of SMP!!! */
607 /* Disable the interrupt */
608 wrmsrl(HV_X64_MSR_SINT0
+ VMBUS_MESSAGE_SINT
, shared_sint
.as_uint64
);
610 rdmsrl(HV_X64_MSR_SIMP
, simp
.as_uint64
);
611 simp
.simp_enabled
= 0;
612 simp
.base_simp_gpa
= 0;
614 wrmsrl(HV_X64_MSR_SIMP
, simp
.as_uint64
);
616 rdmsrl(HV_X64_MSR_SIEFP
, siefp
.as_uint64
);
617 siefp
.siefp_enabled
= 0;
618 siefp
.base_siefp_gpa
= 0;
620 wrmsrl(HV_X64_MSR_SIEFP
, siefp
.as_uint64
);
622 /* Disable the global synic bit */
623 rdmsrl(HV_X64_MSR_SCONTROL
, sctrl
.as_uint64
);
625 wrmsrl(HV_X64_MSR_SCONTROL
, sctrl
.as_uint64
);