| blob | a8e7bc04d9bf365277332f0174b617da5747da45 |
1 // SPDX-License-Identifier: GPL-2.0
4 #include <linux/objtool.h>
5 #include <linux/percpu.h>
7 #include <asm/debugreg.h>
8 #include <asm/mmu_context.h>
28 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
30 /*
31 * Hyper-V requires all of these, so mark them as supported even though
32 * they are just treated the same as all-context.
33 */
34 #define VMX_VPID_EXTENT_SUPPORTED_MASK \
35 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \
36 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \
37 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \
38 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
40 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
43 VMX_VMREAD_BITMAP,
44 VMX_VMWRITE_BITMAP,
45 VMX_BITMAP_NR
46 };
49 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP])
50 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP])
53 u16 encoding;
54 u16 offset;
55 };
57 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
59 };
64 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
66 };
71 {
89 #ifdef CONFIG_X86_64
91 #else
93 #endif
95 }
112 /*
113 * PML and the preemption timer can be emulated, but the
114 * processor cannot vmwrite to fields that don't exist
115 * on bare metal.
116 */
132 }
137 #ifdef CONFIG_X86_64
139 #else
141 #endif
143 }
145 }
147 /*
148 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
149 * set the success or error code of an emulated VMX instruction (as specified
150 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated
151 * instruction.
152 */
154 {
159 }
162 {
168 }
171 u32 vm_instruction_error)
172 {
178 /*
179 * We don't need to force sync to shadow VMCS because
180 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all
181 * fields and thus must be synced.
182 */
187 }
190 {
193 /*
194 * failValid writes the error number to the current VMCS, which
195 * can't be done if there isn't a current VMCS.
196 */
202 }
205 {
206 /* TODO: not to reset guest simply here. */
209 }
212 {
214 }
217 {
219 }
222 {
226 }
229 {
230 #ifdef CONFIG_KVM_HYPERV
237 }
245 }
246 #endif
247 }
250 {
251 #ifdef CONFIG_KVM_HYPERV
253 /*
254 * When Enlightened VMEntry is enabled on the calling CPU we treat
255 * memory area pointer by vmptr as Enlightened VMCS (as there's no good
256 * way to distinguish it from VMCS12) and we must not corrupt it by
257 * writing to the non-existent 'launch_state' field. The area doesn't
258 * have to be the currently active EVMCS on the calling CPU and there's
259 * nothing KVM has to do to transition it from 'active' to 'non-active'
260 * state. It is possible that the area will stay mapped as
261 * vmx->nested.hv_evmcs but this shouldn't be a problem.
262 */
271 #else
273 #endif
274 }
278 {
289 #ifdef CONFIG_X86_64
292 #endif
293 }
296 {
313 /*
314 * All lazily updated registers will be reloaded from VMCS12 on both
315 * vmentry and vmexit.
316 */
318 }
320 /*
321 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
322 * just stops using VMX.
323 */
325 {
347 }
352 /*
353 * Unpin physical memory we referred to in the vmcs02. The APIC access
354 * page's backing page (yeah, confusing) shouldn't actually be accessed,
355 * and if it is written, the contents are irrelevant.
356 */
367 }
369 /*
370 * Ensure that the current vmcs of the logical processor is the
371 * vmcs01 of the vcpu before calling free_nested().
372 */
374 {
378 }
380 #define EPTP_PA_MASK GENMASK_ULL(51, 12)
383 {
386 }
389 gpa_t addr)
390 {
392 uint i;
401 eptp))
403 }
406 }
410 {
414 u32 vm_exit_reason;
420 /*
421 * It should be impossible to trigger a nested PML Full VM-Exit
422 * for anything other than an EPT Violation from L2. KVM *can*
423 * trigger nEPT page fault injection in response to an EPT
424 * Misconfig, e.g. if the MMIO SPTE was stale and L1's EPT
425 * tables also changed, but KVM should not treat EPT Misconfig
426 * VM-Exits as writes.
427 */
430 /*
431 * PML Full and EPT Violation VM-Exits both use bit 12 to report
432 * "NMI unblocking due to IRET", i.e. the bit can be propagated
433 * as-is from the original EXIT_QUALIFICATION.
434 */
444 EPT_VIOLATION_GVA_TRANSLATED);
446 }
448 /*
449 * Although the caller (kvm_inject_emulated_page_fault) would
450 * have already synced the faulting address in the shadow EPT
451 * tables for the current EPTP12, we also need to sync it for
452 * any other cached EPTP02s based on the same EP4TA, since the
453 * TLB associates mappings to the EP4TA rather than the full EPTP.
454 */
457 }
461 }
464 {
472 }
475 {
485 }
488 {
491 }
494 u16 error_code)
495 {
499 inequality =
503 }
506 u32 error_code)
507 {
510 /*
511 * Drop bits 31:16 of the error code when performing the #PF mask+match
512 * check. All VMCS fields involved are 32 bits, but Intel CPUs never
513 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected
514 * error code. Including the to-be-dropped bits in the check might
515 * result in an "impossible" or missed exit from L1's perspective.
516 */
521 }
525 {
534 }
538 {
546 }
550 {
558 }
560 /*
561 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1
562 * itself utilizing x2APIC. All MSRs were previously set to be intercepted,
563 * only the "disable intercept" case needs to be handled.
564 */
568 {
574 }
577 {
585 }
586 }
588 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \
589 static inline \
590 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \
591 unsigned long *msr_bitmap_l1, \
592 unsigned long *msr_bitmap_l0, u32 msr) \
593 { \
594 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \
595 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \
596 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \
597 else \
598 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \
599 }
607 {
614 }
616 /*
617 * Merge L0's and L1's MSR bitmap, return false to indicate that
618 * we do not use the hardware.
619 */
622 {
629 /* Nothing to do if the MSR bitmap is not in use. */
634 /*
635 * MSR bitmap update can be skipped when:
636 * - MSR bitmap for L1 hasn't changed.
637 * - Nested hypervisor (L1) is attempting to launch the same L2 as
638 * before.
639 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature
640 * and tells KVM (L0) there were no changes in MSR bitmap for L2.
641 */
648 }
655 /*
656 * To keep the control flow simple, pay eight 8-byte writes (sixteen
657 * 4-byte writes on 32-bit systems) up front to enable intercepts for
658 * the x2APIC MSR range and selectively toggle those relevant to L2.
659 */
664 /*
665 * L0 need not intercept reads for MSRs between 0x800
666 * and 0x8ff, it just lets the processor take the value
667 * from the virtual-APIC page; take those 256 bits
668 * directly from the L1 bitmap.
669 */
674 }
675 }
686 MSR_TYPE_W);
690 MSR_TYPE_W);
691 }
692 }
694 /*
695 * Always check vmcs01's bitmap to honor userspace MSR filters and any
696 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through.
697 */
698 #ifdef CONFIG_X86_64
707 #endif
722 }
726 {
740 VMCS12_SIZE);
741 }
745 {
759 VMCS12_SIZE);
760 }
762 /*
763 * In nested virtualization, check if L1 has set
764 * VM_EXIT_ACK_INTR_ON_EXIT
765 */
767 {
769 VM_EXIT_ACK_INTR_ON_EXIT;
770 }
774 {
778 else
780 }
784 {
791 /*
792 * If virtualize x2apic mode is enabled,
793 * virtualize apic access must be disabled.
794 */
799 /*
800 * If virtual interrupt delivery is enabled,
801 * we must exit on external interrupts.
802 */
806 /*
807 * bits 15:8 should be zero in posted_intr_nv,
808 * the descriptor address has been already checked
809 * in nested_get_vmcs12_pages.
810 *
811 * bits 5:0 of posted_intr_desc_addr should be zero.
812 */
820 /* tpr shadow is needed by all apicv features. */
825 }
829 {
838 }
842 {
852 }
856 {
863 }
867 {
876 }
880 {
885 }
889 {
894 }
898 {
907 }
911 {
912 /* x2APIC MSR accesses are not allowed */
921 }
925 {
932 }
936 {
941 }
944 {
950 }
952 /*
953 * Load guest's/host's msr at nested entry/exit.
954 * return 0 for success, entry index for failure.
955 *
956 * One of the failure modes for MSR load/store is when a list exceeds the
957 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch
958 * as possible, process all valid entries before failing rather than precheck
959 * for a capacity violation.
960 */
962 {
963 u32 i;
977 }
983 }
989 }
990 }
992 fail:
993 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */
995 }
998 u32 msr_index,
1000 {
1003 /*
1004 * If the L0 hypervisor stored a more accurate value for the TSC that
1005 * does not include the time taken for emulation of the L2->L1
1006 * VM-exit in L0, use the more accurate value.
1007 */
1010 MSR_IA32_TSC);
1017 }
1018 }
1022 msr_index);
1024 }
1026 }
1030 {
1038 }
1044 }
1046 }
1049 {
1050 u64 data;
1051 u32 i;
1073 }
1074 }
1076 }
1079 {
1084 u32 i;
1092 }
1094 }
1097 u32 msr_index)
1098 {
1112 /*
1113 * Emulated VMEntry does not fail here. Instead a less
1114 * accurate value will be returned by
1115 * nested_vmx_get_vmexit_msr_value() by reading KVM's
1116 * internal MSR state instead of reading the value from
1117 * the vmcs02 VMExit MSR-store area.
1118 */
1121 msr_index);
1123 }
1129 }
1130 }
1132 /*
1133 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are
1134 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected
1135 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to
1136 * @entry_failure_code.
1137 */
1141 {
1145 }
1147 /*
1148 * If PAE paging and EPT are both on, CR3 is not used by the CPU and
1149 * must not be dereferenced.
1150 */
1155 }
1160 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
1167 }
1169 /*
1170 * Returns if KVM is able to config CPU to tag TLB entries
1171 * populated by L2 differently than TLB entries populated
1172 * by L1.
1173 *
1174 * If L0 uses EPT, L1 and L2 run with different EPTP because
1175 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries
1176 * are tagged with different EPTP.
1177 *
1178 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged
1179 * with different VPID (L1 entries are tagged with vmx->vpid
1180 * while L2 entries are tagged with vmx->nested.vpid02).
1181 */
1183 {
1188 }
1193 {
1196 /* Handle pending Hyper-V TLB flush requests */
1199 /*
1200 * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings
1201 * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a
1202 * full TLB flush from the guest's perspective. This is required even
1203 * if VPID is disabled in the host as KVM may need to synchronize the
1204 * MMU in response to the guest TLB flush.
1205 *
1206 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use.
1207 * EPT is a special snowflake, as guest-physical mappings aren't
1208 * flushed on VPID invalidations, including VM-Enter or VM-Exit with
1209 * VPID disabled. As a result, KVM _never_ needs to sync nEPT
1210 * entries on VM-Enter because L1 can't rely on VM-Enter to flush
1211 * those mappings.
1212 */
1216 }
1218 /* L2 should never have a VPID if VPID is disabled. */
1221 /*
1222 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then
1223 * emulate a guest TLB flush as KVM does not track vpid12 history nor
1224 * is the VPID incorporated into the MMU context. I.e. KVM must assume
1225 * that the new vpid12 has never been used and thus represents a new
1226 * guest ASID that cannot have entries in the TLB.
1227 */
1232 }
1234 /*
1235 * If VPID is enabled, used by vmc12, and vpid12 is not changing but
1236 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and
1237 * KVM was unable to allocate a VPID for L2, flush the current context
1238 * as the effective ASID is common to both L1 and L2.
1239 */
1242 }
1245 {
1250 }
1253 {
1255 VMX_BASIC_INOUT |
1256 VMX_BASIC_TRUE_CTLS;
1266 /*
1267 * Except for 32BIT_PHYS_ADDR_ONLY, which is an anti-feature bit (has
1268 * inverted polarity), the incoming value must not set feature bits or
1269 * reserved bits that aren't allowed/supported by KVM. Fields, i.e.
1270 * multi-bit values, are explicitly checked below.
1271 */
1275 /*
1276 * KVM does not emulate a version of VMX that constrains physical
1277 * addresses of VMX structures (e.g. VMCS) to 32-bits.
1278 */
1291 }
1295 {
1319 }
1320 }
1322 static int
1324 {
1326 u64 supported;
1332 /* Check must-be-1 bits are still 1. */
1336 /* Check must-be-0 bits are still 0. */
1344 }
1347 {
1349 VMX_MISC_ACTIVITY_HLT |
1350 VMX_MISC_ACTIVITY_SHUTDOWN |
1351 VMX_MISC_ACTIVITY_WAIT_SIPI |
1352 VMX_MISC_INTEL_PT |
1353 VMX_MISC_RDMSR_IN_SMM |
1354 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
1355 VMX_MISC_VMXOFF_BLOCK_SMI |
1356 VMX_MISC_ZERO_LEN_INS;
1365 /*
1366 * The incoming value must not set feature bits or reserved bits that
1367 * aren't allowed/supported by KVM. Fields, i.e. multi-bit values, are
1368 * explicitly checked below.
1369 */
1374 PIN_BASED_VMX_PREEMPTION_TIMER) &&
1392 }
1395 {
1399 /* Every bit is either reserved or a feature bit. */
1406 }
1409 {
1417 }
1418 }
1421 {
1424 /*
1425 * 1 bits (which indicates bits which "must-be-1" during VMX operation)
1426 * must be 1 in the restored value.
1427 */
1433 }
1435 /*
1436 * Called when userspace is restoring VMX MSRs.
1437 *
1438 * Returns 0 on success, non-0 otherwise.
1439 */
1441 {
1444 /*
1445 * Don't allow changes to the VMX capability MSRs while the vCPU
1446 * is in VMX operation.
1447 */
1458 /*
1459 * The "non-true" VMX capability MSRs are generated from the
1460 * "true" MSRs, so we do not support restoring them directly.
1461 *
1462 * If userspace wants to emulate VMX_BASIC[55]=0, userspace
1463 * should restore the "true" MSRs with the must-be-1 bits
1464 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
1465 * DEFAULT SETTINGS".
1466 */
1481 /*
1482 * These MSRs are generated based on the vCPU's CPUID, so we
1483 * do not support restoring them directly.
1484 */
1497 /*
1498 * The rest of the VMX capability MSRs do not support restore.
1499 */
1501 }
1502 }
1504 /* Returns 0 on success, non-0 otherwise. */
1506 {
1577 }
1580 }
1582 /*
1583 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have
1584 * been modified by the L1 guest. Note, "writable" in this context means
1585 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of
1586 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only"
1587 * VM-exit information fields (which are actually writable if the vCPU is
1588 * configured to support "VMWRITE to any supported field in the VMCS").
1589 */
1591 {
1609 }
1615 }
1618 {
1620 shadow_read_write_fields,
1621 shadow_read_only_fields
1622 };
1624 max_shadow_read_write_fields,
1625 max_shadow_read_only_fields
1626 };
1644 }
1645 }
1649 }
1652 {
1653 #ifdef CONFIG_KVM_HYPERV
1658 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */
1663 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) {
1667 }
1670 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
1675 /*
1676 * Not present in struct vmcs12:
1677 * vmcs12->guest_ssp = evmcs->guest_ssp;
1678 */
1679 }
1682 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
1685 }
1688 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
1690 }
1693 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
1695 }
1698 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
1705 }
1708 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
1726 /*
1727 * Not present in struct vmcs12:
1728 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet;
1729 * vmcs12->host_ssp = evmcs->host_ssp;
1730 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr;
1731 */
1732 }
1735 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
1741 }
1744 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
1747 }
1750 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
1752 }
1755 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
1792 }
1795 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
1801 }
1804 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
1813 }
1816 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
1823 }
1826 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
1829 }
1832 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
1849 /*
1850 * Not present in struct vmcs12:
1851 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet;
1852 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl;
1853 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr;
1854 */
1855 }
1857 /*
1858 * Not used?
1859 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr;
1860 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr;
1861 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr;
1862 * vmcs12->page_fault_error_code_mask =
1863 * evmcs->page_fault_error_code_mask;
1864 * vmcs12->page_fault_error_code_match =
1865 * evmcs->page_fault_error_code_match;
1866 * vmcs12->cr3_target_count = evmcs->cr3_target_count;
1867 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count;
1868 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count;
1869 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count;
1870 */
1872 /*
1873 * Read only fields:
1874 * vmcs12->guest_physical_address = evmcs->guest_physical_address;
1875 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error;
1876 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason;
1877 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info;
1878 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code;
1879 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field;
1880 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code;
1881 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len;
1882 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info;
1883 * vmcs12->exit_qualification = evmcs->exit_qualification;
1884 * vmcs12->guest_linear_address = evmcs->guest_linear_address;
1885 *
1886 * Not present in struct vmcs12:
1887 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx;
1888 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi;
1889 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi;
1890 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip;
1891 */
1897 }
1900 {
1901 #ifdef CONFIG_KVM_HYPERV
1905 /*
1906 * Should not be changed by KVM:
1907 *
1908 * evmcs->host_es_selector = vmcs12->host_es_selector;
1909 * evmcs->host_cs_selector = vmcs12->host_cs_selector;
1910 * evmcs->host_ss_selector = vmcs12->host_ss_selector;
1911 * evmcs->host_ds_selector = vmcs12->host_ds_selector;
1912 * evmcs->host_fs_selector = vmcs12->host_fs_selector;
1913 * evmcs->host_gs_selector = vmcs12->host_gs_selector;
1914 * evmcs->host_tr_selector = vmcs12->host_tr_selector;
1915 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat;
1916 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer;
1917 * evmcs->host_cr0 = vmcs12->host_cr0;
1918 * evmcs->host_cr3 = vmcs12->host_cr3;
1919 * evmcs->host_cr4 = vmcs12->host_cr4;
1920 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp;
1921 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip;
1922 * evmcs->host_rip = vmcs12->host_rip;
1923 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs;
1924 * evmcs->host_fs_base = vmcs12->host_fs_base;
1925 * evmcs->host_gs_base = vmcs12->host_gs_base;
1926 * evmcs->host_tr_base = vmcs12->host_tr_base;
1927 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base;
1928 * evmcs->host_idtr_base = vmcs12->host_idtr_base;
1929 * evmcs->host_rsp = vmcs12->host_rsp;
1930 * sync_vmcs02_to_vmcs12() doesn't read these:
1931 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a;
1932 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b;
1933 * evmcs->msr_bitmap = vmcs12->msr_bitmap;
1934 * evmcs->ept_pointer = vmcs12->ept_pointer;
1935 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap;
1936 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr;
1937 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr;
1938 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr;
1939 * evmcs->tpr_threshold = vmcs12->tpr_threshold;
1940 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id;
1941 * evmcs->exception_bitmap = vmcs12->exception_bitmap;
1942 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer;
1943 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control;
1944 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls;
1945 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control;
1946 * evmcs->page_fault_error_code_mask =
1947 * vmcs12->page_fault_error_code_mask;
1948 * evmcs->page_fault_error_code_match =
1949 * vmcs12->page_fault_error_code_match;
1950 * evmcs->cr3_target_count = vmcs12->cr3_target_count;
1951 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr;
1952 * evmcs->tsc_offset = vmcs12->tsc_offset;
1953 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl;
1954 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask;
1955 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask;
1956 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow;
1957 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow;
1958 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count;
1959 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count;
1960 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count;
1961 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl;
1962 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl;
1963 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap;
1964 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier;
1965 *
1966 * Not present in struct vmcs12:
1967 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx;
1968 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi;
1969 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi;
1970 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip;
1971 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet;
1972 * evmcs->host_ssp = vmcs12->host_ssp;
1973 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr;
1974 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet;
1975 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl;
1976 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr;
1977 * evmcs->guest_ssp = vmcs12->guest_ssp;
1978 */
2075 }
2077 /*
2078 * This is an equivalent of the nested hypervisor executing the vmptrld
2079 * instruction.
2080 */
2083 {
2084 #ifdef CONFIG_KVM_HYPERV
2087 u64 evmcs_gpa;
2096 }
2109 /*
2110 * Currently, KVM only supports eVMCS version 1
2111 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this
2112 * value to first u32 field of eVMCS which should specify eVMCS
2113 * VersionNumber.
2114 *
2115 * Guest should be aware of supported eVMCS versions by host by
2116 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is
2117 * expected to set this CPUID leaf according to the value
2118 * returned in vmcs_version from nested_enable_evmcs().
2119 *
2120 * However, it turns out that Microsoft Hyper-V fails to comply
2121 * to their own invented interface: When Hyper-V use eVMCS, it
2122 * just sets first u32 field of eVMCS to revision_id specified
2123 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number
2124 * which is one of the supported versions specified in
2125 * CPUID.0x4000000A.EAX[0:15].
2126 *
2127 * To overcome Hyper-V bug, we accept here either a supported
2128 * eVMCS version or VMCS12 revision_id as valid values for first
2129 * u32 field of eVMCS.
2130 */
2135 }
2140 /*
2141 * Unlike normal vmcs12, enlightened vmcs12 is not fully
2142 * reloaded from guest's memory (read only fields, fields not
2143 * present in struct hv_enlightened_vmcs, ...). Make sure there
2144 * are no leftovers.
2145 */
2150 }
2152 }
2154 /*
2155 * Clean fields data can't be used on VMLAUNCH and when we switch
2156 * between different L2 guests as KVM keeps a single VMCS12 per L1.
2157 */
2163 }
2166 #else
2168 #endif
2169 }
2172 {
2177 else
2181 }
2184 {
2193 }
2196 {
2201 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
2207 }
2209 }
2212 u64 preemption_timeout)
2213 {
2216 /*
2217 * A timer value of zero is architecturally guaranteed to cause
2218 * a VMExit prior to executing any instructions in the guest.
2219 */
2223 }
2233 HRTIMER_MODE_ABS_PINNED);
2234 }
2237 {
2243 else
2245 }
2248 {
2251 /*
2252 * If vmcs02 hasn't been initialized, set the constant vmcs02 state
2253 * according to L0's settings (vmcs12 is irrelevant here). Host
2254 * fields that come from L0 and are not constant, e.g. HOST_CR3,
2255 * will be set as needed prior to VMLAUNCH/VMRESUME.
2256 */
2261 /*
2262 * We don't care what the EPTP value is we just need to guarantee
2263 * it's valid so we don't get a false positive when doing early
2264 * consistency checks.
2265 */
2273 /* All VMFUNCs are currently emulated through L0 vmexits. */
2283 /*
2284 * PML is emulated for L2, but never enabled in hardware as the MMU
2285 * handles A/D emulation. Disabling PML for L2 also avoids having to
2286 * deal with filtering out L2 GPAs from the buffer.
2287 */
2291 }
2299 /*
2300 * Set the MSR load/store lists to match L0's settings. Only the
2301 * addresses are constant (for vmcs02), the counts can change based
2302 * on L2's behavior, e.g. switching to/from long mode.
2303 */
2309 }
2313 {
2321 else
2323 }
2324 }
2328 {
2329 u32 exec_control;
2335 /*
2336 * PIN CONTROLS
2337 */
2342 /* Posted interrupts setting is only taken from vmcs12. */
2349 }
2352 /*
2353 * EXEC CONTROLS
2354 */
2364 #ifdef CONFIG_X86_64
2365 else
2367 CPU_BASED_CR8_STORE_EXITING;
2368 #endif
2370 /*
2371 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
2372 * for I/O port accesses.
2373 */
2377 /*
2378 * This bit will be computed in nested_get_vmcs12_pages, because
2379 * we do not have access to L1's MSR bitmap yet. For now, keep
2380 * the same bit as before, hoping to avoid multiple VMWRITEs that
2381 * only set/clear this bit.
2382 */
2388 /*
2389 * SECONDARY EXEC CONTROLS
2390 */
2394 /* Take the following fields only from vmcs12 */
2396 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2397 SECONDARY_EXEC_ENABLE_INVPCID |
2398 SECONDARY_EXEC_ENABLE_RDTSCP |
2399 SECONDARY_EXEC_ENABLE_XSAVES |
2400 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2401 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2402 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2403 SECONDARY_EXEC_ENABLE_VMFUNC |
2404 SECONDARY_EXEC_DESC);
2407 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
2410 /* PML is emulated and never enabled in hardware for L2. */
2413 /* VMCS shadowing for L2 is emulated for now */
2416 /*
2417 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4()
2418 * will not have to rewrite the controls just for this bit.
2419 */
2434 }
2436 /*
2437 * ENTRY CONTROLS
2438 *
2439 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
2440 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate
2441 * on the related bits (if supported by the CPU) in the hope that
2442 * we can avoid VMWrites during vmx_set_efer().
2443 *
2444 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is
2445 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to
2446 * do the same for L2.
2447 */
2457 }
2460 /*
2461 * EXIT CONTROLS
2462 *
2463 * L2->L1 exit controls are emulated - the hardware exit is to L0 so
2464 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
2465 * bits may be modified by vmx_set_efer() in prepare_vmcs02().
2466 */
2470 else
2474 /*
2475 * Interrupt/Exception Fields
2476 */
2490 }
2491 }
2494 {
2498 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
2538 }
2541 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
2548 /*
2549 * L1 may access the L2's PDPTR, so save them to construct
2550 * vmcs12
2551 */
2557 }
2562 }
2567 /*
2568 * Whether page-faults are trapped is determined by a combination of
2569 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0
2570 * doesn't care about page faults then we should set all of these to
2571 * L1's desires. However, if L0 does care about (some) page faults, it
2572 * is not easy (if at all possible?) to merge L0 and L1's desires, we
2573 * simply ask to exit on each and every L2 page fault. This is done by
2574 * setting MASK=MATCH=0 and (see below) EB.PF=1.
2575 * Note that below we don't need special code to set EB.PF beyond the
2576 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
2577 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
2578 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
2579 */
2581 /*
2582 * TODO: if both L0 and L1 need the same MASK and MATCH,
2583 * go ahead and use it?
2584 */
2590 }
2597 }
2599 /*
2600 * Make sure the msr_autostore list is up to date before we set the
2601 * count in the vmcs02.
2602 */
2610 }
2612 /*
2613 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
2614 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
2615 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
2616 * guest in a way that will both be appropriate to L1's requests, and our
2617 * needs. In addition to modifying the active vmcs (which is vmcs02), this
2618 * function also has additional necessary side-effects, like setting various
2619 * vcpu->arch fields.
2620 * Returns 0 on success, 1 on failure. Invalid state exit qualification code
2621 * is assigned to entry_failure_code on failure.
2622 */
2626 {
2637 }
2646 }
2652 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
2653 * bitwise-or of what L1 wants to trap for L2, and what we want to
2654 * trap. Note that CR0.TS also needs updating - we do this later.
2655 */
2666 }
2686 /*
2687 * Override the CR0/CR4 read shadows after setting the effective guest
2688 * CR0/CR4. The common helpers also set the shadows, but they don't
2689 * account for vmcs12's cr0/4_guest_host_mask.
2690 */
2698 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
2701 /*
2702 * Guest state is invalid and unrestricted guest is disabled,
2703 * which means L1 attempted VMEntry to L2 with invalid state.
2704 * Fail the VMEntry.
2705 *
2706 * However when force loading the guest state (SMM exit or
2707 * loading nested state after migration, it is possible to
2708 * have invalid guest state now, which will be later fixed by
2709 * restoring L2 register state
2710 */
2714 }
2716 /* Shadow page tables on either EPT or shadow page tables. */
2721 /*
2722 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12
2723 * on nested VM-Exit, which can occur without actually running L2 and
2724 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with
2725 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the
2726 * transition to HLT instead of running L2.
2727 */
2731 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */
2738 }
2746 }
2751 /*
2752 * It was observed that genuine Hyper-V running in L1 doesn't reset
2753 * 'hv_clean_fields' by itself, it only sets the corresponding dirty
2754 * bits when it changes a field in eVMCS. Mark all fields as clean
2755 * here.
2756 */
2761 }
2764 {
2774 }
2777 {
2780 /* Check for memory type validity */
2792 }
2794 /* Page-walk levels validity. */
2806 }
2808 /* Reserved bits should not be set */
2812 /* AD, if set, should be supported */
2816 }
2819 }
2821 /*
2822 * Checks related to VM-Execution Control Fields
2823 */
2826 {
2874 }
2875 }
2878 }
2880 /*
2881 * Checks related to VM-Exit Control Fields
2882 */
2885 {
2895 }
2897 /*
2898 * Checks related to VM-Entry Control Fields
2899 */
2902 {
2910 /*
2911 * From the Intel SDM, volume 3:
2912 * Fields relevant to VM-entry event injection must be set properly.
2913 * These fields are the VM-entry interruption-information field, the
2914 * VM-entry exception error code, and the VM-entry instruction length.
2915 */
2923 SECONDARY_EXEC_UNRESTRICTED_GUEST);
2926 /* VM-entry interruption-info field: interruption type */
2932 /* VM-entry interruption-info field: vector */
2938 /* VM-entry interruption-info field: deliver error code */
2939 should_have_error_code =
2945 /* VM-entry exception error code */
2950 /* VM-entry interruption-info field: reserved bits */
2954 /* VM-entry instruction length */
2963 }
2964 }
2970 }
2974 {
2980 #ifdef CONFIG_KVM_HYPERV
2983 #endif
2986 }
2990 {
2991 #ifdef CONFIG_X86_64
2995 #endif
2997 }
3001 {
3030 }
3052 /*
3053 * If the load IA32_EFER VM-exit control is 1, bits reserved in the
3054 * IA32_EFER MSR must be 0 in the field for that register. In addition,
3055 * the values of the LMA and LME bits in the field must each be that of
3056 * the host address-space size VM-exit control.
3057 */
3063 }
3066 }
3070 {
3096 }
3098 /*
3099 * Checks related to Guest Non-register State
3100 */
3102 {
3109 }
3114 {
3134 }
3148 /*
3149 * If the load IA32_EFER VM-entry control is 1, the following checks
3150 * are performed on the field for the IA32_EFER MSR:
3151 * - Bits reserved in the IA32_EFER MSR must be 0.
3152 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
3153 * the IA-32e mode guest VM-exit control. It must also be identical
3154 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
3155 * CR0.PG) is 1.
3156 */
3164 }
3175 }
3178 {
3195 /*
3196 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
3197 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to
3198 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e.
3199 * there is no need to preserve other bits or save/restore the field.
3200 */
3207 }
3213 }
3232 }
3234 /*
3235 * VMExit clears RFLAGS.IF and DR7, even on a consistency check.
3236 */
3242 /*
3243 * A non-failing VMEntry means we somehow entered guest mode with
3244 * an illegal RIP, and that's just the tip of the iceberg. There
3245 * is no telling what memory has been modified or what state has
3246 * been exposed to unknown code. Hitting this all but guarantees
3247 * a (very critical) hardware issue.
3248 */
3250 VMX_EXIT_REASONS_FAILED_VMENTRY));
3253 }
3255 #ifdef CONFIG_KVM_HYPERV
3257 {
3260 /*
3261 * hv_evmcs may end up being not mapped after migration (when
3262 * L2 was running), map it here to make sure vmcs12 changes are
3263 * properly reflected.
3264 */
3274 /*
3275 * Post migration VMCS12 always provides the most actual
3276 * information, copy it to eVMCS upon entry.
3277 */
3279 }
3282 }
3283 #endif
3286 {
3293 /*
3294 * Reload the guest's PDPTRs since after a migration
3295 * the guest CR3 might be restored prior to setting the nested
3296 * state which can lead to a load of wrong PDPTRs.
3297 */
3300 }
3310 __func__);
3313 KVM_INTERNAL_ERROR_EMULATION;
3316 }
3317 }
3327 /*
3328 * The processor will never use the TPR shadow, simply
3329 * clear the bit from the execution control. Such a
3330 * configuration is useless, but it happens in tests.
3331 * For any other configuration, failing the vm entry is
3332 * _not_ what the processor does but it's basically the
3333 * only possibility we have.
3334 */
3337 /*
3338 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to
3339 * force VM-Entry to fail.
3340 */
3342 }
3343 }
3355 /*
3356 * Defer the KVM_INTERNAL_EXIT until KVM tries to
3357 * access the contents of the VMCS12 posted interrupt
3358 * descriptor. (Note that KVM may do this when it
3359 * should not, per the architectural specification.)
3360 */
3363 }
3364 }
3367 else
3371 }
3374 {
3375 #ifdef CONFIG_KVM_HYPERV
3376 /*
3377 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy
3378 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory
3379 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post
3380 * migration.
3381 */
3384 __func__);
3387 KVM_INTERNAL_ERROR_EMULATION;
3391 }
3392 #endif
3398 }
3401 {
3404 gpa_t dst;
3412 /*
3413 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is
3414 * set is already checked as part of A/D emulation.
3415 */
3423 }
3435 }
3437 /*
3438 * Intel's VMX Instruction Reference specifies a common set of prerequisites
3439 * for running VMX instructions (except VMXON, whose prerequisites are
3440 * slightly different). It also specifies what exception to inject otherwise.
3441 * Note that many of these exceptions have priority over VM exits, so they
3442 * don't have to be checked again here.
3443 */
3445 {
3449 }
3454 }
3457 }
3460 {
3465 }
3470 /*
3471 * If from_vmentry is false, this is being called from state restore (either RSM
3472 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume.
3473 *
3474 * Returns:
3475 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode
3476 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail
3477 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit
3478 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error
3479 */
3482 {
3490 };
3491 u32 failed_index;
3501 KVM_ISA_VMX);
3520 /*
3521 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and*
3522 * nested early checks are disabled. In the event of a "late" VM-Fail,
3523 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its
3524 * software model to the pre-VMEntry host state. When EPT is disabled,
3525 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes
3526 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing
3527 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to
3528 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested
3529 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is
3530 * guaranteed to be overwritten with a shadow CR3 prior to re-entering
3531 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
3532 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
3533 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
3534 * path would need to manually save/restore vmcs01.GUEST_CR3.
3535 */
3547 }
3552 }
3559 }
3560 }
3568 }
3578 }
3580 /*
3581 * The MMU is not initialized to point at the right entities yet and
3582 * "get pages" would need to read data from the guest (i.e. we will
3583 * need to perform gpa to hpa translation). Request a call
3584 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs
3585 * have already been set at vmentry time and should not be reset.
3586 */
3588 }
3590 /*
3591 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI
3592 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can
3593 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit
3594 * unconditionally.
3595 */
3599 /*
3600 * Do not start the preemption timer hrtimer until after we know
3601 * we are successful, so that only nested_vmx_vmexit needs to cancel
3602 * the timer.
3603 */
3608 }
3610 /*
3611 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
3612 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
3613 * returned as far as L1 is concerned. It will only return (and set
3614 * the success flag) when L2 exits (see nested_vmx_vmexit()).
3615 */
3618 /*
3619 * A failed consistency check that leads to a VMExit during L1's
3620 * VMEnter to L2 is a variation of a normal VMexit, as explained in
3621 * 26.7 "VM-entry failures during or after loading guest state".
3622 */
3623 vmentry_fail_vmexit_guest_mode:
3628 vmentry_fail_vmexit:
3639 }
3641 /*
3642 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
3643 * for running an L2 nested guest.
3644 */
3646 {
3660 }
3673 /*
3674 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
3675 * that there *is* a valid VMCS pointer, RFLAGS.CF is set
3676 * rather than RFLAGS.ZF, and no error number is stored to the
3677 * VM-instruction error field.
3678 */
3686 /* Enlightened VMCS doesn't have launch state */
3690 }
3692 /*
3693 * The nested entry process starts with enforcing various prerequisites
3694 * on vmcs12 as required by the Intel SDM, and act appropriately when
3695 * they fail: As the SDM explains, some conditions should cause the
3696 * instruction to fail, while others will cause the instruction to seem
3697 * to succeed, but return an EXIT_REASON_INVALID_STATE.
3698 * To speed up the normal (success) code path, we should avoid checking
3699 * for misconfigurations which will anyway be caught by the processor
3700 * when using the merged vmcs02.
3701 */
3707 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
3719 /*
3720 * We're finally done with prerequisite checking, and can start with
3721 * the nested entry.
3722 */
3729 /* Emulate processing of posted interrupts on VM-Enter. */
3735 }
3737 /* Hide L1D cache contents from the nested guest. */
3740 /*
3741 * Must happen outside of nested_vmx_enter_non_root_mode() as it will
3742 * also be used as part of restoring nVMX state for
3743 * snapshot restore (migration).
3744 *
3745 * In this flow, it is assumed that vmcs12 cache was
3746 * transferred as part of captured nVMX state and should
3747 * therefore not be read from guest memory (which may not
3748 * exist on destination host yet).
3749 */
3754 /*
3755 * If we're entering a halted L2 vcpu and the L2 vcpu won't be
3756 * awakened by event injection or by an NMI-window VM-exit or
3757 * by an interrupt-window VM-exit, halt the vcpu.
3758 */
3765 }
3773 }
3777 vmentry_failed:
3785 }
3787 /*
3788 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
3789 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK).
3790 * This function returns the new value we should put in vmcs12.guest_cr0.
3791 * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
3792 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
3793 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
3794 * didn't trap the bit, because if L1 did, so would L0).
3795 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have
3796 * been modified by L2, and L1 knows it. So just leave the old value of
3797 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
3798 * isn't relevant, because if L0 traps this bit it can set it to anything.
3799 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
3800 * changed these bits, and therefore they need to be updated, but L0
3801 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather
3802 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
3803 */
3806 {
3807 return
3812 }
3816 {
3817 return
3822 }
3827 {
3828 u32 idt_vectoring;
3831 /*
3832 * Per the SDM, VM-Exits due to double and triple faults are never
3833 * considered to occur during event delivery, even if the double/triple
3834 * fault is the result of an escalating vectoring issue.
3835 *
3836 * Note, the SDM qualifies the double fault behavior with "The original
3837 * event results in a double-fault exception". It's unclear why the
3838 * qualification exists since exits due to double fault can occur only
3839 * while vectoring a different exception (injected events are never
3840 * subject to interception), i.e. there's _always_ an original event.
3841 *
3842 * The SDM also uses NMI as a confusing example for the "original event
3843 * causes the VM exit directly" clause. NMI isn't special in any way,
3844 * the same rule applies to all events that cause an exit directly.
3845 * NMI is an odd choice for the example because NMIs can only occur on
3846 * instruction boundaries, i.e. they _can't_ occur during vectoring.
3847 */
3867 }
3887 }
3888 }
3892 {
3894 gfn_t gfn;
3896 /*
3897 * Don't need to mark the APIC access page dirty; it is never
3898 * written to by the CPU during APIC virtualization.
3899 */
3904 }
3909 }
3910 }
3913 {
3917 u16 status;
3943 }
3944 }
3949 mmio_needed:
3952 }
3955 {
3971 }
3973 /*
3974 * Unlike AMD's Paged Real Mode, which reports an error code on #PF
3975 * VM-Exits even if the CPU is in Real Mode, Intel VMX never sets the
3976 * "has error code" flags on VM-Exit if the CPU is in Real Mode.
3977 */
3979 /*
3980 * Intel CPUs do not generate error codes with bits 31:16 set,
3981 * and more importantly VMX disallows setting bits 31:16 in the
3982 * injected error code for VM-Entry. Drop the bits to mimic
3983 * hardware and avoid inducing failure on nested VM-Entry if L1
3984 * chooses to inject the exception back to L2. AMD CPUs _do_
3985 * generate "full" 32-bit error codes, so KVM allows userspace
3986 * to inject exception error codes with bits 31:16 set.
3987 */
3990 }
3994 else
4002 }
4004 /*
4005 * Returns true if a debug trap is (likely) pending delivery. Infer the class
4006 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6).
4007 * Using the payload is flawed because code breakpoints (fault-like) and data
4008 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e.
4009 * this will return false positives if a to-be-injected code breakpoint #DB is
4010 * pending (from KVM's perspective, but not "pending" across an instruction
4011 * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it
4012 * too is trap-like.
4013 *
4014 * KVM "works" despite these flaws as ICEBP isn't currently supported by the
4015 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the
4016 * #DB has already happened), and MTF isn't marked pending on code breakpoints
4017 * from the emulator (because such #DBs are fault-like and thus don't trigger
4018 * actions that fire on instruction retire).
4019 */
4021 {
4025 /* General Detect #DBs are always fault-like. */
4027 }
4029 /*
4030 * Returns true if there's a pending #DB exception that is lower priority than
4031 * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by
4032 * KVM, but could theoretically be injected by userspace. Note, this code is
4033 * imperfect, see above.
4034 */
4036 {
4038 }
4040 /*
4041 * Certain VM-exits set the 'pending debug exceptions' field to indicate a
4042 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM
4043 * represents these debug traps with a payload that is said to be compatible
4044 * with the 'pending debug exceptions' field, write the payload to the VMCS
4045 * field if a VM-exit is delivered before the debug trap.
4046 */
4048 {
4054 }
4057 {
4060 }
4063 {
4072 /*
4073 * Virtual Interrupt Delivery doesn't require manual injection. Either
4074 * the interrupt is already in GUEST_RVI and will be recognized by CPU
4075 * at VM-Entry, or there is a KVM_REQ_EVENT pending and KVM will move
4076 * the interrupt from the PIR to RVI prior to entering the guest.
4077 */
4099 }
4102 }
4104 /*
4105 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor
4106 * edits to fill in missing examples, e.g. #DB due to split-lock accesses,
4107 * and less minor edits to splice in the priority of VMX Non-Root specific
4108 * events, e.g. MTF and NMI/INTR-window exiting.
4109 *
4110 * 1 Hardware Reset and Machine Checks
4111 * - RESET
4112 * - Machine Check
4113 *
4114 * 2 Trap on Task Switch
4115 * - T flag in TSS is set (on task switch)
4116 *
4117 * 3 External Hardware Interventions
4118 * - FLUSH
4119 * - STOPCLK
4120 * - SMI
4121 * - INIT
4122 *
4123 * 3.5 Monitor Trap Flag (MTF) VM-exit[1]
4124 *
4125 * 4 Traps on Previous Instruction
4126 * - Breakpoints
4127 * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O
4128 * breakpoint, or #DB due to a split-lock access)
4129 *
4130 * 4.3 VMX-preemption timer expired VM-exit
4131 *
4132 * 4.6 NMI-window exiting VM-exit[2]
4133 *
4134 * 5 Nonmaskable Interrupts (NMI)
4135 *
4136 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery
4137 *
4138 * 6 Maskable Hardware Interrupts
4139 *
4140 * 7 Code Breakpoint Fault
4141 *
4142 * 8 Faults from Fetching Next Instruction
4143 * - Code-Segment Limit Violation
4144 * - Code Page Fault
4145 * - Control protection exception (missing ENDBRANCH at target of indirect
4146 * call or jump)
4147 *
4148 * 9 Faults from Decoding Next Instruction
4149 * - Instruction length > 15 bytes
4150 * - Invalid Opcode
4151 * - Coprocessor Not Available
4152 *
4153 *10 Faults on Executing Instruction
4154 * - Overflow
4155 * - Bound error
4156 * - Invalid TSS
4157 * - Segment Not Present
4158 * - Stack fault
4159 * - General Protection
4160 * - Data Page Fault
4161 * - Alignment Check
4162 * - x86 FPU Floating-point exception
4163 * - SIMD floating-point exception
4164 * - Virtualization exception
4165 * - Control protection exception
4166 *
4167 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs),
4168 * INIT signals, and higher priority events take priority over MTF VM exits.
4169 * MTF VM exits take priority over debug-trap exceptions and lower priority
4170 * events.
4171 *
4172 * [2] Debug-trap exceptions and higher priority events take priority over VM exits
4173 * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption
4174 * timer take priority over VM exits caused by the "NMI-window exiting"
4175 * VM-execution control and lower priority events.
4176 *
4177 * [3] Debug-trap exceptions and higher priority events take priority over VM exits
4178 * caused by "NMI-window exiting". VM exits caused by this control take
4179 * priority over non-maskable interrupts (NMIs) and lower priority events.
4180 *
4181 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to
4182 * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus,
4183 * non-maskable interrupts (NMIs) and higher priority events take priority over
4184 * delivery of a virtual interrupt; delivery of a virtual interrupt takes
4185 * priority over external interrupts and lower priority events.
4186 */
4188 {
4191 /*
4192 * Only a pending nested run blocks a pending exception. If there is a
4193 * previously injected event, the pending exception occurred while said
4194 * event was being delivered and thus needs to be handled.
4195 */
4197 /*
4198 * New events (not exceptions) are only recognized at instruction
4199 * boundaries. If an event needs reinjection, then KVM is handling a
4200 * VM-Exit that occurred _during_ instruction execution; new events are
4201 * blocked until the instruction completes.
4202 */
4215 /* MTF is discarded if the vCPU is in WFS. */
4218 }
4230 }
4231 /* Fallthrough, the SIPI is completely ignored. */
4232 }
4234 /*
4235 * Process exceptions that are higher priority than Monitor Trap Flag:
4236 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but
4237 * could theoretically come in from userspace), and ICEBP (INT1).
4238 *
4239 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except
4240 * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF
4241 * across SMI/RSM as it should; that needs to be addressed in order to
4242 * prioritize SMI over MTF and trap-like #DBs.
4243 */
4251 }
4258 }
4266 }
4274 }
4280 }
4287 }
4293 }
4304 /*
4305 * The NMI-triggered VM exit counts as injection:
4306 * clear this one and block further NMIs.
4307 */
4311 }
4324 }
4331 }
4337 /*
4338 * If the IRQ is L2's PI notification vector, process posted
4339 * interrupts for L2 instead of injecting VM-Exit, as the
4340 * detection/morphing architecturally occurs when the IRQ is
4341 * delivered to the CPU. Note, only interrupts that are routed
4342 * through the local APIC trigger posted interrupt processing,
4343 * and enabling posted interrupts requires ACK-on-exit.
4344 */
4349 }
4354 /*
4355 * ACK the interrupt _after_ emulating VM-Exit, as the IRQ must
4356 * be marked as in-service in vmcs01.GUEST_INTERRUPT_STATUS.SVI
4357 * if APICv is active.
4358 */
4361 }
4363 no_vmexit:
4365 }
4368 {
4369 ktime_t remaining =
4371 u64 value;
4379 }
4382 {
4423 }
4426 }
4430 {
4471 }
4475 {
4494 }
4496 /*
4497 * Update the guest state fields of vmcs12 to reflect changes that
4498 * occurred while L2 was running. (The "IA-32e mode guest" bit of the
4499 * VM-entry controls is also updated, since this is really a guest
4500 * state bit.)
4501 */
4503 {
4529 else
4538 /*
4539 * In some cases (usually, nested EPT), L2 is allowed to change its
4540 * own CR3 without exiting. If it has changed it, we must keep it.
4541 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
4542 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
4543 *
4544 * Additionally, restore L2's PDPTR to vmcs12.
4545 */
4553 }
4554 }
4570 }
4572 /*
4573 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
4574 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
4575 * and this function updates it to reflect the changes to the guest state while
4576 * L2 was running (and perhaps made some exits which were handled directly by L0
4577 * without going back to L1), and to reflect the exit reason.
4578 * Note that we do not have to copy here all VMCS fields, just those that
4579 * could have changed by the L2 guest or the exit - i.e., the guest-state and
4580 * exit-information fields only. Other fields are modified by L1 with VMWRITE,
4581 * which already writes to vmcs12 directly.
4582 */
4586 {
4587 /* update exit information fields: */
4593 /*
4594 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched
4595 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other
4596 * exit info fields are unmodified.
4597 */
4601 /* vm_entry_intr_info_field is cleared on exit. Emulate this
4602 * instead of reading the real value. */
4605 /*
4606 * Transfer the event that L0 or L1 may wanted to inject into
4607 * L2 to IDT_VECTORING_INFO_FIELD.
4608 */
4616 /*
4617 * According to spec, there's no need to store the guest's
4618 * MSRs if the exit is due to a VM-entry failure that occurs
4619 * during or after loading the guest state. Since this exit
4620 * does not fall in that category, we need to save the MSRs.
4621 */
4626 VMX_ABORT_SAVE_GUEST_MSR_FAIL);
4627 }
4628 }
4630 /*
4631 * A part of what we need to when the nested L2 guest exits and we want to
4632 * run its L1 parent, is to reset L1's guest state to the host state specified
4633 * in vmcs12.
4634 * This function is to be called not only on normal nested exit, but also on
4635 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
4636 * Failures During or After Loading Guest State").
4637 * This function should be called when the active VMCS is L1's (vmcs01).
4638 */
4641 {
4649 else
4658 /*
4659 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
4660 * actually changed, because vmx_set_cr0 refers to efer set above.
4661 *
4662 * CR0_GUEST_HOST_MASK is already set in the original vmcs01
4663 * (KVM doesn't change it);
4664 */
4668 /* Same as above - no reason to call set_cr4_guest_host_mask(). */
4674 /*
4675 * Only PDPTE load can fail as the value of cr3 was checked on entry and
4676 * couldn't have changed.
4677 */
4691 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */
4698 }
4704 /* Set L1 segment info according to Intel SDM
4705 27.5.2 Loading Host Segment and Descriptor-Table Registers */
4714 };
4717 else
4728 };
4747 };
4762 }
4765 {
4778 }
4785 }
4788 {
4792 gpa_t gpa;
4798 /*
4799 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
4800 * as vmcs01.GUEST_DR7 contains a userspace defined value
4801 * and vcpu->arch.dr7 is not squirreled away before the
4802 * nested VMENTER (not worth adding a variable in nested_vmx).
4803 */
4806 else
4808 }
4810 /*
4811 * Note that calling vmx_set_{efer,cr0,cr4} is important as they
4812 * handle a variety of side effects to KVM's software model.
4813 */
4826 /*
4827 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
4828 * from vmcs01 (if necessary). The PDPTRs are not loaded on
4829 * VMFail, like everything else we just need to ensure our
4830 * software model is up-to-date.
4831 */
4837 /*
4838 * This nasty bit of open coding is a compromise between blindly
4839 * loading L1's MSRs using the exit load lists (incorrect emulation
4840 * of VMFail), leaving the nested VM's MSRs in the software model
4841 * (incorrect behavior) and snapshotting the modified MSRs (too
4842 * expensive since the lists are unbound by hardware). For each
4843 * MSR that was (prematurely) loaded from the nested VMEntry load
4844 * list, reload it from the exit load list if it exists and differs
4845 * from the guest value. The intent is to stuff host state as
4846 * silently as possible, not to fully process the exit load list.
4847 */
4855 }
4864 }
4875 }
4882 }
4883 }
4884 }
4888 vmabort:
4890 }
4892 /*
4893 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
4894 * and modify vmcs12 to make it see what it would expect to see there if
4895 * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
4896 */
4899 {
4903 /* Pending MTF traps are discarded on VM-Exit. */
4906 /* trying to cancel vmlaunch/vmresume is a bug */
4909 #ifdef CONFIG_KVM_HYPERV
4911 /*
4912 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map
4913 * Enlightened VMCS after migration and we still need to
4914 * do that when something is forcing L2->L1 exit prior to
4915 * the first L2 run.
4916 */
4918 }
4919 #endif
4921 /* Service pending TLB flush requests for L2 before switching to L1. */
4924 /*
4925 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
4926 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are
4927 * up-to-date before switching to L1.
4928 */
4941 }
4950 /*
4951 * Must happen outside of sync_vmcs02_to_vmcs12() as it will
4952 * also be used to capture vmcs12 cache as part of
4953 * capturing nVMX state for snapshot (migration).
4954 *
4955 * Otherwise, this flush will dirty guest memory at a
4956 * point it is already assumed by user-space to be
4957 * immutable.
4958 */
4961 /*
4962 * The only expected VM-instruction error is "VM entry with
4963 * invalid control field(s)." Anything else indicates a
4964 * problem with L0. And we should never get here with a
4965 * VMFail of any type if early consistency checks are enabled.
4966 */
4968 VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4970 }
4972 /*
4973 * Drop events/exceptions that were queued for re-injection to L2
4974 * (picked up via vmx_complete_interrupts()), as well as exceptions
4975 * that were pending for L2. Note, this must NOT be hoisted above
4976 * prepare_vmcs12(), events/exceptions queued for re-injection need to
4977 * be captured in vmcs12 (see vmcs12_save_pending_event()).
4978 */
4985 /*
4986 * If IBRS is advertised to the vCPU, KVM must flush the indirect
4987 * branch predictors when transitioning from L2 to L1, as L1 expects
4988 * hardware (KVM in this case) to provide separate predictor modes.
4989 * Bare metal isolates VMX root (host) from VMX non-root (guest), but
4990 * doesn't isolate different VMCSs, i.e. in this case, doesn't provide
4991 * separate modes for L2 vs L1.
4992 */
4996 /* Update any VMCS fields that might have changed while L2 ran */
5009 }
5014 }
5016 /* Unpin physical memory we referred to in vmcs02 */
5025 }
5030 }
5036 /* in case we halted in L2 */
5046 KVM_ISA_VMX);
5051 }
5053 /*
5054 * After an early L2 VM-entry failure, we're now back
5055 * in L1 which thinks it just finished a VMLAUNCH or
5056 * VMRESUME instruction, so we need to set the failure
5057 * flag and the VM-instruction error field of the VMCS
5058 * accordingly, and skip the emulated instruction.
5059 */
5062 /*
5063 * Restore L1's host state to KVM's software model. We're here
5064 * because a consistency check was caught by hardware, which
5065 * means some amount of guest state has been propagated to KVM's
5066 * model and needs to be unwound to the host's state.
5067 */
5071 }
5074 {
5077 }
5079 /*
5080 * Decode the memory-address operand of a vmx instruction, as recorded on an
5081 * exit caused by such an instruction (run by a guest hypervisor).
5082 * On success, returns 0. When the operand is invalid, returns 1 and throws
5083 * #UD, #GP, or #SS.
5084 */
5087 {
5088 gva_t off;
5092 /*
5093 * According to Vol. 3B, "Information for VM Exits Due to Instruction
5094 * Execution", on an exit, vmx_instruction_info holds most of the
5095 * addressing components of the operand. Only the displacement part
5096 * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
5097 * For how an actual address is calculated from all these components,
5098 * refer to Vol. 1, "Operand Addressing".
5099 */
5112 }
5114 /* Addr = segment_base + offset */
5115 /* offset = base + [index * scale] + displacement */
5127 /*
5128 * The effective address, i.e. @off, of a memory operand is truncated
5129 * based on the address size of the instruction. Note that this is
5130 * the *effective address*, i.e. the address prior to accounting for
5131 * the segment's base.
5132 */
5138 /* Checks for #GP/#SS exceptions. */
5141 /*
5142 * The virtual/linear address is never truncated in 64-bit
5143 * mode, e.g. a 32-bit address size can yield a 64-bit virtual
5144 * address when using FS/GS with a non-zero base.
5145 */
5148 else
5152 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
5153 * non-canonical form. This is the only check on the memory
5154 * destination for long mode!
5155 */
5158 /*
5159 * When not in long mode, the virtual/linear address is
5160 * unconditionally truncated to 32 bits regardless of the
5161 * address size.
5162 */
5165 /* Protected mode: apply checks for segment validity in the
5166 * following order:
5167 * - segment type check (#GP(0) may be thrown)
5168 * - usability check (#GP(0)/#SS(0))
5169 * - limit check (#GP(0)/#SS(0))
5170 */
5172 /* #GP(0) if the destination operand is located in a
5173 * read-only data segment or any code segment.
5174 */
5176 else
5177 /* #GP(0) if the source operand is located in an
5178 * execute-only code segment
5179 */
5184 }
5185 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
5186 */
5189 /*
5190 * Protected mode: #GP(0)/#SS(0) if the memory operand is
5191 * outside the segment limit. All CPUs that support VMX ignore
5192 * limit checks for flat segments, i.e. segments with base==0,
5193 * limit==0xffffffff and of type expand-up data or code.
5194 */
5198 }
5205 }
5208 }
5212 {
5213 gva_t gva;
5222 }
5228 }
5231 }
5233 /*
5234 * Allocate a shadow VMCS and associate it with the currently loaded
5235 * VMCS, unless such a shadow VMCS already exists. The newly allocated
5236 * VMCS is also VMCLEARed, so that it is ready for use.
5237 */
5239 {
5243 /*
5244 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it
5245 * when L1 executes VMXOFF or the vCPU is forced out of nested
5246 * operation. VMXON faults if the CPU is already post-VMXON, so it
5247 * should be impossible to already have an allocated shadow VMCS. KVM
5248 * doesn't support virtualization of VMCS shadowing, so vmcs01 should
5249 * always be the loaded VMCS.
5250 */
5259 }
5262 {
5283 HRTIMER_MODE_ABS_PINNED);
5294 }
5298 out_shadow_vmcs:
5301 out_cached_shadow_vmcs12:
5304 out_cached_vmcs12:
5307 out_vmcs02:
5309 }
5311 /* Emulate the VMXON instruction. */
5313 {
5315 gpa_t vmptr;
5321 /*
5322 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter
5323 * the guest and so cannot rely on hardware to perform the check,
5324 * which has higher priority than VM-Exit (see Intel SDM's pseudocode
5325 * for VMXON).
5326 *
5327 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86
5328 * and !COMPATIBILITY modes. For an unrestricted guest, KVM doesn't
5329 * force any of the relevant guest state. For a restricted guest, KVM
5330 * does force CR0.PE=1, but only to also force VM86 in order to emulate
5331 * Real Mode, and so there's no need to check CR0.PE manually.
5332 */
5336 }
5338 /*
5339 * The CPL is checked for "not in VMX operation" and for "in VMX root",
5340 * and has higher priority than the VM-Fail due to being post-VMXON,
5341 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0. In VMX non-root,
5342 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits
5343 * from L2 to L1, i.e. there's no need to check for the vCPU being in
5344 * VMX non-root.
5345 *
5346 * Forwarding the VM-Exit unconditionally, i.e. without performing the
5347 * #UD checks (see above), is functionally ok because KVM doesn't allow
5348 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's
5349 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are
5350 * missed by hardware due to shadowing CR0 and/or CR4.
5351 */
5355 }
5360 /*
5361 * Invalid CR0/CR4 generates #GP. These checks are performed if and
5362 * only if the vCPU isn't already in VMX operation, i.e. effectively
5363 * have lower priority than the VM-Fail above.
5364 */
5369 }
5375 }
5380 /*
5381 * SDM 3: 24.11.5
5382 * The first 4 bytes of VMXON region contain the supported
5383 * VMCS revision identifier
5384 *
5385 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
5386 * which replaces physical address width with 32
5387 */
5401 }
5404 {
5413 /* copy to memory all shadowed fields in case
5414 they were modified */
5417 }
5420 /* Flush VMCS12 to guest memory */
5428 }
5430 /* Emulate the VMXOFF instruction */
5432 {
5442 }
5444 /* Emulate the VMCLEAR instruction */
5446 {
5449 gpa_t vmptr;
5468 /*
5469 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode
5470 * for VMCLEAR includes a "ensure that data for VMCS referenced
5471 * by the operand is in memory" clause that guards writes to
5472 * memory, i.e. doing nothing for I/O is architecturally valid.
5473 *
5474 * FIXME: Suppress failures if and only if no memslot is found,
5475 * i.e. exit to userspace if __copy_to_user() fails.
5476 */
5479 launch_state),
5481 }
5484 }
5486 /* Emulate the VMLAUNCH instruction */
5488 {
5490 }
5492 /* Emulate the VMRESUME instruction */
5494 {
5497 }
5500 {
5508 u64 value;
5516 /* Decode instruction info and find the field to read */
5520 /*
5521 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
5522 * any VMREAD sets the ALU flags for VMfailInvalid.
5523 */
5536 /* Read the field, zero-extended to a u64 value */
5539 /*
5540 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an
5541 * enlightened VMCS is active VMREAD/VMWRITE instructions are
5542 * unsupported. Unfortunately, certain versions of Windows 11
5543 * don't comply with this requirement which is not enforced in
5544 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a
5545 * workaround, as misbehaving guests will panic on VM-Fail.
5546 * Note, enlightened VMCS is incompatible with shadow VMCS so
5547 * all VMREADs from L2 should go to L1.
5548 */
5556 /* Read the field, zero-extended to a u64 value */
5558 }
5560 /*
5561 * Now copy part of this value to register or memory, as requested.
5562 * Note that the number of bits actually copied is 32 or 64 depending
5563 * on the guest's mode (32 or 64 bit), not on the given field's length.
5564 */
5572 /* _system ok, nested_vmx_check_permission has verified cpl=0 */
5576 }
5579 }
5582 {
5584 #define SHADOW_FIELD_RW(x, y) case x:
5589 }
5591 }
5594 {
5596 #define SHADOW_FIELD_RO(x, y) case x:
5601 }
5603 }
5606 {
5615 gva_t gva;
5618 /*
5619 * The value to write might be 32 or 64 bits, depending on L1's long
5620 * mode, and eventually we need to write that into a field of several
5621 * possible lengths. The code below first zero-extends the value to 64
5622 * bit (value), and then copies only the appropriate number of
5623 * bits into the vmcs12 field.
5624 */
5630 /*
5631 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
5632 * any VMWRITE sets the ALU flags for VMfailInvalid.
5633 */
5649 }
5657 /*
5658 * If the vCPU supports "VMWRITE to any supported field in the
5659 * VMCS," then the "read-only" fields are actually read/write.
5660 */
5665 /*
5666 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties
5667 * vmcs12, else we may crush a field or consume a stale value.
5668 */
5672 /*
5673 * Some Intel CPUs intentionally drop the reserved bits of the AR byte
5674 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM
5675 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE
5676 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD
5677 * from L1 will return a different value than VMREAD from L2 (L1 sees
5678 * the stripped down value, L2 sees the full value as stored by KVM).
5679 */
5685 /*
5686 * Do not track vmcs12 dirty-state if in guest-mode as we actually
5687 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated
5688 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't
5689 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path.
5690 */
5692 /*
5693 * L1 can read these fields without exiting, ensure the
5694 * shadow VMCS is up-to-date.
5695 */
5705 }
5707 }
5710 }
5713 {
5720 }
5723 }
5725 /* Emulate the VMPTRLD instruction */
5727 {
5729 gpa_t vmptr;
5744 /* Forbid normal VMPTRLD if Enlightened version was used */
5753 /*
5754 * Reads from an unbacked page return all 1s,
5755 * which means that the 32 bits located at the
5756 * given physical address won't match the required
5757 * VMCS12_REVISION identifier.
5758 */
5760 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5761 }
5767 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5768 }
5774 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5775 }
5779 /*
5780 * Load VMCS12 from guest memory since it is not already
5781 * cached.
5782 */
5784 VMCS12_SIZE)) {
5786 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5787 }
5790 }
5793 }
5795 /* Emulate the VMPTRST instruction */
5797 {
5802 gva_t gva;
5814 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
5821 }
5823 /* Emulate the INVEPT instruction */
5825 {
5830 gva_t gva;
5838 SECONDARY_EXEC_ENABLE_EPT) ||
5842 }
5856 /* According to the Intel VMX instruction reference, the memory
5857 * operand is read even if it isn't needed (e.g., for type==global)
5858 */
5866 /*
5867 * Nested EPT roots are always held through guest_mmu,
5868 * not root_mmu.
5869 */
5876 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5888 }
5896 }
5902 }
5905 {
5907 u32 vmx_instruction_info;
5909 gva_t gva;
5912 u64 vpid;
5913 u64 gla;
5915 u16 vpid02;
5919 SECONDARY_EXEC_ENABLE_VPID) ||
5923 }
5937 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5939 /* according to the intel vmx instruction reference, the memory
5940 * operand is read even if it isn't needed (e.g., for type==global)
5941 */
5951 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5956 /*
5957 * LAM doesn't apply to addresses that are inputs to TLB
5958 * invalidation.
5959 */
5963 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5970 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5979 }
5981 /*
5982 * Sync the shadow page tables if EPT is disabled, L1 is invalidating
5983 * linear mappings for L2 (tagged with L2's VPID). Free all guest
5984 * roots as VPIDs are not tracked in the MMU role.
5985 *
5986 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share
5987 * an MMU when EPT is disabled.
5988 *
5989 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR.
5990 */
5995 }
5999 {
6001 u64 new_eptp;
6012 /*
6013 * If the (L2) guest does a vmfunc to the currently
6014 * active ept pointer, we don't have to do anything else
6015 */
6025 }
6028 }
6031 {
6036 /*
6037 * VMFUNC should never execute cleanly while L1 is active; KVM supports
6038 * VMFUNC for nested VMs, but not for L1.
6039 */
6043 }
6047 /*
6048 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC
6049 * is enabled in vmcs02 if and only if it's enabled in vmcs12.
6050 */
6054 }
6066 }
6069 fail:
6070 /*
6071 * This is effectively a reflected VM-Exit, as opposed to a synthesized
6072 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode
6073 * EXIT_REASON_VMFUNC as the exit reason.
6074 */
6079 }
6081 /*
6082 * Return true if an IO instruction with the specified port and size should cause
6083 * a VM-exit into L1.
6084 */
6087 {
6090 u8 b;
6100 else
6110 port++;
6111 size--;
6113 }
6116 }
6120 {
6134 }
6136 /*
6137 * Return 1 if we should exit from L2 to L1 to handle an MSR access,
6138 * rather than handle it ourselves in L0. I.e., check whether L1 expressed
6139 * disinterest in the current event (read or write a specific MSR) by using an
6140 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
6141 */
6145 {
6147 gpa_t bitmap;
6152 /*
6153 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
6154 * for the four combinations of read/write and low/high MSR numbers.
6155 * First we need to figure out which of the four to use:
6156 */
6163 }
6165 /* Then read the msr_index'th bit from this bitmap: */
6173 }
6175 /*
6176 * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
6177 * rather than handle it ourselves in L0. I.e., check if L1 wanted to
6178 * intercept (via guest_host_mask etc.) the current event.
6179 */
6182 {
6211 }
6222 CPU_BASED_CR3_STORE_EXITING)
6227 CPU_BASED_CR8_STORE_EXITING)
6230 }
6233 /*
6234 * lmsw can change bits 1..3 of cr0, and only set bit 0 of
6235 * cr0. Other attempted changes are ignored, with no exit.
6236 */
6246 }
6248 }
6252 {
6253 u32 encls_leaf;
6263 }
6267 {
6268 u32 vmx_instruction_info;
6270 u8 b;
6275 /* Decode instruction info and find the field to access */
6279 /* Out-of-range fields always cause a VM exit from L2 to L1 */
6287 }
6290 {
6296 /*
6297 * An MTF VM-exit may be injected into the guest by setting the
6298 * interruption-type to 7 (other event) and the vector field to 0. Such
6299 * is the case regardless of the 'monitor trap flag' VM-execution
6300 * control.
6301 */
6304 }
6306 /*
6307 * Return true if L0 wants to handle an exit from L2 regardless of whether or not
6308 * L1 wants the exit. Only call this when in is_guest_mode (L2).
6309 */
6312 {
6313 u32 intr_info;
6341 /*
6342 * L0 always deals with the EPT violation. If nested EPT is
6343 * used, and the nested mmu code discovers that the address is
6344 * missing in the guest EPT table (EPT12), the EPT violation
6345 * will be injected with nested_ept_inject_page_fault()
6346 */
6349 /*
6350 * L2 never uses directly L1's EPT, but rather L0's own EPT
6351 * table (shadow on EPT) or a merged EPT table that L0 built
6352 * (EPT on EPT). So any problems with the structure of the
6353 * table is L0's fault.
6354 */
6359 /*
6360 * PML is emulated for an L1 VMM and should never be enabled in
6361 * vmcs02, always "handle" PML_FULL by exiting to userspace.
6362 */
6365 /* VM functions are emulated through L2->L0 vmexits. */
6368 /*
6369 * At present, bus lock VM exit is never exposed to L1.
6370 * Handle L2's bus locks in L0 directly.
6371 */
6373 #ifdef CONFIG_KVM_HYPERV
6375 /* Hyper-V L2 TLB flush hypercall is handled by L0 */
6379 #endif
6382 }
6384 }
6386 /*
6387 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in
6388 * is_guest_mode (L2).
6389 */
6392 {
6394 u32 intr_info;
6442 /*
6443 * VMX instructions trap unconditionally. This allows L1 to
6444 * emulate them for its L2 guest, i.e., allows 3-level nesting!
6445 */
6469 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
6477 /*
6478 * The controls for "virtualize APIC accesses," "APIC-
6479 * register virtualization," and "virtual-interrupt
6480 * delivery" only come from vmcs12.
6481 */
6484 return
6492 /*
6493 * This should never happen, since it is not possible to
6494 * set XSS to a non-zero value---neither in L1 nor in L2.
6495 * If if it were, XSS would have to be checked against
6496 * the XSS exit bitmap in vmcs12.
6497 */
6502 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE);
6506 /* Notify VM exit is not exposed to L1 */
6510 }
6511 }
6513 /*
6514 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was
6515 * reflected into L1.
6516 */
6518 {
6522 u32 exit_intr_info;
6526 /*
6527 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM
6528 * has already loaded L2's state.
6529 */
6537 }
6541 /* If L0 (KVM) wants the exit, it trumps L1's desires. */
6545 /* If L1 doesn't want the exit, handle it in L0. */
6549 /*
6550 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For
6551 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would
6552 * need to be synthesized by querying the in-kernel LAPIC, but external
6553 * interrupts are never reflected to L1 so it's a non-issue.
6554 */
6561 }
6564 reflect_vmexit:
6567 }
6571 u32 user_data_size)
6572 {
6583 };
6601 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */
6609 }
6629 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE;
6632 }
6633 }
6634 }
6645 /*
6646 * When running L2, the authoritative vmcs12 state is in the
6647 * vmcs02. When running L1, the authoritative vmcs12 state is
6648 * in the shadow or enlightened vmcs linked to vmcs01, unless
6649 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative
6650 * vmcs12 state is in the vmcs12 already.
6651 */
6659 /*
6660 * L1 hypervisor is not obliged to keep eVMCS
6661 * clean fields data always up-to-date while
6662 * not in guest mode, 'hv_clean_fields' is only
6663 * supposed to be actual upon vmentry so we need
6664 * to ignore it here and do full copy.
6665 */
6669 }
6670 }
6675 /*
6676 * Copy over the full allocated size of vmcs12 rather than just the size
6677 * of the struct.
6678 */
6687 }
6688 out:
6690 }
6693 {
6697 }
6699 }
6704 {
6722 /*
6723 * KVM_STATE_NESTED_EVMCS used to signal that KVM should
6724 * enable eVMCS capability on vCPU. However, since then
6725 * code was changed such that flag signals vmcs12 should
6726 * be copied into eVMCS in guest memory.
6727 *
6728 * To preserve backwards compatibility, allow user
6729 * to set this flag even when there is no VMXON region.
6730 */
6739 }
6752 /*
6753 * SMM temporarily disables VMX, so we cannot be in guest mode,
6754 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags
6755 * must be zero.
6756 */
6782 /* Empty 'VMXON' state is permitted if no VMCS loaded */
6784 /* See vmx_has_valid_vmcs12. */
6789 else
6791 }
6799 #ifdef CONFIG_KVM_HYPERV
6801 /*
6802 * nested_vmx_handle_enlightened_vmptrld() cannot be called
6803 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be
6804 * restored yet. EVMCS will be mapped from
6805 * nested_get_vmcs12_pages().
6806 */
6809 #endif
6812 }
6820 }
6853 }
6858 }
6865 }
6883 error_guest_mode:
6886 }
6889 {
6893 }
6894 }
6896 /*
6897 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo
6898 * that madness to get the encoding for comparison.
6899 */
6900 #define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10)))
6903 {
6904 /*
6905 * Note these are the so called "index" of the VMCS field encoding, not
6906 * the index into vmcs12.
6907 */
6911 /*
6912 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in
6913 * vmcs12, regardless of whether or not the associated feature is
6914 * exposed to L1. Simply find the field with the highest index.
6915 */
6918 /* The vmcs12 table is very, very sparsely populated. */
6925 }
6928 }
6932 {
6934 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
6938 PIN_BASED_EXT_INTR_MASK |
6939 PIN_BASED_NMI_EXITING |
6940 PIN_BASED_VIRTUAL_NMIS |
6943 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
6944 PIN_BASED_VMX_PREEMPTION_TIMER;
6945 }
6949 {
6951 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
6955 #ifdef CONFIG_X86_64
6956 VM_EXIT_HOST_ADDR_SPACE_SIZE |
6957 #endif
6959 VM_EXIT_CLEAR_BNDCFGS;
6961 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
6964 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
6966 /* We support free control of debug control saving. */
6968 }
6972 {
6974 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
6978 #ifdef CONFIG_X86_64
6979 VM_ENTRY_IA32E_MODE |
6980 #endif
6984 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL);
6986 /* We support free control of debug control loading. */
6988 }
6992 {
6994 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
6998 CPU_BASED_INTR_WINDOW_EXITING |
7002 CPU_BASED_CR3_STORE_EXITING |
7003 #ifdef CONFIG_X86_64
7005 #endif
7011 /*
7012 * We can allow some features even when not supported by the
7013 * hardware. For example, L1 can specify an MSR bitmap - and we
7014 * can use it to avoid exits to L1 - even when L0 runs L2
7015 * without MSR bitmaps.
7016 */
7018 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
7019 CPU_BASED_USE_MSR_BITMAPS;
7021 /* We support free control of CR3 access interception. */
7024 }
7029 {
7034 SECONDARY_EXEC_DESC |
7035 SECONDARY_EXEC_ENABLE_RDTSCP |
7036 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
7037 SECONDARY_EXEC_WBINVD_EXITING |
7038 SECONDARY_EXEC_APIC_REGISTER_VIRT |
7039 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
7040 SECONDARY_EXEC_RDRAND_EXITING |
7041 SECONDARY_EXEC_ENABLE_INVPCID |
7042 SECONDARY_EXEC_ENABLE_VMFUNC |
7043 SECONDARY_EXEC_RDSEED_EXITING |
7044 SECONDARY_EXEC_ENABLE_XSAVES |
7045 SECONDARY_EXEC_TSC_SCALING |
7046 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
7048 /*
7049 * We can emulate "VMCS shadowing," even if the hardware
7050 * doesn't support it.
7051 */
7053 SECONDARY_EXEC_SHADOW_VMCS;
7056 /* nested EPT: emulate EPT also to L1 */
7058 SECONDARY_EXEC_ENABLE_EPT;
7060 VMX_EPT_PAGE_WALK_4_BIT |
7061 VMX_EPT_PAGE_WALK_5_BIT |
7062 VMX_EPTP_WB_BIT |
7063 VMX_EPT_INVEPT_BIT |
7064 VMX_EPT_EXECUTE_ONLY_BIT;
7069 VMX_EPT_1GB_PAGE_BIT;
7072 SECONDARY_EXEC_ENABLE_PML;
7074 }
7076 /*
7077 * Advertise EPTP switching irrespective of hardware support,
7078 * KVM emulates it in software so long as VMFUNC is supported.
7079 */
7082 }
7084 /*
7085 * Old versions of KVM use the single-context version without
7086 * checking for support, so declare that it is supported even
7087 * though it is treated as global context. The alternative is
7088 * not failing the single-context invvpid, and it is worse.
7089 */
7092 SECONDARY_EXEC_ENABLE_VPID;
7094 VMX_VPID_EXTENT_SUPPORTED_MASK;
7095 }
7099 SECONDARY_EXEC_UNRESTRICTED_GUEST;
7103 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
7107 }
7111 {
7114 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
7115 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
7116 VMX_MISC_ACTIVITY_HLT |
7117 VMX_MISC_ACTIVITY_WAIT_SIPI;
7119 }
7122 {
7123 /*
7124 * This MSR reports some information about VMX support. We
7125 * should return information about the VMX we emulate for the
7126 * guest, and the VMCS structure we give it - not about the
7127 * VMX support of the underlying hardware.
7128 */
7130 X86_MEMTYPE_WB);
7135 }
7138 {
7139 /*
7140 * These MSRs specify bits which the guest must keep fixed on
7141 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
7142 * We picked the standard core2 setting.
7143 */
7144 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
7145 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE
7149 /* These MSRs specify bits which the guest must keep fixed off. */
7155 }
7157 /*
7158 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
7159 * returned for the various VMX controls MSRs when nested VMX is enabled.
7160 * The same values should also be used to verify that vmcs12 control fields are
7161 * valid during nested entry from L1 to L2.
7162 * Each of these control msrs has a low and high 32-bit half: A low bit is on
7163 * if the corresponding bit in the (32-bit) control field *must* be on, and a
7164 * bit in the high half is on if the corresponding bit in the control field
7165 * may be on. See also vmx_control_verify().
7166 */
7168 {
7171 /*
7172 * Note that as a general rule, the high half of the MSRs (bits in
7173 * the control fields which may be 1) should be initialized by the
7174 * intersection of the underlying hardware's MSR (i.e., features which
7175 * can be supported) and the list of features we want to expose -
7176 * because they are known to be properly supported in our code.
7177 * Also, usually, the low half of the MSRs (bits which must be 1) can
7178 * be set to 0, meaning that L1 may turn off any of these bits. The
7179 * reason is that if one of these bits is necessary, it will appear
7180 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
7181 * fields of vmcs01 and vmcs02, will turn these bits off - and
7182 * nested_vmx_l1_wants_exit() will not pass related exits to L1.
7183 * These rules have exceptions below.
7184 */
7202 }
7205 {
7211 }
7212 }
7215 {
7222 /*
7223 * The vmx_bitmap is not tied to a VM and so should
7224 * not be charged to a memcg.
7225 */
7231 }
7232 }
7235 }
7251 }
7263 #ifdef CONFIG_KVM_HYPERV
7267 #endif
7268 };