| // SPDX-License-Identifier: GPL-2.0 |
| |
| #include <linux/objtool.h> |
| #include <linux/percpu.h> |
| |
| #include <asm/debugreg.h> |
| #include <asm/mmu_context.h> |
| |
| #include "cpuid.h" |
| #include "hyperv.h" |
| #include "mmu.h" |
| #include "nested.h" |
| #include "pmu.h" |
| #include "sgx.h" |
| #include "trace.h" |
| #include "vmx.h" |
| #include "x86.h" |
| #include "smm.h" |
| |
| static bool __read_mostly enable_shadow_vmcs = 1; |
| module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); |
| |
| static bool __read_mostly nested_early_check = 0; |
| module_param(nested_early_check, bool, S_IRUGO); |
| |
| #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK |
| |
| /* |
| * Hyper-V requires all of these, so mark them as supported even though |
| * they are just treated the same as all-context. |
| */ |
| #define VMX_VPID_EXTENT_SUPPORTED_MASK \ |
| (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ |
| VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ |
| VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ |
| VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) |
| |
| #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 |
| |
| enum { |
| VMX_VMREAD_BITMAP, |
| VMX_VMWRITE_BITMAP, |
| VMX_BITMAP_NR |
| }; |
| static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; |
| |
| #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) |
| #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) |
| |
| struct shadow_vmcs_field { |
| u16 encoding; |
| u16 offset; |
| }; |
| static struct shadow_vmcs_field shadow_read_only_fields[] = { |
| #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, |
| #include "vmcs_shadow_fields.h" |
| }; |
| static int max_shadow_read_only_fields = |
| ARRAY_SIZE(shadow_read_only_fields); |
| |
| static struct shadow_vmcs_field shadow_read_write_fields[] = { |
| #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, |
| #include "vmcs_shadow_fields.h" |
| }; |
| static int max_shadow_read_write_fields = |
| ARRAY_SIZE(shadow_read_write_fields); |
| |
| static void init_vmcs_shadow_fields(void) |
| { |
| int i, j; |
| |
| memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); |
| memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); |
| |
| for (i = j = 0; i < max_shadow_read_only_fields; i++) { |
| struct shadow_vmcs_field entry = shadow_read_only_fields[i]; |
| u16 field = entry.encoding; |
| |
| if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && |
| (i + 1 == max_shadow_read_only_fields || |
| shadow_read_only_fields[i + 1].encoding != field + 1)) |
| pr_err("Missing field from shadow_read_only_field %x\n", |
| field + 1); |
| |
| clear_bit(field, vmx_vmread_bitmap); |
| if (field & 1) |
| #ifdef CONFIG_X86_64 |
| continue; |
| #else |
| entry.offset += sizeof(u32); |
| #endif |
| shadow_read_only_fields[j++] = entry; |
| } |
| max_shadow_read_only_fields = j; |
| |
| for (i = j = 0; i < max_shadow_read_write_fields; i++) { |
| struct shadow_vmcs_field entry = shadow_read_write_fields[i]; |
| u16 field = entry.encoding; |
| |
| if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && |
| (i + 1 == max_shadow_read_write_fields || |
| shadow_read_write_fields[i + 1].encoding != field + 1)) |
| pr_err("Missing field from shadow_read_write_field %x\n", |
| field + 1); |
| |
| WARN_ONCE(field >= GUEST_ES_AR_BYTES && |
| field <= GUEST_TR_AR_BYTES, |
| "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); |
| |
| /* |
| * PML and the preemption timer can be emulated, but the |
| * processor cannot vmwrite to fields that don't exist |
| * on bare metal. |
| */ |
| switch (field) { |
| case GUEST_PML_INDEX: |
| if (!cpu_has_vmx_pml()) |
| continue; |
| break; |
| case VMX_PREEMPTION_TIMER_VALUE: |
| if (!cpu_has_vmx_preemption_timer()) |
| continue; |
| break; |
| case GUEST_INTR_STATUS: |
| if (!cpu_has_vmx_apicv()) |
| continue; |
| break; |
| default: |
| break; |
| } |
| |
| clear_bit(field, vmx_vmwrite_bitmap); |
| clear_bit(field, vmx_vmread_bitmap); |
| if (field & 1) |
| #ifdef CONFIG_X86_64 |
| continue; |
| #else |
| entry.offset += sizeof(u32); |
| #endif |
| shadow_read_write_fields[j++] = entry; |
| } |
| max_shadow_read_write_fields = j; |
| } |
| |
| /* |
| * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), |
| * set the success or error code of an emulated VMX instruction (as specified |
| * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated |
| * instruction. |
| */ |
| static int nested_vmx_succeed(struct kvm_vcpu *vcpu) |
| { |
| vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) |
| & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | |
| X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); |
| return kvm_skip_emulated_instruction(vcpu); |
| } |
| |
| static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) |
| { |
| vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) |
| & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | |
| X86_EFLAGS_SF | X86_EFLAGS_OF)) |
| | X86_EFLAGS_CF); |
| return kvm_skip_emulated_instruction(vcpu); |
| } |
| |
| static int nested_vmx_failValid(struct kvm_vcpu *vcpu, |
| u32 vm_instruction_error) |
| { |
| vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) |
| & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | |
| X86_EFLAGS_SF | X86_EFLAGS_OF)) |
| | X86_EFLAGS_ZF); |
| get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; |
| /* |
| * We don't need to force sync to shadow VMCS because |
| * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all |
| * fields and thus must be synced. |
| */ |
| if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) |
| to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; |
| |
| return kvm_skip_emulated_instruction(vcpu); |
| } |
| |
| static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* |
| * failValid writes the error number to the current VMCS, which |
| * can't be done if there isn't a current VMCS. |
| */ |
| if (vmx->nested.current_vmptr == INVALID_GPA && |
| !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) |
| return nested_vmx_failInvalid(vcpu); |
| |
| return nested_vmx_failValid(vcpu, vm_instruction_error); |
| } |
| |
| static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) |
| { |
| /* TODO: not to reset guest simply here. */ |
| kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); |
| pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); |
| } |
| |
| static inline bool vmx_control_verify(u32 control, u32 low, u32 high) |
| { |
| return fixed_bits_valid(control, low, high); |
| } |
| |
| static inline u64 vmx_control_msr(u32 low, u32 high) |
| { |
| return low | ((u64)high << 32); |
| } |
| |
| static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) |
| { |
| secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); |
| vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); |
| vmx->nested.need_vmcs12_to_shadow_sync = false; |
| } |
| |
| static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) |
| { |
| struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { |
| kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); |
| vmx->nested.hv_evmcs = NULL; |
| } |
| |
| vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; |
| |
| if (hv_vcpu) { |
| hv_vcpu->nested.pa_page_gpa = INVALID_GPA; |
| hv_vcpu->nested.vm_id = 0; |
| hv_vcpu->nested.vp_id = 0; |
| } |
| } |
| |
| static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, |
| struct loaded_vmcs *prev) |
| { |
| struct vmcs_host_state *dest, *src; |
| |
| if (unlikely(!vmx->guest_state_loaded)) |
| return; |
| |
| src = &prev->host_state; |
| dest = &vmx->loaded_vmcs->host_state; |
| |
| vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); |
| dest->ldt_sel = src->ldt_sel; |
| #ifdef CONFIG_X86_64 |
| dest->ds_sel = src->ds_sel; |
| dest->es_sel = src->es_sel; |
| #endif |
| } |
| |
| static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct loaded_vmcs *prev; |
| int cpu; |
| |
| if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) |
| return; |
| |
| cpu = get_cpu(); |
| prev = vmx->loaded_vmcs; |
| vmx->loaded_vmcs = vmcs; |
| vmx_vcpu_load_vmcs(vcpu, cpu, prev); |
| vmx_sync_vmcs_host_state(vmx, prev); |
| put_cpu(); |
| |
| vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; |
| |
| /* |
| * All lazily updated registers will be reloaded from VMCS12 on both |
| * vmentry and vmexit. |
| */ |
| vcpu->arch.regs_dirty = 0; |
| } |
| |
| /* |
| * Free whatever needs to be freed from vmx->nested when L1 goes down, or |
| * just stops using VMX. |
| */ |
| static void free_nested(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) |
| vmx_switch_vmcs(vcpu, &vmx->vmcs01); |
| |
| if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) |
| return; |
| |
| kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); |
| |
| vmx->nested.vmxon = false; |
| vmx->nested.smm.vmxon = false; |
| vmx->nested.vmxon_ptr = INVALID_GPA; |
| free_vpid(vmx->nested.vpid02); |
| vmx->nested.posted_intr_nv = -1; |
| vmx->nested.current_vmptr = INVALID_GPA; |
| if (enable_shadow_vmcs) { |
| vmx_disable_shadow_vmcs(vmx); |
| vmcs_clear(vmx->vmcs01.shadow_vmcs); |
| free_vmcs(vmx->vmcs01.shadow_vmcs); |
| vmx->vmcs01.shadow_vmcs = NULL; |
| } |
| kfree(vmx->nested.cached_vmcs12); |
| vmx->nested.cached_vmcs12 = NULL; |
| kfree(vmx->nested.cached_shadow_vmcs12); |
| vmx->nested.cached_shadow_vmcs12 = NULL; |
| /* |
| * Unpin physical memory we referred to in the vmcs02. The APIC access |
| * page's backing page (yeah, confusing) shouldn't actually be accessed, |
| * and if it is written, the contents are irrelevant. |
| */ |
| kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); |
| kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); |
| kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); |
| vmx->nested.pi_desc = NULL; |
| |
| kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); |
| |
| nested_release_evmcs(vcpu); |
| |
| free_loaded_vmcs(&vmx->nested.vmcs02); |
| } |
| |
| /* |
| * Ensure that the current vmcs of the logical processor is the |
| * vmcs01 of the vcpu before calling free_nested(). |
| */ |
| void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) |
| { |
| vcpu_load(vcpu); |
| vmx_leave_nested(vcpu); |
| vcpu_put(vcpu); |
| } |
| |
| #define EPTP_PA_MASK GENMASK_ULL(51, 12) |
| |
| static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) |
| { |
| return VALID_PAGE(root_hpa) && |
| ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); |
| } |
| |
| static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, |
| gpa_t addr) |
| { |
| uint i; |
| struct kvm_mmu_root_info *cached_root; |
| |
| WARN_ON_ONCE(!mmu_is_nested(vcpu)); |
| |
| for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { |
| cached_root = &vcpu->arch.mmu->prev_roots[i]; |
| |
| if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, |
| eptp)) |
| vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa); |
| } |
| } |
| |
| static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, |
| struct x86_exception *fault) |
| { |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| u32 vm_exit_reason; |
| unsigned long exit_qualification = vcpu->arch.exit_qualification; |
| |
| if (vmx->nested.pml_full) { |
| vm_exit_reason = EXIT_REASON_PML_FULL; |
| vmx->nested.pml_full = false; |
| exit_qualification &= INTR_INFO_UNBLOCK_NMI; |
| } else { |
| if (fault->error_code & PFERR_RSVD_MASK) |
| vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; |
| else |
| vm_exit_reason = EXIT_REASON_EPT_VIOLATION; |
| |
| /* |
| * Although the caller (kvm_inject_emulated_page_fault) would |
| * have already synced the faulting address in the shadow EPT |
| * tables for the current EPTP12, we also need to sync it for |
| * any other cached EPTP02s based on the same EP4TA, since the |
| * TLB associates mappings to the EP4TA rather than the full EPTP. |
| */ |
| nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, |
| fault->address); |
| } |
| |
| nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); |
| vmcs12->guest_physical_address = fault->address; |
| } |
| |
| static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; |
| int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); |
| |
| kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, |
| nested_ept_ad_enabled(vcpu), |
| nested_ept_get_eptp(vcpu)); |
| } |
| |
| static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) |
| { |
| WARN_ON(mmu_is_nested(vcpu)); |
| |
| vcpu->arch.mmu = &vcpu->arch.guest_mmu; |
| nested_ept_new_eptp(vcpu); |
| vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; |
| vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; |
| vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; |
| |
| vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; |
| } |
| |
| static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) |
| { |
| vcpu->arch.mmu = &vcpu->arch.root_mmu; |
| vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; |
| } |
| |
| static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, |
| u16 error_code) |
| { |
| bool inequality, bit; |
| |
| bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; |
| inequality = |
| (error_code & vmcs12->page_fault_error_code_mask) != |
| vmcs12->page_fault_error_code_match; |
| return inequality ^ bit; |
| } |
| |
| static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, |
| u32 error_code) |
| { |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| |
| /* |
| * Drop bits 31:16 of the error code when performing the #PF mask+match |
| * check. All VMCS fields involved are 32 bits, but Intel CPUs never |
| * set bits 31:16 and VMX disallows setting bits 31:16 in the injected |
| * error code. Including the to-be-dropped bits in the check might |
| * result in an "impossible" or missed exit from L1's perspective. |
| */ |
| if (vector == PF_VECTOR) |
| return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); |
| |
| return (vmcs12->exception_bitmap & (1u << vector)); |
| } |
| |
| static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) |
| return 0; |
| |
| if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || |
| CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) |
| return 0; |
| |
| if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) |
| return 0; |
| |
| if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| /* |
| * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 |
| * itself utilizing x2APIC. All MSRs were previously set to be intercepted, |
| * only the "disable intercept" case needs to be handled. |
| */ |
| static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, |
| unsigned long *msr_bitmap_l0, |
| u32 msr, int type) |
| { |
| if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) |
| vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); |
| |
| if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) |
| vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); |
| } |
| |
| static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) |
| { |
| int msr; |
| |
| for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { |
| unsigned word = msr / BITS_PER_LONG; |
| |
| msr_bitmap[word] = ~0; |
| msr_bitmap[word + (0x800 / sizeof(long))] = ~0; |
| } |
| } |
| |
| #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ |
| static inline \ |
| void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ |
| unsigned long *msr_bitmap_l1, \ |
| unsigned long *msr_bitmap_l0, u32 msr) \ |
| { \ |
| if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ |
| vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ |
| vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ |
| else \ |
| vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ |
| } |
| BUILD_NVMX_MSR_INTERCEPT_HELPER(read) |
| BUILD_NVMX_MSR_INTERCEPT_HELPER(write) |
| |
| static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, |
| unsigned long *msr_bitmap_l1, |
| unsigned long *msr_bitmap_l0, |
| u32 msr, int types) |
| { |
| if (types & MSR_TYPE_R) |
| nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, |
| msr_bitmap_l0, msr); |
| if (types & MSR_TYPE_W) |
| nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, |
| msr_bitmap_l0, msr); |
| } |
| |
| /* |
| * Merge L0's and L1's MSR bitmap, return false to indicate that |
| * we do not use the hardware. |
| */ |
| static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| int msr; |
| unsigned long *msr_bitmap_l1; |
| unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; |
| struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; |
| struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; |
| |
| /* Nothing to do if the MSR bitmap is not in use. */ |
| if (!cpu_has_vmx_msr_bitmap() || |
| !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) |
| return false; |
| |
| /* |
| * MSR bitmap update can be skipped when: |
| * - MSR bitmap for L1 hasn't changed. |
| * - Nested hypervisor (L1) is attempting to launch the same L2 as |
| * before. |
| * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature |
| * and tells KVM (L0) there were no changes in MSR bitmap for L2. |
| */ |
| if (!vmx->nested.force_msr_bitmap_recalc && evmcs && |
| evmcs->hv_enlightenments_control.msr_bitmap && |
| evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) |
| return true; |
| |
| if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) |
| return false; |
| |
| msr_bitmap_l1 = (unsigned long *)map->hva; |
| |
| /* |
| * To keep the control flow simple, pay eight 8-byte writes (sixteen |
| * 4-byte writes on 32-bit systems) up front to enable intercepts for |
| * the x2APIC MSR range and selectively toggle those relevant to L2. |
| */ |
| enable_x2apic_msr_intercepts(msr_bitmap_l0); |
| |
| if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { |
| if (nested_cpu_has_apic_reg_virt(vmcs12)) { |
| /* |
| * L0 need not intercept reads for MSRs between 0x800 |
| * and 0x8ff, it just lets the processor take the value |
| * from the virtual-APIC page; take those 256 bits |
| * directly from the L1 bitmap. |
| */ |
| for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { |
| unsigned word = msr / BITS_PER_LONG; |
| |
| msr_bitmap_l0[word] = msr_bitmap_l1[word]; |
| } |
| } |
| |
| nested_vmx_disable_intercept_for_x2apic_msr( |
| msr_bitmap_l1, msr_bitmap_l0, |
| X2APIC_MSR(APIC_TASKPRI), |
| MSR_TYPE_R | MSR_TYPE_W); |
| |
| if (nested_cpu_has_vid(vmcs12)) { |
| nested_vmx_disable_intercept_for_x2apic_msr( |
| msr_bitmap_l1, msr_bitmap_l0, |
| X2APIC_MSR(APIC_EOI), |
| MSR_TYPE_W); |
| nested_vmx_disable_intercept_for_x2apic_msr( |
| msr_bitmap_l1, msr_bitmap_l0, |
| X2APIC_MSR(APIC_SELF_IPI), |
| MSR_TYPE_W); |
| } |
| } |
| |
| /* |
| * Always check vmcs01's bitmap to honor userspace MSR filters and any |
| * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. |
| */ |
| #ifdef CONFIG_X86_64 |
| nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, |
| MSR_FS_BASE, MSR_TYPE_RW); |
| |
| nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, |
| MSR_GS_BASE, MSR_TYPE_RW); |
| |
| nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, |
| MSR_KERNEL_GS_BASE, MSR_TYPE_RW); |
| #endif |
| nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, |
| MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); |
| |
| nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, |
| MSR_IA32_PRED_CMD, MSR_TYPE_W); |
| |
| kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); |
| |
| vmx->nested.force_msr_bitmap_recalc = false; |
| |
| return true; |
| } |
| |
| static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; |
| |
| if (!nested_cpu_has_shadow_vmcs(vmcs12) || |
| vmcs12->vmcs_link_pointer == INVALID_GPA) |
| return; |
| |
| if (ghc->gpa != vmcs12->vmcs_link_pointer && |
| kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, |
| vmcs12->vmcs_link_pointer, VMCS12_SIZE)) |
| return; |
| |
| kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), |
| VMCS12_SIZE); |
| } |
| |
| static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; |
| |
| if (!nested_cpu_has_shadow_vmcs(vmcs12) || |
| vmcs12->vmcs_link_pointer == INVALID_GPA) |
| return; |
| |
| if (ghc->gpa != vmcs12->vmcs_link_pointer && |
| kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, |
| vmcs12->vmcs_link_pointer, VMCS12_SIZE)) |
| return; |
| |
| kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), |
| VMCS12_SIZE); |
| } |
| |
| /* |
| * In nested virtualization, check if L1 has set |
| * VM_EXIT_ACK_INTR_ON_EXIT |
| */ |
| static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) |
| { |
| return get_vmcs12(vcpu)->vm_exit_controls & |
| VM_EXIT_ACK_INTR_ON_EXIT; |
| } |
| |
| static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && |
| CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) |
| return -EINVAL; |
| else |
| return 0; |
| } |
| |
| static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && |
| !nested_cpu_has_apic_reg_virt(vmcs12) && |
| !nested_cpu_has_vid(vmcs12) && |
| !nested_cpu_has_posted_intr(vmcs12)) |
| return 0; |
| |
| /* |
| * If virtualize x2apic mode is enabled, |
| * virtualize apic access must be disabled. |
| */ |
| if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && |
| nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) |
| return -EINVAL; |
| |
| /* |
| * If virtual interrupt delivery is enabled, |
| * we must exit on external interrupts. |
| */ |
| if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) |
| return -EINVAL; |
| |
| /* |
| * bits 15:8 should be zero in posted_intr_nv, |
| * the descriptor address has been already checked |
| * in nested_get_vmcs12_pages. |
| * |
| * bits 5:0 of posted_intr_desc_addr should be zero. |
| */ |
| if (nested_cpu_has_posted_intr(vmcs12) && |
| (CC(!nested_cpu_has_vid(vmcs12)) || |
| CC(!nested_exit_intr_ack_set(vcpu)) || |
| CC((vmcs12->posted_intr_nv & 0xff00)) || |
| CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) |
| return -EINVAL; |
| |
| /* tpr shadow is needed by all apicv features. */ |
| if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, |
| u32 count, u64 addr) |
| { |
| if (count == 0) |
| return 0; |
| |
| if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || |
| !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (CC(nested_vmx_check_msr_switch(vcpu, |
| vmcs12->vm_exit_msr_load_count, |
| vmcs12->vm_exit_msr_load_addr)) || |
| CC(nested_vmx_check_msr_switch(vcpu, |
| vmcs12->vm_exit_msr_store_count, |
| vmcs12->vm_exit_msr_store_addr))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (CC(nested_vmx_check_msr_switch(vcpu, |
| vmcs12->vm_entry_msr_load_count, |
| vmcs12->vm_entry_msr_load_addr))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has_pml(vmcs12)) |
| return 0; |
| |
| if (CC(!nested_cpu_has_ept(vmcs12)) || |
| CC(!page_address_valid(vcpu, vmcs12->pml_address))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && |
| !nested_cpu_has_ept(vmcs12))) |
| return -EINVAL; |
| return 0; |
| } |
| |
| static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && |
| !nested_cpu_has_ept(vmcs12))) |
| return -EINVAL; |
| return 0; |
| } |
| |
| static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (!nested_cpu_has_shadow_vmcs(vmcs12)) |
| return 0; |
| |
| if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || |
| CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, |
| struct vmx_msr_entry *e) |
| { |
| /* x2APIC MSR accesses are not allowed */ |
| if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) |
| return -EINVAL; |
| if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ |
| CC(e->index == MSR_IA32_UCODE_REV)) |
| return -EINVAL; |
| if (CC(e->reserved != 0)) |
| return -EINVAL; |
| return 0; |
| } |
| |
| static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, |
| struct vmx_msr_entry *e) |
| { |
| if (CC(e->index == MSR_FS_BASE) || |
| CC(e->index == MSR_GS_BASE) || |
| CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ |
| nested_vmx_msr_check_common(vcpu, e)) |
| return -EINVAL; |
| return 0; |
| } |
| |
| static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, |
| struct vmx_msr_entry *e) |
| { |
| if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ |
| nested_vmx_msr_check_common(vcpu, e)) |
| return -EINVAL; |
| return 0; |
| } |
| |
| static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, |
| vmx->nested.msrs.misc_high); |
| |
| return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; |
| } |
| |
| /* |
| * Load guest's/host's msr at nested entry/exit. |
| * return 0 for success, entry index for failure. |
| * |
| * One of the failure modes for MSR load/store is when a list exceeds the |
| * virtual hardware's capacity. To maintain compatibility with hardware inasmuch |
| * as possible, process all valid entries before failing rather than precheck |
| * for a capacity violation. |
| */ |
| static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) |
| { |
| u32 i; |
| struct vmx_msr_entry e; |
| u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); |
| |
| for (i = 0; i < count; i++) { |
| if (unlikely(i >= max_msr_list_size)) |
| goto fail; |
| |
| if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), |
| &e, sizeof(e))) { |
| pr_debug_ratelimited( |
| "%s cannot read MSR entry (%u, 0x%08llx)\n", |
| __func__, i, gpa + i * sizeof(e)); |
| goto fail; |
| } |
| if (nested_vmx_load_msr_check(vcpu, &e)) { |
| pr_debug_ratelimited( |
| "%s check failed (%u, 0x%x, 0x%x)\n", |
| __func__, i, e.index, e.reserved); |
| goto fail; |
| } |
| if (kvm_set_msr(vcpu, e.index, e.value)) { |
| pr_debug_ratelimited( |
| "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", |
| __func__, i, e.index, e.value); |
| goto fail; |
| } |
| } |
| return 0; |
| fail: |
| /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ |
| return i + 1; |
| } |
| |
| static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, |
| u32 msr_index, |
| u64 *data) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* |
| * If the L0 hypervisor stored a more accurate value for the TSC that |
| * does not include the time taken for emulation of the L2->L1 |
| * VM-exit in L0, use the more accurate value. |
| */ |
| if (msr_index == MSR_IA32_TSC) { |
| int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, |
| MSR_IA32_TSC); |
| |
| if (i >= 0) { |
| u64 val = vmx->msr_autostore.guest.val[i].value; |
| |
| *data = kvm_read_l1_tsc(vcpu, val); |
| return true; |
| } |
| } |
| |
| if (kvm_get_msr(vcpu, msr_index, data)) { |
| pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, |
| msr_index); |
| return false; |
| } |
| return true; |
| } |
| |
| static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, |
| struct vmx_msr_entry *e) |
| { |
| if (kvm_vcpu_read_guest(vcpu, |
| gpa + i * sizeof(*e), |
| e, 2 * sizeof(u32))) { |
| pr_debug_ratelimited( |
| "%s cannot read MSR entry (%u, 0x%08llx)\n", |
| __func__, i, gpa + i * sizeof(*e)); |
| return false; |
| } |
| if (nested_vmx_store_msr_check(vcpu, e)) { |
| pr_debug_ratelimited( |
| "%s check failed (%u, 0x%x, 0x%x)\n", |
| __func__, i, e->index, e->reserved); |
| return false; |
| } |
| return true; |
| } |
| |
| static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) |
| { |
| u64 data; |
| u32 i; |
| struct vmx_msr_entry e; |
| u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); |
| |
| for (i = 0; i < count; i++) { |
| if (unlikely(i >= max_msr_list_size)) |
| return -EINVAL; |
| |
| if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) |
| return -EINVAL; |
| |
| if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) |
| return -EINVAL; |
| |
| if (kvm_vcpu_write_guest(vcpu, |
| gpa + i * sizeof(e) + |
| offsetof(struct vmx_msr_entry, value), |
| &data, sizeof(data))) { |
| pr_debug_ratelimited( |
| "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", |
| __func__, i, e.index, data); |
| return -EINVAL; |
| } |
| } |
| return 0; |
| } |
| |
| static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) |
| { |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| u32 count = vmcs12->vm_exit_msr_store_count; |
| u64 gpa = vmcs12->vm_exit_msr_store_addr; |
| struct vmx_msr_entry e; |
| u32 i; |
| |
| for (i = 0; i < count; i++) { |
| if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) |
| return false; |
| |
| if (e.index == msr_index) |
| return true; |
| } |
| return false; |
| } |
| |
| static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, |
| u32 msr_index) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct vmx_msrs *autostore = &vmx->msr_autostore.guest; |
| bool in_vmcs12_store_list; |
| int msr_autostore_slot; |
| bool in_autostore_list; |
| int last; |
| |
| msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); |
| in_autostore_list = msr_autostore_slot >= 0; |
| in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); |
| |
| if (in_vmcs12_store_list && !in_autostore_list) { |
| if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { |
| /* |
| * Emulated VMEntry does not fail here. Instead a less |
| * accurate value will be returned by |
| * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() |
| * instead of reading the value from the vmcs02 VMExit |
| * MSR-store area. |
| */ |
| pr_warn_ratelimited( |
| "Not enough msr entries in msr_autostore. Can't add msr %x\n", |
| msr_index); |
| return; |
| } |
| last = autostore->nr++; |
| autostore->val[last].index = msr_index; |
| } else if (!in_vmcs12_store_list && in_autostore_list) { |
| last = --autostore->nr; |
| autostore->val[msr_autostore_slot] = autostore->val[last]; |
| } |
| } |
| |
| /* |
| * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are |
| * emulating VM-Entry into a guest with EPT enabled. On failure, the expected |
| * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to |
| * @entry_failure_code. |
| */ |
| static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, |
| bool nested_ept, bool reload_pdptrs, |
| enum vm_entry_failure_code *entry_failure_code) |
| { |
| if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { |
| *entry_failure_code = ENTRY_FAIL_DEFAULT; |
| return -EINVAL; |
| } |
| |
| /* |
| * If PAE paging and EPT are both on, CR3 is not used by the CPU and |
| * must not be dereferenced. |
| */ |
| if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && |
| CC(!load_pdptrs(vcpu, cr3))) { |
| *entry_failure_code = ENTRY_FAIL_PDPTE; |
| return -EINVAL; |
| } |
| |
| vcpu->arch.cr3 = cr3; |
| kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); |
| |
| /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ |
| kvm_init_mmu(vcpu); |
| |
| if (!nested_ept) |
| kvm_mmu_new_pgd(vcpu, cr3); |
| |
| return 0; |
| } |
| |
| /* |
| * Returns if KVM is able to config CPU to tag TLB entries |
| * populated by L2 differently than TLB entries populated |
| * by L1. |
| * |
| * If L0 uses EPT, L1 and L2 run with different EPTP because |
| * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries |
| * are tagged with different EPTP. |
| * |
| * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged |
| * with different VPID (L1 entries are tagged with vmx->vpid |
| * while L2 entries are tagged with vmx->nested.vpid02). |
| */ |
| static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) |
| { |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| |
| return enable_ept || |
| (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); |
| } |
| |
| static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12, |
| bool is_vmenter) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* |
| * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or |
| * L2's VP_ID upon request from the guest. Make sure we check for |
| * pending entries in the right FIFO upon L1/L2 transition as these |
| * requests are put by other vCPUs asynchronously. |
| */ |
| if (to_hv_vcpu(vcpu) && enable_ept) |
| kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); |
| |
| /* |
| * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings |
| * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a |
| * full TLB flush from the guest's perspective. This is required even |
| * if VPID is disabled in the host as KVM may need to synchronize the |
| * MMU in response to the guest TLB flush. |
| * |
| * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. |
| * EPT is a special snowflake, as guest-physical mappings aren't |
| * flushed on VPID invalidations, including VM-Enter or VM-Exit with |
| * VPID disabled. As a result, KVM _never_ needs to sync nEPT |
| * entries on VM-Enter because L1 can't rely on VM-Enter to flush |
| * those mappings. |
| */ |
| if (!nested_cpu_has_vpid(vmcs12)) { |
| kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); |
| return; |
| } |
| |
| /* L2 should never have a VPID if VPID is disabled. */ |
| WARN_ON(!enable_vpid); |
| |
| /* |
| * VPID is enabled and in use by vmcs12. If vpid12 is changing, then |
| * emulate a guest TLB flush as KVM does not track vpid12 history nor |
| * is the VPID incorporated into the MMU context. I.e. KVM must assume |
| * that the new vpid12 has never been used and thus represents a new |
| * guest ASID that cannot have entries in the TLB. |
| */ |
| if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { |
| vmx->nested.last_vpid = vmcs12->virtual_processor_id; |
| kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); |
| return; |
| } |
| |
| /* |
| * If VPID is enabled, used by vmc12, and vpid12 is not changing but |
| * does not have a unique TLB tag (ASID), i.e. EPT is disabled and |
| * KVM was unable to allocate a VPID for L2, flush the current context |
| * as the effective ASID is common to both L1 and L2. |
| */ |
| if (!nested_has_guest_tlb_tag(vcpu)) |
| kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); |
| } |
| |
| static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) |
| { |
| superset &= mask; |
| subset &= mask; |
| |
| return (superset | subset) == superset; |
| } |
| |
| static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) |
| { |
| const u64 feature_and_reserved = |
| /* feature (except bit 48; see below) */ |
| BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | |
| /* reserved */ |
| BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); |
| u64 vmx_basic = vmcs_config.nested.basic; |
| |
| if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) |
| return -EINVAL; |
| |
| /* |
| * KVM does not emulate a version of VMX that constrains physical |
| * addresses of VMX structures (e.g. VMCS) to 32-bits. |
| */ |
| if (data & BIT_ULL(48)) |
| return -EINVAL; |
| |
| if (vmx_basic_vmcs_revision_id(vmx_basic) != |
| vmx_basic_vmcs_revision_id(data)) |
| return -EINVAL; |
| |
| if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) |
| return -EINVAL; |
| |
| vmx->nested.msrs.basic = data; |
| return 0; |
| } |
| |
| static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index, |
| u32 **low, u32 **high) |
| { |
| switch (msr_index) { |
| case MSR_IA32_VMX_TRUE_PINBASED_CTLS: |
| *low = &msrs->pinbased_ctls_low; |
| *high = &msrs->pinbased_ctls_high; |
| break; |
| case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: |
| *low = &msrs->procbased_ctls_low; |
| *high = &msrs->procbased_ctls_high; |
| break; |
| case MSR_IA32_VMX_TRUE_EXIT_CTLS: |
| *low = &msrs->exit_ctls_low; |
| *high = &msrs->exit_ctls_high; |
| break; |
| case MSR_IA32_VMX_TRUE_ENTRY_CTLS: |
| *low = &msrs->entry_ctls_low; |
| *high = &msrs->entry_ctls_high; |
| break; |
| case MSR_IA32_VMX_PROCBASED_CTLS2: |
| *low = &msrs->secondary_ctls_low; |
| *high = &msrs->secondary_ctls_high; |
| break; |
| default: |
| BUG(); |
| } |
| } |
| |
| static int |
| vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) |
| { |
| u32 *lowp, *highp; |
| u64 supported; |
| |
| vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); |
| |
| supported = vmx_control_msr(*lowp, *highp); |
| |
| /* Check must-be-1 bits are still 1. */ |
| if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) |
| return -EINVAL; |
| |
| /* Check must-be-0 bits are still 0. */ |
| if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) |
| return -EINVAL; |
| |
| vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); |
| *lowp = data; |
| *highp = data >> 32; |
| return 0; |
| } |
| |
| static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) |
| { |
| const u64 feature_and_reserved_bits = |
| /* feature */ |
| BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | |
| BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | |
| /* reserved */ |
| GENMASK_ULL(13, 9) | BIT_ULL(31); |
| u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, |
| vmcs_config.nested.misc_high); |
| |
| if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) |
| return -EINVAL; |
| |
| if ((vmx->nested.msrs.pinbased_ctls_high & |
| PIN_BASED_VMX_PREEMPTION_TIMER) && |
| vmx_misc_preemption_timer_rate(data) != |
| vmx_misc_preemption_timer_rate(vmx_misc)) |
| return -EINVAL; |
| |
| if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) |
| return -EINVAL; |
| |
| if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) |
| return -EINVAL; |
| |
| if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) |
| return -EINVAL; |
| |
| vmx->nested.msrs.misc_low = data; |
| vmx->nested.msrs.misc_high = data >> 32; |
| |
| return 0; |
| } |
| |
| static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) |
| { |
| u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, |
| vmcs_config.nested.vpid_caps); |
| |
| /* Every bit is either reserved or a feature bit. */ |
| if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) |
| return -EINVAL; |
| |
| vmx->nested.msrs.ept_caps = data; |
| vmx->nested.msrs.vpid_caps = data >> 32; |
| return 0; |
| } |
| |
| static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index) |
| { |
| switch (msr_index) { |
| case MSR_IA32_VMX_CR0_FIXED0: |
| return &msrs->cr0_fixed0; |
| case MSR_IA32_VMX_CR4_FIXED0: |
| return &msrs->cr4_fixed0; |
| default: |
| BUG(); |
| } |
| } |
| |
| static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) |
| { |
| const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); |
| |
| /* |
| * 1 bits (which indicates bits which "must-be-1" during VMX operation) |
| * must be 1 in the restored value. |
| */ |
| if (!is_bitwise_subset(data, *msr, -1ULL)) |
| return -EINVAL; |
| |
| *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; |
| return 0; |
| } |
| |
| /* |
| * Called when userspace is restoring VMX MSRs. |
| * |
| * Returns 0 on success, non-0 otherwise. |
| */ |
| int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* |
| * Don't allow changes to the VMX capability MSRs while the vCPU |
| * is in VMX operation. |
| */ |
| if (vmx->nested.vmxon) |
| return -EBUSY; |
| |
| switch (msr_index) { |
| case MSR_IA32_VMX_BASIC: |
| return vmx_restore_vmx_basic(vmx, data); |
| case MSR_IA32_VMX_PINBASED_CTLS: |
| case MSR_IA32_VMX_PROCBASED_CTLS: |
| case MSR_IA32_VMX_EXIT_CTLS: |
| case MSR_IA32_VMX_ENTRY_CTLS: |
| /* |
| * The "non-true" VMX capability MSRs are generated from the |
| * "true" MSRs, so we do not support restoring them directly. |
| * |
| * If userspace wants to emulate VMX_BASIC[55]=0, userspace |
| * should restore the "true" MSRs with the must-be-1 bits |
| * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND |
| * DEFAULT SETTINGS". |
| */ |
| return -EINVAL; |
| case MSR_IA32_VMX_TRUE_PINBASED_CTLS: |
| case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: |
| case MSR_IA32_VMX_TRUE_EXIT_CTLS: |
| case MSR_IA32_VMX_TRUE_ENTRY_CTLS: |
| case MSR_IA32_VMX_PROCBASED_CTLS2: |
| return vmx_restore_control_msr(vmx, msr_index, data); |
| case MSR_IA32_VMX_MISC: |
| return vmx_restore_vmx_misc(vmx, data); |
| case MSR_IA32_VMX_CR0_FIXED0: |
| case MSR_IA32_VMX_CR4_FIXED0: |
| return vmx_restore_fixed0_msr(vmx, msr_index, data); |
| case MSR_IA32_VMX_CR0_FIXED1: |
| case MSR_IA32_VMX_CR4_FIXED1: |
| /* |
| * These MSRs are generated based on the vCPU's CPUID, so we |
| * do not support restoring them directly. |
| */ |
| return -EINVAL; |
| case MSR_IA32_VMX_EPT_VPID_CAP: |
| return vmx_restore_vmx_ept_vpid_cap(vmx, data); |
| case MSR_IA32_VMX_VMCS_ENUM: |
| vmx->nested.msrs.vmcs_enum = data; |
| return 0; |
| case MSR_IA32_VMX_VMFUNC: |
| if (data & ~vmcs_config.nested.vmfunc_controls) |
| return -EINVAL; |
| vmx->nested.msrs.vmfunc_controls = data; |
| return 0; |
| default: |
| /* |
| * The rest of the VMX capability MSRs do not support restore. |
| */ |
| return -EINVAL; |
| } |
| } |
| |
| /* Returns 0 on success, non-0 otherwise. */ |
| int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) |
| { |
| switch (msr_index) { |
| case MSR_IA32_VMX_BASIC: |
| *pdata = msrs->basic; |
| break; |
| case MSR_IA32_VMX_TRUE_PINBASED_CTLS: |
| case MSR_IA32_VMX_PINBASED_CTLS: |
| *pdata = vmx_control_msr( |
| msrs->pinbased_ctls_low, |
| msrs->pinbased_ctls_high); |
| if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) |
| *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; |
| break; |
| case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: |
| case MSR_IA32_VMX_PROCBASED_CTLS: |
| *pdata = vmx_control_msr( |
| msrs->procbased_ctls_low, |
| msrs->procbased_ctls_high); |
| if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) |
| *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; |
| break; |
| case MSR_IA32_VMX_TRUE_EXIT_CTLS: |
| case MSR_IA32_VMX_EXIT_CTLS: |
| *pdata = vmx_control_msr( |
| msrs->exit_ctls_low, |
| msrs->exit_ctls_high); |
| if (msr_index == MSR_IA32_VMX_EXIT_CTLS) |
| *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; |
| break; |
| case MSR_IA32_VMX_TRUE_ENTRY_CTLS: |
| case MSR_IA32_VMX_ENTRY_CTLS: |
| *pdata = vmx_control_msr( |
| msrs->entry_ctls_low, |
| msrs->entry_ctls_high); |
| if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) |
| *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; |
| break; |
| case MSR_IA32_VMX_MISC: |
| *pdata = vmx_control_msr( |
| msrs->misc_low, |
| msrs->misc_high); |
| break; |
| case MSR_IA32_VMX_CR0_FIXED0: |
| *pdata = msrs->cr0_fixed0; |
| break; |
| case MSR_IA32_VMX_CR0_FIXED1: |
| *pdata = msrs->cr0_fixed1; |
| break; |
| case MSR_IA32_VMX_CR4_FIXED0: |
| *pdata = msrs->cr4_fixed0; |
| break; |
| case MSR_IA32_VMX_CR4_FIXED1: |
| *pdata = msrs->cr4_fixed1; |
| break; |
| case MSR_IA32_VMX_VMCS_ENUM: |
| *pdata = msrs->vmcs_enum; |
| break; |
| case MSR_IA32_VMX_PROCBASED_CTLS2: |
| *pdata = vmx_control_msr( |
| msrs->secondary_ctls_low, |
| msrs->secondary_ctls_high); |
| break; |
| case MSR_IA32_VMX_EPT_VPID_CAP: |
| *pdata = msrs->ept_caps | |
| ((u64)msrs->vpid_caps << 32); |
| break; |
| case MSR_IA32_VMX_VMFUNC: |
| *pdata = msrs->vmfunc_controls; |
| break; |
| default: |
| return 1; |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Copy the writable VMCS shadow fields back to the VMCS12, in case they have |
| * been modified by the L1 guest. Note, "writable" in this context means |
| * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of |
| * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" |
| * VM-exit information fields (which are actually writable if the vCPU is |
| * configured to support "VMWRITE to any supported field in the VMCS"). |
| */ |
| static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) |
| { |
| struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; |
| struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); |
| struct shadow_vmcs_field field; |
| unsigned long val; |
| int i; |
| |
| if (WARN_ON(!shadow_vmcs)) |
| return; |
| |
| preempt_disable(); |
| |
| vmcs_load(shadow_vmcs); |
| |
| for (i = 0; i < max_shadow_read_write_fields; i++) { |
| field = shadow_read_write_fields[i]; |
| val = __vmcs_readl(field.encoding); |
| vmcs12_write_any(vmcs12, field.encoding, field.offset, val); |
| } |
| |
| vmcs_clear(shadow_vmcs); |
| vmcs_load(vmx->loaded_vmcs->vmcs); |
| |
| preempt_enable(); |
| } |
| |
| static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) |
| { |
| const struct shadow_vmcs_field *fields[] = { |
| shadow_read_write_fields, |
| shadow_read_only_fields |
| }; |
| const int max_fields[] = { |
| max_shadow_read_write_fields, |
| max_shadow_read_only_fields |
| }; |
| struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; |
| struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); |
| struct shadow_vmcs_field field; |
| unsigned long val; |
| int i, q; |
| |
| if (WARN_ON(!shadow_vmcs)) |
| return; |
| |
| vmcs_load(shadow_vmcs); |
| |
| for (q = 0; q < ARRAY_SIZE(fields); q++) { |
| for (i = 0; i < max_fields[q]; i++) { |
| field = fields[q][i]; |
| val = vmcs12_read_any(vmcs12, field.encoding, |
| field.offset); |
| __vmcs_writel(field.encoding, val); |
| } |
| } |
| |
| vmcs_clear(shadow_vmcs); |
| vmcs_load(vmx->loaded_vmcs->vmcs); |
| } |
| |
| static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) |
| { |
| struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; |
| struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; |
| struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); |
| |
| /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ |
| vmcs12->tpr_threshold = evmcs->tpr_threshold; |
| vmcs12->guest_rip = evmcs->guest_rip; |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { |
| hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; |
| hv_vcpu->nested.vm_id = evmcs->hv_vm_id; |
| hv_vcpu->nested.vp_id = evmcs->hv_vp_id; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { |
| vmcs12->guest_rsp = evmcs->guest_rsp; |
| vmcs12->guest_rflags = evmcs->guest_rflags; |
| vmcs12->guest_interruptibility_info = |
| evmcs->guest_interruptibility_info; |
| /* |
| * Not present in struct vmcs12: |
| * vmcs12->guest_ssp = evmcs->guest_ssp; |
| */ |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { |
| vmcs12->cpu_based_vm_exec_control = |
| evmcs->cpu_based_vm_exec_control; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { |
| vmcs12->exception_bitmap = evmcs->exception_bitmap; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { |
| vmcs12->vm_entry_controls = evmcs->vm_entry_controls; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { |
| vmcs12->vm_entry_intr_info_field = |
| evmcs->vm_entry_intr_info_field; |
| vmcs12->vm_entry_exception_error_code = |
| evmcs->vm_entry_exception_error_code; |
| vmcs12->vm_entry_instruction_len = |
| evmcs->vm_entry_instruction_len; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { |
| vmcs12->host_ia32_pat = evmcs->host_ia32_pat; |
| vmcs12->host_ia32_efer = evmcs->host_ia32_efer; |
| vmcs12->host_cr0 = evmcs->host_cr0; |
| vmcs12->host_cr3 = evmcs->host_cr3; |
| vmcs12->host_cr4 = evmcs->host_cr4; |
| vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; |
| vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; |
| vmcs12->host_rip = evmcs->host_rip; |
| vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; |
| vmcs12->host_es_selector = evmcs->host_es_selector; |
| vmcs12->host_cs_selector = evmcs->host_cs_selector; |
| vmcs12->host_ss_selector = evmcs->host_ss_selector; |
| vmcs12->host_ds_selector = evmcs->host_ds_selector; |
| vmcs12->host_fs_selector = evmcs->host_fs_selector; |
| vmcs12->host_gs_selector = evmcs->host_gs_selector; |
| vmcs12->host_tr_selector = evmcs->host_tr_selector; |
| vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; |
| /* |
| * Not present in struct vmcs12: |
| * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; |
| * vmcs12->host_ssp = evmcs->host_ssp; |
| * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; |
| */ |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { |
| vmcs12->pin_based_vm_exec_control = |
| evmcs->pin_based_vm_exec_control; |
| vmcs12->vm_exit_controls = evmcs->vm_exit_controls; |
| vmcs12->secondary_vm_exec_control = |
| evmcs->secondary_vm_exec_control; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { |
| vmcs12->io_bitmap_a = evmcs->io_bitmap_a; |
| vmcs12->io_bitmap_b = evmcs->io_bitmap_b; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { |
| vmcs12->msr_bitmap = evmcs->msr_bitmap; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { |
| vmcs12->guest_es_base = evmcs->guest_es_base; |
| vmcs12->guest_cs_base = evmcs->guest_cs_base; |
| vmcs12->guest_ss_base = evmcs->guest_ss_base; |
| vmcs12->guest_ds_base = evmcs->guest_ds_base; |
| vmcs12->guest_fs_base = evmcs->guest_fs_base; |
| vmcs12->guest_gs_base = evmcs->guest_gs_base; |
| vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; |
| vmcs12->guest_tr_base = evmcs->guest_tr_base; |
| vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; |
| vmcs12->guest_idtr_base = evmcs->guest_idtr_base; |
| vmcs12->guest_es_limit = evmcs->guest_es_limit; |
| vmcs12->guest_cs_limit = evmcs->guest_cs_limit; |
| vmcs12->guest_ss_limit = evmcs->guest_ss_limit; |
| vmcs12->guest_ds_limit = evmcs->guest_ds_limit; |
| vmcs12->guest_fs_limit = evmcs->guest_fs_limit; |
| vmcs12->guest_gs_limit = evmcs->guest_gs_limit; |
| vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; |
| vmcs12->guest_tr_limit = evmcs->guest_tr_limit; |
| vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; |
| vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; |
| vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; |
| vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; |
| vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; |
| vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; |
| vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; |
| vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; |
| vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; |
| vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; |
| vmcs12->guest_es_selector = evmcs->guest_es_selector; |
| vmcs12->guest_cs_selector = evmcs->guest_cs_selector; |
| vmcs12->guest_ss_selector = evmcs->guest_ss_selector; |
| vmcs12->guest_ds_selector = evmcs->guest_ds_selector; |
| vmcs12->guest_fs_selector = evmcs->guest_fs_selector; |
| vmcs12->guest_gs_selector = evmcs->guest_gs_selector; |
| vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; |
| vmcs12->guest_tr_selector = evmcs->guest_tr_selector; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { |
| vmcs12->tsc_offset = evmcs->tsc_offset; |
| vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; |
| vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; |
| vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; |
| vmcs12->tsc_multiplier = evmcs->tsc_multiplier; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { |
| vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; |
| vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; |
| vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; |
| vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; |
| vmcs12->guest_cr0 = evmcs->guest_cr0; |
| vmcs12->guest_cr3 = evmcs->guest_cr3; |
| vmcs12->guest_cr4 = evmcs->guest_cr4; |
| vmcs12->guest_dr7 = evmcs->guest_dr7; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { |
| vmcs12->host_fs_base = evmcs->host_fs_base; |
| vmcs12->host_gs_base = evmcs->host_gs_base; |
| vmcs12->host_tr_base = evmcs->host_tr_base; |
| vmcs12->host_gdtr_base = evmcs->host_gdtr_base; |
| vmcs12->host_idtr_base = evmcs->host_idtr_base; |
| vmcs12->host_rsp = evmcs->host_rsp; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { |
| vmcs12->ept_pointer = evmcs->ept_pointer; |
| vmcs12->virtual_processor_id = evmcs->virtual_processor_id; |
| } |
| |
| if (unlikely(!(hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { |
| vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; |
| vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; |
| vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; |
| vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; |
| vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; |
| vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; |
| vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; |
| vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; |
| vmcs12->guest_pending_dbg_exceptions = |
| evmcs->guest_pending_dbg_exceptions; |
| vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; |
| vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; |
| vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; |
| vmcs12->guest_activity_state = evmcs->guest_activity_state; |
| vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; |
| vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; |
| /* |
| * Not present in struct vmcs12: |
| * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; |
| * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; |
| * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; |
| */ |
| } |
| |
| /* |
| * Not used? |
| * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; |
| * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; |
| * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; |
| * vmcs12->page_fault_error_code_mask = |
| * evmcs->page_fault_error_code_mask; |
| * vmcs12->page_fault_error_code_match = |
| * evmcs->page_fault_error_code_match; |
| * vmcs12->cr3_target_count = evmcs->cr3_target_count; |
| * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; |
| * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; |
| * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; |
| */ |
| |
| /* |
| * Read only fields: |
| * vmcs12->guest_physical_address = evmcs->guest_physical_address; |
| * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; |
| * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; |
| * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; |
| * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; |
| * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; |
| * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; |
| * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; |
| * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; |
| * vmcs12->exit_qualification = evmcs->exit_qualification; |
| * vmcs12->guest_linear_address = evmcs->guest_linear_address; |
| * |
| * Not present in struct vmcs12: |
| * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; |
| * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; |
| * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; |
| * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; |
| */ |
| |
| return; |
| } |
| |
| static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) |
| { |
| struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; |
| struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; |
| |
| /* |
| * Should not be changed by KVM: |
| * |
| * evmcs->host_es_selector = vmcs12->host_es_selector; |
| * evmcs->host_cs_selector = vmcs12->host_cs_selector; |
| * evmcs->host_ss_selector = vmcs12->host_ss_selector; |
| * evmcs->host_ds_selector = vmcs12->host_ds_selector; |
| * evmcs->host_fs_selector = vmcs12->host_fs_selector; |
| * evmcs->host_gs_selector = vmcs12->host_gs_selector; |
| * evmcs->host_tr_selector = vmcs12->host_tr_selector; |
| * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; |
| * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; |
| * evmcs->host_cr0 = vmcs12->host_cr0; |
| * evmcs->host_cr3 = vmcs12->host_cr3; |
| * evmcs->host_cr4 = vmcs12->host_cr4; |
| * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; |
| * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; |
| * evmcs->host_rip = vmcs12->host_rip; |
| * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; |
| * evmcs->host_fs_base = vmcs12->host_fs_base; |
| * evmcs->host_gs_base = vmcs12->host_gs_base; |
| * evmcs->host_tr_base = vmcs12->host_tr_base; |
| * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; |
| * evmcs->host_idtr_base = vmcs12->host_idtr_base; |
| * evmcs->host_rsp = vmcs12->host_rsp; |
| * sync_vmcs02_to_vmcs12() doesn't read these: |
| * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; |
| * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; |
| * evmcs->msr_bitmap = vmcs12->msr_bitmap; |
| * evmcs->ept_pointer = vmcs12->ept_pointer; |
| * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; |
| * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; |
| * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; |
| * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; |
| * evmcs->tpr_threshold = vmcs12->tpr_threshold; |
| * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; |
| * evmcs->exception_bitmap = vmcs12->exception_bitmap; |
| * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; |
| * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; |
| * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; |
| * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; |
| * evmcs->page_fault_error_code_mask = |
| * vmcs12->page_fault_error_code_mask; |
| * evmcs->page_fault_error_code_match = |
| * vmcs12->page_fault_error_code_match; |
| * evmcs->cr3_target_count = vmcs12->cr3_target_count; |
| * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; |
| * evmcs->tsc_offset = vmcs12->tsc_offset; |
| * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; |
| * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; |
| * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; |
| * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; |
| * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; |
| * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; |
| * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; |
| * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; |
| * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; |
| * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; |
| * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; |
| * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; |
| * |
| * Not present in struct vmcs12: |
| * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; |
| * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; |
| * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; |
| * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; |
| * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; |
| * evmcs->host_ssp = vmcs12->host_ssp; |
| * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; |
| * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; |
| * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; |
| * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; |
| * evmcs->guest_ssp = vmcs12->guest_ssp; |
| */ |
| |
| evmcs->guest_es_selector = vmcs12->guest_es_selector; |
| evmcs->guest_cs_selector = vmcs12->guest_cs_selector; |
| evmcs->guest_ss_selector = vmcs12->guest_ss_selector; |
| evmcs->guest_ds_selector = vmcs12->guest_ds_selector; |
| evmcs->guest_fs_selector = vmcs12->guest_fs_selector; |
| evmcs->guest_gs_selector = vmcs12->guest_gs_selector; |
| evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; |
| evmcs->guest_tr_selector = vmcs12->guest_tr_selector; |
| |
| evmcs->guest_es_limit = vmcs12->guest_es_limit; |
| evmcs->guest_cs_limit = vmcs12->guest_cs_limit; |
| evmcs->guest_ss_limit = vmcs12->guest_ss_limit; |
| evmcs->guest_ds_limit = vmcs12->guest_ds_limit; |
| evmcs->guest_fs_limit = vmcs12->guest_fs_limit; |
| evmcs->guest_gs_limit = vmcs12->guest_gs_limit; |
| evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; |
| evmcs->guest_tr_limit = vmcs12->guest_tr_limit; |
| evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; |
| evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; |
| |
| evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; |
| evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; |
| evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; |
| evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; |
| evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; |
| evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; |
| evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; |
| evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; |
| |
| evmcs->guest_es_base = vmcs12->guest_es_base; |
| evmcs->guest_cs_base = vmcs12->guest_cs_base; |
| evmcs->guest_ss_base = vmcs12->guest_ss_base; |
| evmcs->guest_ds_base = vmcs12->guest_ds_base; |
| evmcs->guest_fs_base = vmcs12->guest_fs_base; |
| evmcs->guest_gs_base = vmcs12->guest_gs_base; |
| evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; |
| evmcs->guest_tr_base = vmcs12->guest_tr_base; |
| evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; |
| evmcs->guest_idtr_base = vmcs12->guest_idtr_base; |
| |
| evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; |
| evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; |
| |
| evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; |
| evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; |
| evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; |
| evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; |
| |
| evmcs->guest_pending_dbg_exceptions = |
| vmcs12->guest_pending_dbg_exceptions; |
| evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; |
| evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; |
| |
| evmcs->guest_activity_state = vmcs12->guest_activity_state; |
| evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; |
| |
| evmcs->guest_cr0 = vmcs12->guest_cr0; |
| evmcs->guest_cr3 = vmcs12->guest_cr3; |
| evmcs->guest_cr4 = vmcs12->guest_cr4; |
| evmcs->guest_dr7 = vmcs12->guest_dr7; |
| |
| evmcs->guest_physical_address = vmcs12->guest_physical_address; |
| |
| evmcs->vm_instruction_error = vmcs12->vm_instruction_error; |
| evmcs->vm_exit_reason = vmcs12->vm_exit_reason; |
| evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; |
| evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; |
| evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; |
| evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; |
| evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; |
| evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; |
| |
| evmcs->exit_qualification = vmcs12->exit_qualification; |
| |
| evmcs->guest_linear_address = vmcs12->guest_linear_address; |
| evmcs->guest_rsp = vmcs12->guest_rsp; |
| evmcs->guest_rflags = vmcs12->guest_rflags; |
| |
| evmcs->guest_interruptibility_info = |
| vmcs12->guest_interruptibility_info; |
| evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; |
| evmcs->vm_entry_controls = vmcs12->vm_entry_controls; |
| evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; |
| evmcs->vm_entry_exception_error_code = |
| vmcs12->vm_entry_exception_error_code; |
| evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; |
| |
| evmcs->guest_rip = vmcs12->guest_rip; |
| |
| evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; |
| |
| return; |
| } |
| |
| /* |
| * This is an equivalent of the nested hypervisor executing the vmptrld |
| * instruction. |
| */ |
| static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( |
| struct kvm_vcpu *vcpu, bool from_launch) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| bool evmcs_gpa_changed = false; |
| u64 evmcs_gpa; |
| |
| if (likely(!guest_cpuid_has_evmcs(vcpu))) |
| return EVMPTRLD_DISABLED; |
| |
| evmcs_gpa = nested_get_evmptr(vcpu); |
| if (!evmptr_is_valid(evmcs_gpa)) { |
| nested_release_evmcs(vcpu); |
| return EVMPTRLD_DISABLED; |
| } |
| |
| if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { |
| vmx->nested.current_vmptr = INVALID_GPA; |
| |
| nested_release_evmcs(vcpu); |
| |
| if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), |
| &vmx->nested.hv_evmcs_map)) |
| return EVMPTRLD_ERROR; |
| |
| vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; |
| |
| /* |
| * Currently, KVM only supports eVMCS version 1 |
| * (== KVM_EVMCS_VERSION) and thus we expect guest to set this |
| * value to first u32 field of eVMCS which should specify eVMCS |
| * VersionNumber. |
| * |
| * Guest should be aware of supported eVMCS versions by host by |
| * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is |
| * expected to set this CPUID leaf according to the value |
| * returned in vmcs_version from nested_enable_evmcs(). |
| * |
| * However, it turns out that Microsoft Hyper-V fails to comply |
| * to their own invented interface: When Hyper-V use eVMCS, it |
| * just sets first u32 field of eVMCS to revision_id specified |
| * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number |
| * which is one of the supported versions specified in |
| * CPUID.0x4000000A.EAX[0:15]. |
| * |
| * To overcome Hyper-V bug, we accept here either a supported |
| * eVMCS version or VMCS12 revision_id as valid values for first |
| * u32 field of eVMCS. |
| */ |
| if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && |
| (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { |
| nested_release_evmcs(vcpu); |
| return EVMPTRLD_VMFAIL; |
| } |
| |
| vmx->nested.hv_evmcs_vmptr = evmcs_gpa; |
| |
| evmcs_gpa_changed = true; |
| /* |
| * Unlike normal vmcs12, enlightened vmcs12 is not fully |
| * reloaded from guest's memory (read only fields, fields not |
| * present in struct hv_enlightened_vmcs, ...). Make sure there |
| * are no leftovers. |
| */ |
| if (from_launch) { |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| memset(vmcs12, 0, sizeof(*vmcs12)); |
| vmcs12->hdr.revision_id = VMCS12_REVISION; |
| } |
| |
| } |
| |
| /* |
| * Clean fields data can't be used on VMLAUNCH and when we switch |
| * between different L2 guests as KVM keeps a single VMCS12 per L1. |
| */ |
| if (from_launch || evmcs_gpa_changed) { |
| vmx->nested.hv_evmcs->hv_clean_fields &= |
| ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; |
| |
| vmx->nested.force_msr_bitmap_recalc = true; |
| } |
| |
| return EVMPTRLD_SUCCEEDED; |
| } |
| |
| void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) |
| copy_vmcs12_to_enlightened(vmx); |
| else |
| copy_vmcs12_to_shadow(vmx); |
| |
| vmx->nested.need_vmcs12_to_shadow_sync = false; |
| } |
| |
| static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) |
| { |
| struct vcpu_vmx *vmx = |
| container_of(timer, struct vcpu_vmx, nested.preemption_timer); |
| |
| vmx->nested.preemption_timer_expired = true; |
| kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); |
| kvm_vcpu_kick(&vmx->vcpu); |
| |
| return HRTIMER_NORESTART; |
| } |
| |
| static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| |
| u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> |
| VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; |
| |
| if (!vmx->nested.has_preemption_timer_deadline) { |
| vmx->nested.preemption_timer_deadline = |
| vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; |
| vmx->nested.has_preemption_timer_deadline = true; |
| } |
| return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; |
| } |
| |
| static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, |
| u64 preemption_timeout) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* |
| * A timer value of zero is architecturally guaranteed to cause |
| * a VMExit prior to executing any instructions in the guest. |
| */ |
| if (preemption_timeout == 0) { |
| vmx_preemption_timer_fn(&vmx->nested.preemption_timer); |
| return; |
| } |
| |
| if (vcpu->arch.virtual_tsc_khz == 0) |
| return; |
| |
| preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; |
| preemption_timeout *= 1000000; |
| do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); |
| hrtimer_start(&vmx->nested.preemption_timer, |
| ktime_add_ns(ktime_get(), preemption_timeout), |
| HRTIMER_MODE_ABS_PINNED); |
| } |
| |
| static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) |
| { |
| if (vmx->nested.nested_run_pending && |
| (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) |
| return vmcs12->guest_ia32_efer; |
| else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) |
| return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); |
| else |
| return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); |
| } |
| |
| static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) |
| { |
| struct kvm *kvm = vmx->vcpu.kvm; |
| |
| /* |
| * If vmcs02 hasn't been initialized, set the constant vmcs02 state |
| * according to L0's settings (vmcs12 is irrelevant here). Host |
| * fields that come from L0 and are not constant, e.g. HOST_CR3, |
| * will be set as needed prior to VMLAUNCH/VMRESUME. |
| */ |
| if (vmx->nested.vmcs02_initialized) |
| return; |
| vmx->nested.vmcs02_initialized = true; |
| |
| /* |
| * We don't care what the EPTP value is we just need to guarantee |
| * it's valid so we don't get a false positive when doing early |
| * consistency checks. |
| */ |
| if (enable_ept && nested_early_check) |
| vmcs_write64(EPT_POINTER, |
| construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL)); |
| |
| /* All VMFUNCs are currently emulated through L0 vmexits. */ |
| if (cpu_has_vmx_vmfunc()) |
| vmcs_write64(VM_FUNCTION_CONTROL, 0); |
| |
| if (cpu_has_vmx_posted_intr()) |
| vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); |
| |
| if (cpu_has_vmx_msr_bitmap()) |
| vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); |
| |
| /* |
| * PML is emulated for L2, but never enabled in hardware as the MMU |
| * handles A/D emulation. Disabling PML for L2 also avoids having to |
| * deal with filtering out L2 GPAs from the buffer. |
| */ |
| if (enable_pml) { |
| vmcs_write64(PML_ADDRESS, 0); |
| vmcs_write16(GUEST_PML_INDEX, -1); |
| } |
| |
| if (cpu_has_vmx_encls_vmexit()) |
| vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); |
| |
| if (kvm_notify_vmexit_enabled(kvm)) |
| vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); |
| |
| /* |
| * Set the MSR load/store lists to match L0's settings. Only the |
| * addresses are constant (for vmcs02), the counts can change based |
| * on L2's behavior, e.g. switching to/from long mode. |
| */ |
| vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); |
| vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); |
| vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); |
| |
| vmx_set_constant_host_state(vmx); |
| } |
| |
| static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, |
| struct vmcs12 *vmcs12) |
| { |
| prepare_vmcs02_constant_state(vmx); |
| |
| vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); |
| |
| if (enable_vpid) { |
| if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) |
| vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); |
| else |
| vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); |
| } |
| } |
| |
| static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, |
| struct vmcs12 *vmcs12) |
| { |
| u32 exec_control; |
| u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); |
| |
| if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) |
| prepare_vmcs02_early_rare(vmx, vmcs12); |
| |
| /* |
| * PIN CONTROLS |
| */ |
| exec_control = __pin_controls_get(vmcs01); |
| exec_control |= (vmcs12->pin_based_vm_exec_control & |
| ~PIN_BASED_VMX_PREEMPTION_TIMER); |
| |
| /* Posted interrupts setting is only taken from vmcs12. */ |
| vmx->nested.pi_pending = false; |
| if (nested_cpu_has_posted_intr(vmcs12)) |
| vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; |
| else |
| exec_control &= ~PIN_BASED_POSTED_INTR; |
| pin_controls_set(vmx, exec_control); |
| |
| /* |
| * EXEC CONTROLS |
| */ |
| exec_control = __exec_controls_get(vmcs01); /* L0's desires */ |
| exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; |
| exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; |
| exec_control &= ~CPU_BASED_TPR_SHADOW; |
| exec_control |= vmcs12->cpu_based_vm_exec_control; |
| |
| vmx->nested.l1_tpr_threshold = -1; |
| if (exec_control & CPU_BASED_TPR_SHADOW) |
| vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); |
| #ifdef CONFIG_X86_64 |
| else |
| exec_control |= CPU_BASED_CR8_LOAD_EXITING | |
| CPU_BASED_CR8_STORE_EXITING; |
| #endif |
| |
| /* |
| * A vmexit (to either L1 hypervisor or L0 userspace) is always needed |
| * for I/O port accesses. |
| */ |
| exec_control |= CPU_BASED_UNCOND_IO_EXITING; |
| exec_control &= ~CPU_BASED_USE_IO_BITMAPS; |
| |
| /* |
| * This bit will be computed in nested_get_vmcs12_pages, because |
| * we do not have access to L1's MSR bitmap yet. For now, keep |
| * the same bit as before, hoping to avoid multiple VMWRITEs that |
| * only set/clear this bit. |
| */ |
| exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; |
| exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; |
| |
| exec_controls_set(vmx, exec_control); |
| |
| /* |
| * SECONDARY EXEC CONTROLS |
| */ |
| if (cpu_has_secondary_exec_ctrls()) { |
| exec_control = __secondary_exec_controls_get(vmcs01); |
| |
| /* Take the following fields only from vmcs12 */ |
| exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | |
| SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | |
| SECONDARY_EXEC_ENABLE_INVPCID | |
| SECONDARY_EXEC_ENABLE_RDTSCP | |
| SECONDARY_EXEC_XSAVES | |
| SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | |
| SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | |
| SECONDARY_EXEC_APIC_REGISTER_VIRT | |
| SECONDARY_EXEC_ENABLE_VMFUNC | |
| SECONDARY_EXEC_DESC); |
| |
| if (nested_cpu_has(vmcs12, |
| CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) |
| exec_control |= vmcs12->secondary_vm_exec_control; |
| |
| /* PML is emulated and never enabled in hardware for L2. */ |
| exec_control &= ~SECONDARY_EXEC_ENABLE_PML; |
| |
| /* VMCS shadowing for L2 is emulated for now */ |
| exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; |
| |
| /* |
| * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() |
| * will not have to rewrite the controls just for this bit. |
| */ |
| if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && |
| (vmcs12->guest_cr4 & X86_CR4_UMIP)) |
| exec_control |= SECONDARY_EXEC_DESC; |
| |
| if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) |
| vmcs_write16(GUEST_INTR_STATUS, |
| vmcs12->guest_intr_status); |
| |
| if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) |
| exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; |
| |
| if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) |
| vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); |
| |
| secondary_exec_controls_set(vmx, exec_control); |
| } |
| |
| /* |
| * ENTRY CONTROLS |
| * |
| * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE |
| * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate |
| * on the related bits (if supported by the CPU) in the hope that |
| * we can avoid VMWrites during vmx_set_efer(). |
| * |
| * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is |
| * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to |
| * do the same for L2. |
| */ |
| exec_control = __vm_entry_controls_get(vmcs01); |
| exec_control |= (vmcs12->vm_entry_controls & |
| ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); |
| exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); |
| if (cpu_has_load_ia32_efer()) { |
| if (guest_efer & EFER_LMA) |
| exec_control |= VM_ENTRY_IA32E_MODE; |
| if (guest_efer != host_efer) |
| exec_control |= VM_ENTRY_LOAD_IA32_EFER; |
| } |
| vm_entry_controls_set(vmx, exec_control); |
| |
| /* |
| * EXIT CONTROLS |
| * |
| * L2->L1 exit controls are emulated - the hardware exit is to L0 so |
| * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER |
| * bits may be modified by vmx_set_efer() in prepare_vmcs02(). |
| */ |
| exec_control = __vm_exit_controls_get(vmcs01); |
| if (cpu_has_load_ia32_efer() && guest_efer != host_efer) |
| exec_control |= VM_EXIT_LOAD_IA32_EFER; |
| else |
| exec_control &= ~VM_EXIT_LOAD_IA32_EFER; |
| vm_exit_controls_set(vmx, exec_control); |
| |
| /* |
| * Interrupt/Exception Fields |
| */ |
| if (vmx->nested.nested_run_pending) { |
| vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, |
| vmcs12->vm_entry_intr_info_field); |
| vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, |
| vmcs12->vm_entry_exception_error_code); |
| vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, |
| vmcs12->vm_entry_instruction_len); |
| vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, |
| vmcs12->guest_interruptibility_info); |
| vmx->loaded_vmcs->nmi_known_unmasked = |
| !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); |
| } else { |
| vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); |
| } |
| } |
| |
| static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) |
| { |
| struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; |
| |
| if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { |
| vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); |
| vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); |
| vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); |
| vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); |
| vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); |
| vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); |
| vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); |
| vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); |
| vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); |
| vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); |
| vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); |
| vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); |
| vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); |
| vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); |
| vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); |
| vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); |
| vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); |
| vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); |
| vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); |
| vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); |
| vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); |
| vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); |
| vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); |
| vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); |
| vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); |
| vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); |
| vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); |
| vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); |
| vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); |
| vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); |
| vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); |
| vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); |
| vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); |
| vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); |
| vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); |
| vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); |
| |
| vmx->segment_cache.bitmask = 0; |
| } |
| |
| if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { |
| vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); |
| vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, |
| vmcs12->guest_pending_dbg_exceptions); |
| vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); |
| vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); |
| |
| /* |
| * L1 may access the L2's PDPTR, so save them to construct |
| * vmcs12 |
| */ |
| if (enable_ept) { |
| vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); |
| vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); |
| vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); |
| vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); |
| } |
| |
| if (kvm_mpx_supported() && vmx->nested.nested_run_pending && |
| (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) |
| vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); |
| } |
| |
| if (nested_cpu_has_xsaves(vmcs12)) |
| vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); |
| |
| /* |
| * Whether page-faults are trapped is determined by a combination of |
| * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 |
| * doesn't care about page faults then we should set all of these to |
| * L1's desires. However, if L0 does care about (some) page faults, it |
| * is not easy (if at all possible?) to merge L0 and L1's desires, we |
| * simply ask to exit on each and every L2 page fault. This is done by |
| * setting MASK=MATCH=0 and (see below) EB.PF=1. |
| * Note that below we don't need special code to set EB.PF beyond the |
| * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, |
| * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when |
| * !enable_ept, EB.PF is 1, so the "or" will always be 1. |
| */ |
| if (vmx_need_pf_intercept(&vmx->vcpu)) { |
| /* |
| * TODO: if both L0 and L1 need the same MASK and MATCH, |
| * go ahead and use it? |
| */ |
| vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); |
| vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); |
| } else { |
| vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); |
| vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); |
| } |
| |
| if (cpu_has_vmx_apicv()) { |
| vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); |
| vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); |
| vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); |
| vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); |
| } |
| |
| /* |
| * Make sure the msr_autostore list is up to date before we set the |
| * count in the vmcs02. |
| */ |
| prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); |
| |
| vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); |
| vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); |
| vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); |
| |
| set_cr4_guest_host_mask(vmx); |
| } |
| |
| /* |
| * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested |
| * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it |
| * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 |
| * guest in a way that will both be appropriate to L1's requests, and our |
| * needs. In addition to modifying the active vmcs (which is vmcs02), this |
| * function also has additional necessary side-effects, like setting various |
| * vcpu->arch fields. |
| * Returns 0 on success, 1 on failure. Invalid state exit qualification code |
| * is assigned to entry_failure_code on failure. |
| */ |
| static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, |
| bool from_vmentry, |
| enum vm_entry_failure_code *entry_failure_code) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| bool load_guest_pdptrs_vmcs12 = false; |
| |
| if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { |
| prepare_vmcs02_rare(vmx, vmcs12); |
| vmx->nested.dirty_vmcs12 = false; |
| |
| load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || |
| !(vmx->nested.hv_evmcs->hv_clean_fields & |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); |
| } |
| |
| if (vmx->nested.nested_run_pending && |
| (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { |
| kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); |
| vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); |
| } else { |
| kvm_set_dr(vcpu, 7, vcpu->arch.dr7); |
| vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl); |
| } |
| if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || |
| !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) |
| vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); |
| vmx_set_rflags(vcpu, vmcs12->guest_rflags); |
| |
| /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the |
| * bitwise-or of what L1 wants to trap for L2, and what we want to |
| * trap. Note that CR0.TS also needs updating - we do this later. |
| */ |
| vmx_update_exception_bitmap(vcpu); |
| vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; |
| vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); |
| |
| if (vmx->nested.nested_run_pending && |
| (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { |
| vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); |
| vcpu->arch.pat = vmcs12->guest_ia32_pat; |
| } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { |
| vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); |
| } |
| |
| vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( |
| vcpu->arch.l1_tsc_offset, |
| vmx_get_l2_tsc_offset(vcpu), |
| vmx_get_l2_tsc_multiplier(vcpu)); |
| |
| vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( |
| vcpu->arch.l1_tsc_scaling_ratio, |
| vmx_get_l2_tsc_multiplier(vcpu)); |
| |
| vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); |
| if (kvm_caps.has_tsc_control) |
| vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); |
| |
| nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); |
| |
| if (nested_cpu_has_ept(vmcs12)) |
| nested_ept_init_mmu_context(vcpu); |
| |
| /* |
| * Override the CR0/CR4 read shadows after setting the effective guest |
| * CR0/CR4. The common helpers also set the shadows, but they don't |
| * account for vmcs12's cr0/4_guest_host_mask. |
| */ |
| vmx_set_cr0(vcpu, vmcs12->guest_cr0); |
| vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); |
| |
| vmx_set_cr4(vcpu, vmcs12->guest_cr4); |
| vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); |
| |
| vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); |
| /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ |
| vmx_set_efer(vcpu, vcpu->arch.efer); |
| |
| /* |
| * Guest state is invalid and unrestricted guest is disabled, |
| * which means L1 attempted VMEntry to L2 with invalid state. |
| * Fail the VMEntry. |
| * |
| * However when force loading the guest state (SMM exit or |
| * loading nested state after migration, it is possible to |
| * have invalid guest state now, which will be later fixed by |
| * restoring L2 register state |
| */ |
| if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { |
| *entry_failure_code = ENTRY_FAIL_DEFAULT; |
| return -EINVAL; |
| } |
| |
| /* Shadow page tables on either EPT or shadow page tables. */ |
| if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), |
| from_vmentry, entry_failure_code)) |
| return -EINVAL; |
| |
| /* |
| * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 |
| * on nested VM-Exit, which can occur without actually running L2 and |
| * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with |
| * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the |
| * transition to HLT instead of running L2. |
| */ |
| if (enable_ept) |
| vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); |
| |
| /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ |
| if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && |
| is_pae_paging(vcpu)) { |
| vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); |
| vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); |
| vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); |
| vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); |
| } |
| |
| if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && |
| intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && |
| WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, |
| vmcs12->guest_ia32_perf_global_ctrl))) { |
| *entry_failure_code = ENTRY_FAIL_DEFAULT; |
| return -EINVAL; |
| } |
| |
| kvm_rsp_write(vcpu, vmcs12->guest_rsp); |
| kvm_rip_write(vcpu, vmcs12->guest_rip); |
| |
| /* |
| * It was observed that genuine Hyper-V running in L1 doesn't reset |
| * 'hv_clean_fields' by itself, it only sets the corresponding dirty |
| * bits when it changes a field in eVMCS. Mark all fields as clean |
| * here. |
| */ |
| if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) |
| vmx->nested.hv_evmcs->hv_clean_fields |= |
| HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) |
| { |
| if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && |
| nested_cpu_has_virtual_nmis(vmcs12))) |
| return -EINVAL; |
| |
| if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && |
| nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| /* Check for memory type validity */ |
| switch (new_eptp & VMX_EPTP_MT_MASK) { |
| case VMX_EPTP_MT_UC: |
| if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) |
| return false; |
| break; |
| case VMX_EPTP_MT_WB: |
| if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) |
| return false; |
| break; |
| default: |
| return false; |
| } |
| |
| /* Page-walk levels validity. */ |
| switch (new_eptp & VMX_EPTP_PWL_MASK) { |
| case VMX_EPTP_PWL_5: |
| if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) |
| return false; |
| break; |
| case VMX_EPTP_PWL_4: |
| if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) |
| return false; |
| break; |
| default: |
| return false; |
| } |
| |
| /* Reserved bits should not be set */ |
| if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) |
| return false; |
| |
| /* AD, if set, should be supported */ |
| if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { |
| if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| /* |
| * Checks related to VM-Execution Control Fields |
| */ |
| static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, |
| vmx->nested.msrs.pinbased_ctls_low, |
| vmx->nested.msrs.pinbased_ctls_high)) || |
| CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, |
| vmx->nested.msrs.procbased_ctls_low, |
| vmx->nested.msrs.procbased_ctls_high))) |
| return -EINVAL; |
| |
| if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && |
| CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, |
| vmx->nested.msrs.secondary_ctls_low, |
| vmx->nested.msrs.secondary_ctls_high))) |
| return -EINVAL; |
| |
| if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || |
| nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || |
| nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || |
| nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || |
| nested_vmx_check_apic_access_controls(vcpu, vmcs12) || |
| nested_vmx_check_apicv_controls(vcpu, vmcs12) || |
| nested_vmx_check_nmi_controls(vmcs12) || |
| nested_vmx_check_pml_controls(vcpu, vmcs12) || |
| nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || |
| nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || |
| nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || |
| CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) |
| return -EINVAL; |
| |
| if (!nested_cpu_has_preemption_timer(vmcs12) && |
| nested_cpu_has_save_preemption_timer(vmcs12)) |
| return -EINVAL; |
| |
| if (nested_cpu_has_ept(vmcs12) && |
| CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) |
| return -EINVAL; |
| |
| if (nested_cpu_has_vmfunc(vmcs12)) { |
| if (CC(vmcs12->vm_function_control & |
| ~vmx->nested.msrs.vmfunc_controls)) |
| return -EINVAL; |
| |
| if (nested_cpu_has_eptp_switching(vmcs12)) { |
| if (CC(!nested_cpu_has_ept(vmcs12)) || |
| CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) |
| return -EINVAL; |
| } |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Checks related to VM-Exit Control Fields |
| */ |
| static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, |
| vmx->nested.msrs.exit_ctls_low, |
| vmx->nested.msrs.exit_ctls_high)) || |
| CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| /* |
| * Checks related to VM-Entry Control Fields |
| */ |
| static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| |
| if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, |
| vmx->nested.msrs.entry_ctls_low, |
| vmx->nested.msrs.entry_ctls_high))) |
| return -EINVAL; |
| |
| /* |
| * From the Intel SDM, volume 3: |
| * Fields relevant to VM-entry event injection must be set properly. |
| * These fields are the VM-entry interruption-information field, the |
| * VM-entry exception error code, and the VM-entry instruction length. |
| */ |
| if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { |
| u32 intr_info = vmcs12->vm_entry_intr_info_field; |
| u8 vector = intr_info & INTR_INFO_VECTOR_MASK; |
| u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; |
| bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; |
| bool should_have_error_code; |
| bool urg = nested_cpu_has2(vmcs12, |
| SECONDARY_EXEC_UNRESTRICTED_GUEST); |
| bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; |
| |
| /* VM-entry interruption-info field: interruption type */ |
| if (CC(intr_type == INTR_TYPE_RESERVED) || |
| CC(intr_type == INTR_TYPE_OTHER_EVENT && |
| !nested_cpu_supports_monitor_trap_flag(vcpu))) |
| return -EINVAL; |
| |
| /* VM-entry interruption-info field: vector */ |
| if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || |
| CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || |
| CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) |
| return -EINVAL; |
| |
| /* VM-entry interruption-info field: deliver error code */ |
| should_have_error_code = |
| intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && |
| x86_exception_has_error_code(vector); |
| if (CC(has_error_code != should_have_error_code)) |
| return -EINVAL; |
| |
| /* VM-entry exception error code */ |
| if (CC(has_error_code && |
| vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) |
| return -EINVAL; |
| |
| /* VM-entry interruption-info field: reserved bits */ |
| if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) |
| return -EINVAL; |
| |
| /* VM-entry instruction length */ |
| switch (intr_type) { |
| case INTR_TYPE_SOFT_EXCEPTION: |
| case INTR_TYPE_SOFT_INTR: |
| case INTR_TYPE_PRIV_SW_EXCEPTION: |
| if (CC(vmcs12->vm_entry_instruction_len > 15) || |
| CC(vmcs12->vm_entry_instruction_len == 0 && |
| CC(!nested_cpu_has_zero_length_injection(vcpu)))) |
| return -EINVAL; |
| } |
| } |
| |
| if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| if (nested_check_vm_execution_controls(vcpu, vmcs12) || |
| nested_check_vm_exit_controls(vcpu, vmcs12) || |
| nested_check_vm_entry_controls(vcpu, vmcs12)) |
| return -EINVAL; |
| |
| if (guest_cpuid_has_evmcs(vcpu)) |
| return nested_evmcs_check_controls(vmcs12); |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| #ifdef CONFIG_X86_64 |
| if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != |
| !!(vcpu->arch.efer & EFER_LMA))) |
| return -EINVAL; |
| #endif |
| return 0; |
| } |
| |
| static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| bool ia32e; |
| |
| if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || |
| CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || |
| CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) |
| return -EINVAL; |
| |
| if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) |
| return -EINVAL; |
| |
| if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && |
| CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) |
| return -EINVAL; |
| |
| if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && |
| CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), |
| vmcs12->host_ia32_perf_global_ctrl))) |
| return -EINVAL; |
| |
| #ifdef CONFIG_X86_64 |
| ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); |
| #else |
| ia32e = false; |
| #endif |
| |
| if (ia32e) { |
| if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) |
| return -EINVAL; |
| } else { |
| if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || |
| CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || |
| CC((vmcs12->host_rip) >> 32)) |
| return -EINVAL; |
| } |
| |
| if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || |
| CC(vmcs12->host_cs_selector == 0) || |
| CC(vmcs12->host_tr_selector == 0) || |
| CC(vmcs12->host_ss_selector == 0 && !ia32e)) |
| return -EINVAL; |
| |
| if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || |
| CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) |
| return -EINVAL; |
| |
| /* |
| * If the load IA32_EFER VM-exit control is 1, bits reserved in the |
| * IA32_EFER MSR must be 0 in the field for that register. In addition, |
| * the values of the LMA and LME bits in the field must each be that of |
| * the host address-space size VM-exit control. |
| */ |
| if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { |
| if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || |
| CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || |
| CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12) |
| { |
| struct vcpu_vmx *vmx = to_vmx(vcpu); |
| struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; |
| struct vmcs_hdr hdr; |
| |
| if (vmcs12->vmcs_link_pointer == INVALID_GPA) |
| return 0; |
| |
| if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) |
| return -EINVAL; |
| |
| if (ghc->gpa != vmcs12->vmcs_link_pointer && |
| CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, |
| vmcs12->vmcs_link_pointer, VMCS12_SIZE))) |
| return -EINVAL; |
| |
| if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, |
| offsetof(struct vmcs12, hdr), |
| sizeof(hdr)))) |
| return -EINVAL; |
| |
| if (CC(hdr.revision_id != VMCS12_REVISION) || |
| CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| /* |
| * Checks related to Guest Non-register State |
| */ |
| static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) |
| { |
| if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && |
| vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && |
| vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, |
| struct vmcs12 *vmcs12, |
| enum vm_entry_failure_code *entry_failure_code) |
| { |
| bool ia32e; |
| |
| *entry_failure_code = ENTRY_FAIL_DEFAULT; |
| |
| if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || |
| CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) |
| return -EINVAL; |
| |
| if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && |
| CC(!kvm_dr7_valid(vmcs12->guest_dr7))) |
| return -EINVAL; |
| |
| if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && |
| CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) |
| return -EINVAL; |
| |
| if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { |
| *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; |
| return -EINVAL; |
| } |
| |
| if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && |
| CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), |
| vmcs12->guest_ia32_perf_global_ctrl))) |
| return -EINVAL; |
| |
| /* |
| * If the load IA32_EFER VM-entry control is 1, the following checks |
| * are performed on the field for the IA32_EFER MSR: |
| * - Bits reserved in the IA32_EFER MSR must be 0. |
| * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of |
| * the IA-32e mode guest VM-exit control. It must also be identical |
| * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to |
| * CR0.PG) is 1. |
| */ |
| if (to_vmx(vcpu)->nested.nested_run_pending && |
| (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { |
| ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; |
| if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || |
| CC(ia32e != !!(vmcs12 |