From e680366d847a7d93bc2ffef6595fddd8699d38d3 Mon Sep 17 00:00:00 2001 From: maxv Date: Sat, 23 Feb 2019 08:19:16 +0000 Subject: Reorder the functions, and constify setstate. No functional change. --- sys/dev/nvmm/x86/nvmm_x86_svm.c | 519 +++++++++++----------- sys/dev/nvmm/x86/nvmm_x86_vmx.c | 951 ++++++++++++++++++++-------------------- 2 files changed, 731 insertions(+), 739 deletions(-) (limited to 'sys/dev') diff --git a/sys/dev/nvmm/x86/nvmm_x86_svm.c b/sys/dev/nvmm/x86/nvmm_x86_svm.c index dc47f069ea6..756cd4896cd 100644 --- a/sys/dev/nvmm/x86/nvmm_x86_svm.c +++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $ */ +/* $NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $"); #include #include @@ -1468,265 +1468,7 @@ svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) } } -static void -svm_asid_alloc(struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - size_t i, oct, bit; - - mutex_enter(&svm_asidlock); - - for (i = 0; i < svm_maxasid; i++) { - oct = i / 8; - bit = i % 8; - - if (svm_asidmap[oct] & __BIT(bit)) { - continue; - } - - svm_asidmap[oct] |= __BIT(bit); - vmcb->ctrl.guest_asid = i; - mutex_exit(&svm_asidlock); - return; - } - - /* - * No free ASID. Use the last one, which is shared and requires - * special TLB handling. - */ - cpudata->shared_asid = true; - vmcb->ctrl.guest_asid = svm_maxasid - 1; - mutex_exit(&svm_asidlock); -} -static void -svm_asid_free(struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - size_t oct, bit; - - if (cpudata->shared_asid) { - return; - } - - oct = vmcb->ctrl.guest_asid / 8; - bit = vmcb->ctrl.guest_asid % 8; - - mutex_enter(&svm_asidlock); - svm_asidmap[oct] &= ~__BIT(bit); - mutex_exit(&svm_asidlock); -} - -static void -svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - - /* Allow reads/writes of Control Registers. */ - vmcb->ctrl.intercept_cr = 0; - - /* Allow reads/writes of Debug Registers. */ - vmcb->ctrl.intercept_dr = 0; - - /* Allow exceptions 0 to 31. */ - vmcb->ctrl.intercept_vec = 0; - - /* - * Allow: - * - SMI [smm interrupts] - * - VINTR [virtual interrupts] - * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] - * - RIDTR [reads of IDTR] - * - RGDTR [reads of GDTR] - * - RLDTR [reads of LDTR] - * - RTR [reads of TR] - * - WIDTR [writes of IDTR] - * - WGDTR [writes of GDTR] - * - WLDTR [writes of LDTR] - * - WTR [writes of TR] - * - RDTSC [rdtsc instruction] - * - PUSHF [pushf instruction] - * - POPF [popf instruction] - * - IRET [iret instruction] - * - INTN [int $n instructions] - * - INVD [invd instruction] - * - PAUSE [pause instruction] - * - INVLPG [invplg instruction] - * - TASKSW [task switches] - * - * Intercept the rest below. - */ - vmcb->ctrl.intercept_misc1 = - VMCB_CTRL_INTERCEPT_INTR | - VMCB_CTRL_INTERCEPT_NMI | - VMCB_CTRL_INTERCEPT_INIT | - VMCB_CTRL_INTERCEPT_RDPMC | - VMCB_CTRL_INTERCEPT_CPUID | - VMCB_CTRL_INTERCEPT_RSM | - VMCB_CTRL_INTERCEPT_HLT | - VMCB_CTRL_INTERCEPT_INVLPGA | - VMCB_CTRL_INTERCEPT_IOIO_PROT | - VMCB_CTRL_INTERCEPT_MSR_PROT | - VMCB_CTRL_INTERCEPT_FERR_FREEZE | - VMCB_CTRL_INTERCEPT_SHUTDOWN; - - /* - * Allow: - * - ICEBP [icebp instruction] - * - WBINVD [wbinvd instruction] - * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] - * - * Intercept the rest below. - */ - vmcb->ctrl.intercept_misc2 = - VMCB_CTRL_INTERCEPT_VMRUN | - VMCB_CTRL_INTERCEPT_VMMCALL | - VMCB_CTRL_INTERCEPT_VMLOAD | - VMCB_CTRL_INTERCEPT_VMSAVE | - VMCB_CTRL_INTERCEPT_STGI | - VMCB_CTRL_INTERCEPT_CLGI | - VMCB_CTRL_INTERCEPT_SKINIT | - VMCB_CTRL_INTERCEPT_RDTSCP | - VMCB_CTRL_INTERCEPT_MONITOR | - VMCB_CTRL_INTERCEPT_MWAIT | - VMCB_CTRL_INTERCEPT_XSETBV; - - /* Intercept all I/O accesses. */ - memset(cpudata->iobm, 0xFF, IOBM_SIZE); - vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; - - /* - * Allow: - * - EFER [read] - * - STAR [read, write] - * - LSTAR [read, write] - * - CSTAR [read, write] - * - SFMASK [read, write] - * - KERNELGSBASE [read, write] - * - SYSENTER_CS [read, write] - * - SYSENTER_ESP [read, write] - * - SYSENTER_EIP [read, write] - * - FSBASE [read, write] - * - GSBASE [read, write] - * - PAT [read, write] - * - TSC [read] - * - * Intercept the rest. - */ - memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); - vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; - - /* Generate ASID. */ - svm_asid_alloc(vcpu); - - /* Virtual TPR. */ - vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; - - /* Enable Nested Paging. */ - vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; - vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; - - /* Must always be set. */ - vmcb->state.efer = EFER_SVME; - cpudata->gxcr0 = XCR0_X87; - - /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; - cpudata->gfpu.xsh_xcomp_bv = 0; - - /* Set guest TSC to zero, more or less. */ - cpudata->tsc_offset = -cpu_counter(); - - /* These MSRs are static. */ - cpudata->star = rdmsr(MSR_STAR); - cpudata->lstar = rdmsr(MSR_LSTAR); - cpudata->cstar = rdmsr(MSR_CSTAR); - cpudata->sfmask = rdmsr(MSR_SFMASK); -} - -static int -svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata; - int error; - - /* Allocate the SVM cpudata. */ - cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, - UVM_KMF_WIRED|UVM_KMF_ZERO); - vcpu->cpudata = cpudata; - - /* VMCB */ - error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, - VMCB_NPAGES); - if (error) - goto error; - - /* I/O Bitmap */ - error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, - IOBM_NPAGES); - if (error) - goto error; - - /* MSR Bitmap */ - error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, - MSRBM_NPAGES); - if (error) - goto error; - - /* Init the VCPU info. */ - svm_vcpu_init(mach, vcpu); - - return 0; - -error: - if (cpudata->vmcb_pa) { - svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, - VMCB_NPAGES); - } - if (cpudata->iobm_pa) { - svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, - IOBM_NPAGES); - } - if (cpudata->msrbm_pa) { - svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, - MSRBM_NPAGES); - } - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); - return error; -} - -static void -svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - - svm_asid_free(vcpu); - - svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); - svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); - svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); - - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); -} #define SVM_SEG_ATTRIB_TYPE __BITS(4,0) #define SVM_SEG_ATTRIB_DPL __BITS(6,5) @@ -1737,7 +1479,8 @@ svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) #define SVM_SEG_ATTRIB_GRAN __BIT(11) static void -svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) +svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, + struct vmcb_segment *vseg) { vseg->selector = seg->selector; vseg->attrib = @@ -1768,7 +1511,7 @@ svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) } static inline bool -svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, +svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, uint64_t flags) { if (flags & NVMM_X64_STATE_CRS) { @@ -1798,7 +1541,7 @@ svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, static void svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) { - struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; + const struct nvmm_x64_state *state = data; struct svm_cpudata *cpudata = vcpu->cpudata; struct vmcb *vmcb = cpudata->vmcb; struct fxsave *fpustate; @@ -1869,9 +1612,10 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) } if (flags & NVMM_X64_STATE_MSRS) { - /* Bit EFER_SVME is mandatory. */ + /* + * EFER_SVME is mandatory. + */ vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; - vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; @@ -2023,6 +1767,251 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) /* -------------------------------------------------------------------------- */ +static void +svm_asid_alloc(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + size_t i, oct, bit; + + mutex_enter(&svm_asidlock); + + for (i = 0; i < svm_maxasid; i++) { + oct = i / 8; + bit = i % 8; + + if (svm_asidmap[oct] & __BIT(bit)) { + continue; + } + + svm_asidmap[oct] |= __BIT(bit); + vmcb->ctrl.guest_asid = i; + mutex_exit(&svm_asidlock); + return; + } + + /* + * No free ASID. Use the last one, which is shared and requires + * special TLB handling. + */ + cpudata->shared_asid = true; + vmcb->ctrl.guest_asid = svm_maxasid - 1; + mutex_exit(&svm_asidlock); +} + +static void +svm_asid_free(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + size_t oct, bit; + + if (cpudata->shared_asid) { + return; + } + + oct = vmcb->ctrl.guest_asid / 8; + bit = vmcb->ctrl.guest_asid % 8; + + mutex_enter(&svm_asidlock); + svm_asidmap[oct] &= ~__BIT(bit); + mutex_exit(&svm_asidlock); +} + +static void +svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + + /* Allow reads/writes of Control Registers. */ + vmcb->ctrl.intercept_cr = 0; + + /* Allow reads/writes of Debug Registers. */ + vmcb->ctrl.intercept_dr = 0; + + /* Allow exceptions 0 to 31. */ + vmcb->ctrl.intercept_vec = 0; + + /* + * Allow: + * - SMI [smm interrupts] + * - VINTR [virtual interrupts] + * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] + * - RIDTR [reads of IDTR] + * - RGDTR [reads of GDTR] + * - RLDTR [reads of LDTR] + * - RTR [reads of TR] + * - WIDTR [writes of IDTR] + * - WGDTR [writes of GDTR] + * - WLDTR [writes of LDTR] + * - WTR [writes of TR] + * - RDTSC [rdtsc instruction] + * - PUSHF [pushf instruction] + * - POPF [popf instruction] + * - IRET [iret instruction] + * - INTN [int $n instructions] + * - INVD [invd instruction] + * - PAUSE [pause instruction] + * - INVLPG [invplg instruction] + * - TASKSW [task switches] + * + * Intercept the rest below. + */ + vmcb->ctrl.intercept_misc1 = + VMCB_CTRL_INTERCEPT_INTR | + VMCB_CTRL_INTERCEPT_NMI | + VMCB_CTRL_INTERCEPT_INIT | + VMCB_CTRL_INTERCEPT_RDPMC | + VMCB_CTRL_INTERCEPT_CPUID | + VMCB_CTRL_INTERCEPT_RSM | + VMCB_CTRL_INTERCEPT_HLT | + VMCB_CTRL_INTERCEPT_INVLPGA | + VMCB_CTRL_INTERCEPT_IOIO_PROT | + VMCB_CTRL_INTERCEPT_MSR_PROT | + VMCB_CTRL_INTERCEPT_FERR_FREEZE | + VMCB_CTRL_INTERCEPT_SHUTDOWN; + + /* + * Allow: + * - ICEBP [icebp instruction] + * - WBINVD [wbinvd instruction] + * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] + * + * Intercept the rest below. + */ + vmcb->ctrl.intercept_misc2 = + VMCB_CTRL_INTERCEPT_VMRUN | + VMCB_CTRL_INTERCEPT_VMMCALL | + VMCB_CTRL_INTERCEPT_VMLOAD | + VMCB_CTRL_INTERCEPT_VMSAVE | + VMCB_CTRL_INTERCEPT_STGI | + VMCB_CTRL_INTERCEPT_CLGI | + VMCB_CTRL_INTERCEPT_SKINIT | + VMCB_CTRL_INTERCEPT_RDTSCP | + VMCB_CTRL_INTERCEPT_MONITOR | + VMCB_CTRL_INTERCEPT_MWAIT | + VMCB_CTRL_INTERCEPT_XSETBV; + + /* Intercept all I/O accesses. */ + memset(cpudata->iobm, 0xFF, IOBM_SIZE); + vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; + + /* Allow direct access to certain MSRs. */ + memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); + vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; + + /* Generate ASID. */ + svm_asid_alloc(vcpu); + + /* Virtual TPR. */ + vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; + + /* Enable Nested Paging. */ + vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; + vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; + + /* Must always be set. */ + vmcb->state.efer = EFER_SVME; + cpudata->gxcr0 = XCR0_X87; + + /* Init XSAVE header. */ + cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->gfpu.xsh_xcomp_bv = 0; + + /* Set guest TSC to zero, more or less. */ + cpudata->tsc_offset = -cpu_counter(); + + /* These MSRs are static. */ + cpudata->star = rdmsr(MSR_STAR); + cpudata->lstar = rdmsr(MSR_LSTAR); + cpudata->cstar = rdmsr(MSR_CSTAR); + cpudata->sfmask = rdmsr(MSR_SFMASK); +} + +static int +svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata; + int error; + + /* Allocate the SVM cpudata. */ + cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, + roundup(sizeof(*cpudata), PAGE_SIZE), 0, + UVM_KMF_WIRED|UVM_KMF_ZERO); + vcpu->cpudata = cpudata; + + /* VMCB */ + error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, + VMCB_NPAGES); + if (error) + goto error; + + /* I/O Bitmap */ + error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, + IOBM_NPAGES); + if (error) + goto error; + + /* MSR Bitmap */ + error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, + MSRBM_NPAGES); + if (error) + goto error; + + /* Init the VCPU info. */ + svm_vcpu_init(mach, vcpu); + + return 0; + +error: + if (cpudata->vmcb_pa) { + svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, + VMCB_NPAGES); + } + if (cpudata->iobm_pa) { + svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, + IOBM_NPAGES); + } + if (cpudata->msrbm_pa) { + svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, + MSRBM_NPAGES); + } + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); + return error; +} + +static void +svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + + svm_asid_free(vcpu); + + svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); + svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); + svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); + + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); +} + +/* -------------------------------------------------------------------------- */ + static void svm_tlb_flush(struct pmap *pm) { diff --git a/sys/dev/nvmm/x86/nvmm_x86_vmx.c b/sys/dev/nvmm/x86/nvmm_x86_vmx.c index 6174a7de047..fa6aa7fd05a 100644 --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $ */ +/* $NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $"); #include #include @@ -1910,66 +1910,6 @@ vmx_memfree(paddr_t pa, vaddr_t va, size_t npages) /* -------------------------------------------------------------------------- */ -static void -vmx_asid_alloc(struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata = vcpu->cpudata; - size_t i, oct, bit; - - mutex_enter(&vmx_asidlock); - - for (i = 0; i < vmx_maxasid; i++) { - oct = i / 8; - bit = i % 8; - - if (vmx_asidmap[oct] & __BIT(bit)) { - continue; - } - - cpudata->asid = i; - - vmx_asidmap[oct] |= __BIT(bit); - vmx_vmwrite(VMCS_VPID, i); - mutex_exit(&vmx_asidlock); - return; - } - - mutex_exit(&vmx_asidlock); - - panic("%s: impossible", __func__); -} - -static void -vmx_asid_free(struct nvmm_cpu *vcpu) -{ - size_t oct, bit; - uint64_t asid; - - vmx_vmread(VMCS_VPID, &asid); - - oct = asid / 8; - bit = asid % 8; - - mutex_enter(&vmx_asidlock); - vmx_asidmap[oct] &= ~__BIT(bit); - mutex_exit(&vmx_asidlock); -} - -static void -vmx_init_asid(uint32_t maxasid) -{ - size_t allocsz; - - mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE); - - vmx_maxasid = maxasid; - allocsz = roundup(maxasid, 8) / 8; - vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP); - - /* ASID 0 is reserved for the host. */ - vmx_asidmap[0] |= __BIT(0); -} - static void vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) { @@ -1996,518 +1936,566 @@ vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) } } +#define VMX_SEG_ATTRIB_TYPE __BITS(4,0) +#define VMX_SEG_ATTRIB_DPL __BITS(6,5) +#define VMX_SEG_ATTRIB_P __BIT(7) +#define VMX_SEG_ATTRIB_AVL __BIT(12) +#define VMX_SEG_ATTRIB_LONG __BIT(13) +#define VMX_SEG_ATTRIB_DEF32 __BIT(14) +#define VMX_SEG_ATTRIB_GRAN __BIT(15) +#define VMX_SEG_ATTRIB_UNUSABLE __BIT(16) + static void -vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +vmx_vcpu_setstate_seg(const struct nvmm_x64_state_seg *segs, int idx) { - struct vmx_cpudata *cpudata = vcpu->cpudata; - struct vmcs *vmcs = cpudata->vmcs; - struct msr_entry *gmsr = cpudata->gmsr; - extern uint8_t vmx_resume_rip; - uint64_t rev, eptp; + uint64_t attrib; - rev = vmx_get_revision(); + attrib = + __SHIFTIN(segs[idx].attrib.type, VMX_SEG_ATTRIB_TYPE) | + __SHIFTIN(segs[idx].attrib.dpl, VMX_SEG_ATTRIB_DPL) | + __SHIFTIN(segs[idx].attrib.p, VMX_SEG_ATTRIB_P) | + __SHIFTIN(segs[idx].attrib.avl, VMX_SEG_ATTRIB_AVL) | + __SHIFTIN(segs[idx].attrib.lng, VMX_SEG_ATTRIB_LONG) | + __SHIFTIN(segs[idx].attrib.def32, VMX_SEG_ATTRIB_DEF32) | + __SHIFTIN(segs[idx].attrib.gran, VMX_SEG_ATTRIB_GRAN) | + (!segs[idx].attrib.p ? VMX_SEG_ATTRIB_UNUSABLE : 0); - memset(vmcs, 0, VMCS_SIZE); - vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION); - vmcs->abort = 0; + if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) { + vmx_vmwrite(vmx_guest_segs[idx].selector, segs[idx].selector); + vmx_vmwrite(vmx_guest_segs[idx].attrib, attrib); + } + vmx_vmwrite(vmx_guest_segs[idx].limit, segs[idx].limit); + vmx_vmwrite(vmx_guest_segs[idx].base, segs[idx].base); +} - vmx_vmcs_enter(vcpu); +static void +vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx) +{ + uint64_t attrib = 0; - /* No link pointer. */ - vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF); + if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) { + vmx_vmread(vmx_guest_segs[idx].selector, &segs[idx].selector); + vmx_vmread(vmx_guest_segs[idx].attrib, &attrib); + } + vmx_vmread(vmx_guest_segs[idx].limit, &segs[idx].limit); + vmx_vmread(vmx_guest_segs[idx].base, &segs[idx].base); - /* Install the CTLSs. */ - vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls); - vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls); - vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2); - vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls); - vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls); + segs[idx].attrib.type = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_TYPE); + segs[idx].attrib.dpl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DPL); + segs[idx].attrib.p = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_P); + segs[idx].attrib.avl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_AVL); + segs[idx].attrib.lng = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_LONG); + segs[idx].attrib.def32 = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DEF32); + segs[idx].attrib.gran = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_GRAN); + if (attrib & VMX_SEG_ATTRIB_UNUSABLE) { + segs[idx].attrib.p = 0; + } +} - /* Allow direct access to certain MSRs. */ - memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES, - true, false); - vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa); +static inline bool +vmx_state_tlb_flush(const struct nvmm_x64_state *state, uint64_t flags) +{ + uint64_t cr0, cr3, cr4, efer; - /* - * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This - * includes the L1D_FLUSH MSR, to mitigate L1TF. - */ - gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR; - gmsr[VMX_MSRLIST_STAR].val = 0; - gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR; - gmsr[VMX_MSRLIST_LSTAR].val = 0; - gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR; - gmsr[VMX_MSRLIST_CSTAR].val = 0; - gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK; - gmsr[VMX_MSRLIST_SFMASK].val = 0; - gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE; - gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0; - gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD; - gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH; - vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa); - vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa); - vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr); - vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR); + if (flags & NVMM_X64_STATE_CRS) { + vmx_vmread(VMCS_GUEST_CR0, &cr0); + if ((cr0 ^ state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { + return true; + } + vmx_vmread(VMCS_GUEST_CR3, &cr3); + if (cr3 != state->crs[NVMM_X64_CR_CR3]) { + return true; + } + vmx_vmread(VMCS_GUEST_CR4, &cr4); + if ((cr4 ^ state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { + return true; + } + } - /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */ - vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET); - vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET); + if (flags & NVMM_X64_STATE_MSRS) { + vmx_vmread(VMCS_GUEST_IA32_EFER, &efer); + if ((efer ^ + state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { + return true; + } + } - /* Force CR4_VMXE to zero. */ - vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE); + return false; +} - /* Set the Host state for resuming. */ - vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip); - vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0); - vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0); - vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt); - vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT)); - vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER)); - vmx_vmwrite(VMCS_HOST_CR0, rcr0()); +static void +vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) +{ + const struct nvmm_x64_state *state = data; + struct vmx_cpudata *cpudata = vcpu->cpudata; + struct fxsave *fpustate; + uint64_t ctls1, intstate; - /* Generate ASID. */ - vmx_asid_alloc(vcpu); + vmx_vmcs_enter(vcpu); - /* Enable Extended Paging, 4-Level. */ - eptp = - __SHIFTIN(vmx_eptp_type, EPTP_TYPE) | - __SHIFTIN(4-1, EPTP_WALKLEN) | - EPTP_FLAGS_AD | - mach->vm->vm_map.pmap->pm_pdirpa[0]; - vmx_vmwrite(VMCS_EPTP, eptp); + if (vmx_state_tlb_flush(state, flags)) { + cpudata->gtlb_want_flush = true; + } - /* Init IA32_MISC_ENABLE. */ - cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE); - cpudata->gmsr_misc_enable &= - ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN); - cpudata->gmsr_misc_enable |= - (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); + if (flags & NVMM_X64_STATE_SEGS) { + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_CS); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_DS); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_ES); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_FS); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GS); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_SS); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GDT); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_IDT); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_LDT); + vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_TR); + } - /* Must always be set. */ - vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE); - vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE); - cpudata->gxcr0 = XCR0_X87; + CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); + if (flags & NVMM_X64_STATE_GPRS) { + memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); - /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; - cpudata->gfpu.xsh_xcomp_bv = 0; + vmx_vmwrite(VMCS_GUEST_RIP, state->gprs[NVMM_X64_GPR_RIP]); + vmx_vmwrite(VMCS_GUEST_RSP, state->gprs[NVMM_X64_GPR_RSP]); + vmx_vmwrite(VMCS_GUEST_RFLAGS, state->gprs[NVMM_X64_GPR_RFLAGS]); + } - /* Set guest TSC to zero, more or less. */ - cpudata->tsc_offset = -cpu_counter(); + if (flags & NVMM_X64_STATE_CRS) { + /* + * CR0_NE and CR4_VMXE are mandatory. + */ + vmx_vmwrite(VMCS_GUEST_CR0, + state->crs[NVMM_X64_CR_CR0] | CR0_NE); + cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2]; + vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE? + vmx_vmwrite(VMCS_GUEST_CR4, + state->crs[NVMM_X64_CR_CR4] | CR4_VMXE); + cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; - /* These MSRs are static. */ - cpudata->star = rdmsr(MSR_STAR); - cpudata->cstar = rdmsr(MSR_CSTAR); - cpudata->sfmask = rdmsr(MSR_SFMASK); + if (vmx_xcr0_mask != 0) { + /* Clear illegal XCR0 bits, set mandatory X87 bit. */ + cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; + cpudata->gxcr0 &= vmx_xcr0_mask; + cpudata->gxcr0 |= XCR0_X87; + } + } - vmx_vmcs_leave(vcpu); -} + CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); + if (flags & NVMM_X64_STATE_DRS) { + memcpy(cpudata->drs, state->drs, sizeof(state->drs)); -static int -vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata; - int error; + cpudata->drs[NVMM_X64_DR_DR6] &= 0xFFFFFFFF; + vmx_vmwrite(VMCS_GUEST_DR7, cpudata->drs[NVMM_X64_DR_DR7]); + } - /* Allocate the VMX cpudata. */ - cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, - UVM_KMF_WIRED|UVM_KMF_ZERO); - vcpu->cpudata = cpudata; + if (flags & NVMM_X64_STATE_MSRS) { + cpudata->gmsr[VMX_MSRLIST_STAR].val = + state->msrs[NVMM_X64_MSR_STAR]; + cpudata->gmsr[VMX_MSRLIST_LSTAR].val = + state->msrs[NVMM_X64_MSR_LSTAR]; + cpudata->gmsr[VMX_MSRLIST_CSTAR].val = + state->msrs[NVMM_X64_MSR_CSTAR]; + cpudata->gmsr[VMX_MSRLIST_SFMASK].val = + state->msrs[NVMM_X64_MSR_SFMASK]; + cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val = + state->msrs[NVMM_X64_MSR_KERNELGSBASE]; - /* VMCS */ - error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs, - VMCS_NPAGES); - if (error) - goto error; + vmx_vmwrite(VMCS_GUEST_IA32_EFER, + state->msrs[NVMM_X64_MSR_EFER]); + vmx_vmwrite(VMCS_GUEST_IA32_PAT, + state->msrs[NVMM_X64_MSR_PAT]); + vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_CS, + state->msrs[NVMM_X64_MSR_SYSENTER_CS]); + vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_ESP, + state->msrs[NVMM_X64_MSR_SYSENTER_ESP]); + vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_EIP, + state->msrs[NVMM_X64_MSR_SYSENTER_EIP]); - /* MSR Bitmap */ - error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, - MSRBM_NPAGES); - if (error) - goto error; + /* ENTRY_CTLS_LONG_MODE must match EFER_LMA. */ + vmx_vmread(VMCS_ENTRY_CTLS, &ctls1); + if (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) { + ctls1 |= ENTRY_CTLS_LONG_MODE; + } else { + ctls1 &= ~ENTRY_CTLS_LONG_MODE; + } + vmx_vmwrite(VMCS_ENTRY_CTLS, ctls1); + } - /* Guest MSR List */ - error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1); - if (error) - goto error; + if (flags & NVMM_X64_STATE_MISC) { + vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate); + intstate &= ~(INT_STATE_STI|INT_STATE_MOVSS); + if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { + intstate |= INT_STATE_MOVSS; + } + vmx_vmwrite(VMCS_GUEST_INTERRUPTIBILITY, intstate); - kcpuset_create(&cpudata->htlb_want_flush, true); + if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { + vmx_event_waitexit_enable(vcpu, false); + } else { + vmx_event_waitexit_disable(vcpu, false); + } - /* Init the VCPU info. */ - vmx_vcpu_init(mach, vcpu); + if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { + vmx_event_waitexit_enable(vcpu, true); + } else { + vmx_event_waitexit_disable(vcpu, true); + } + } - return 0; + CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); + if (flags & NVMM_X64_STATE_FPU) { + memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, + sizeof(state->fpu)); -error: - if (cpudata->vmcs_pa) { - vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, - VMCS_NPAGES); - } - if (cpudata->msrbm_pa) { - vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, - MSRBM_NPAGES); - } - if (cpudata->gmsr_pa) { - vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); + fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; + fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; + fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; + + if (vmx_xcr0_mask != 0) { + /* Reset XSTATE_BV, to force a reload. */ + cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + } } - kmem_free(cpudata, sizeof(*cpudata)); - return error; + vmx_vmcs_leave(vcpu); } static void -vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) { + struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; struct vmx_cpudata *cpudata = vcpu->cpudata; + uint64_t intstate; vmx_vmcs_enter(vcpu); - vmx_asid_free(vcpu); - vmx_vmcs_leave(vcpu); - kcpuset_destroy(cpudata->htlb_want_flush); + if (flags & NVMM_X64_STATE_SEGS) { + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_CS); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_DS); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_ES); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_FS); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GS); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_SS); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GDT); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_IDT); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_LDT); + vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_TR); + } - vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES); - vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); - vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); -} + CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); + if (flags & NVMM_X64_STATE_GPRS) { + memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); -#define VMX_SEG_ATTRIB_TYPE __BITS(4,0) -#define VMX_SEG_ATTRIB_DPL __BITS(6,5) -#define VMX_SEG_ATTRIB_P __BIT(7) -#define VMX_SEG_ATTRIB_AVL __BIT(12) -#define VMX_SEG_ATTRIB_LONG __BIT(13) -#define VMX_SEG_ATTRIB_DEF32 __BIT(14) -#define VMX_SEG_ATTRIB_GRAN __BIT(15) -#define VMX_SEG_ATTRIB_UNUSABLE __BIT(16) + vmx_vmread(VMCS_GUEST_RIP, &state->gprs[NVMM_X64_GPR_RIP]); + vmx_vmread(VMCS_GUEST_RSP, &state->gprs[NVMM_X64_GPR_RSP]); + vmx_vmread(VMCS_GUEST_RFLAGS, &state->gprs[NVMM_X64_GPR_RFLAGS]); + } -static void -vmx_vcpu_setstate_seg(struct nvmm_x64_state_seg *segs, int idx) -{ - uint64_t attrib; + if (flags & NVMM_X64_STATE_CRS) { + vmx_vmread(VMCS_GUEST_CR0, &state->crs[NVMM_X64_CR_CR0]); + state->crs[NVMM_X64_CR_CR2] = cpudata->gcr2; + vmx_vmread(VMCS_GUEST_CR3, &state->crs[NVMM_X64_CR_CR3]); + vmx_vmread(VMCS_GUEST_CR4, &state->crs[NVMM_X64_CR_CR4]); + state->crs[NVMM_X64_CR_CR8] = cpudata->gcr8; + state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; - attrib = - __SHIFTIN(segs[idx].attrib.type, VMX_SEG_ATTRIB_TYPE) | - __SHIFTIN(segs[idx].attrib.dpl, VMX_SEG_ATTRIB_DPL) | - __SHIFTIN(segs[idx].attrib.p, VMX_SEG_ATTRIB_P) | - __SHIFTIN(segs[idx].attrib.avl, VMX_SEG_ATTRIB_AVL) | - __SHIFTIN(segs[idx].attrib.lng, VMX_SEG_ATTRIB_LONG) | - __SHIFTIN(segs[idx].attrib.def32, VMX_SEG_ATTRIB_DEF32) | - __SHIFTIN(segs[idx].attrib.gran, VMX_SEG_ATTRIB_GRAN) | - (!segs[idx].attrib.p ? VMX_SEG_ATTRIB_UNUSABLE : 0); + /* Hide VMXE. */ + state->crs[NVMM_X64_CR_CR4] &= ~CR4_VMXE; + } - if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) { - vmx_vmwrite(vmx_guest_segs[idx].selector, segs[idx].selector); - vmx_vmwrite(vmx_guest_segs[idx].attrib, attrib); + CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); + if (flags & NVMM_X64_STATE_DRS) { + memcpy(state->drs, cpudata->drs, sizeof(state->drs)); + + vmx_vmread(VMCS_GUEST_DR7, &state->drs[NVMM_X64_DR_DR7]); } - vmx_vmwrite(vmx_guest_segs[idx].limit, segs[idx].limit); - vmx_vmwrite(vmx_guest_segs[idx].base, segs[idx].base); -} -static void -vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx) -{ - uint64_t attrib = 0; + if (flags & NVMM_X64_STATE_MSRS) { + state->msrs[NVMM_X64_MSR_STAR] = + cpudata->gmsr[VMX_MSRLIST_STAR].val; + state->msrs[NVMM_X64_MSR_LSTAR] = + cpudata->gmsr[VMX_MSRLIST_LSTAR].val; + state->msrs[NVMM_X64_MSR_CSTAR] = + cpudata->gmsr[VMX_MSRLIST_CSTAR].val; + state->msrs[NVMM_X64_MSR_SFMASK] = + cpudata->gmsr[VMX_MSRLIST_SFMASK].val; + state->msrs[NVMM_X64_MSR_KERNELGSBASE] = + cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val; - if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) { - vmx_vmread(vmx_guest_segs[idx].selector, &segs[idx].selector); - vmx_vmread(vmx_guest_segs[idx].attrib, &attrib); + vmx_vmread(VMCS_GUEST_IA32_EFER, + &state->msrs[NVMM_X64_MSR_EFER]); + vmx_vmread(VMCS_GUEST_IA32_PAT, + &state->msrs[NVMM_X64_MSR_PAT]); + vmx_vmread(VMCS_GUEST_IA32_SYSENTER_CS, + &state->msrs[NVMM_X64_MSR_SYSENTER_CS]); + vmx_vmread(VMCS_GUEST_IA32_SYSENTER_ESP, + &state->msrs[NVMM_X64_MSR_SYSENTER_ESP]); + vmx_vmread(VMCS_GUEST_IA32_SYSENTER_EIP, + &state->msrs[NVMM_X64_MSR_SYSENTER_EIP]); } - vmx_vmread(vmx_guest_segs[idx].limit, &segs[idx].limit); - vmx_vmread(vmx_guest_segs[idx].base, &segs[idx].base); - segs[idx].attrib.type = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_TYPE); - segs[idx].attrib.dpl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DPL); - segs[idx].attrib.p = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_P); - segs[idx].attrib.avl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_AVL); - segs[idx].attrib.lng = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_LONG); - segs[idx].attrib.def32 = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DEF32); - segs[idx].attrib.gran = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_GRAN); - if (attrib & VMX_SEG_ATTRIB_UNUSABLE) { - segs[idx].attrib.p = 0; + if (flags & NVMM_X64_STATE_MISC) { + vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate); + state->misc[NVMM_X64_MISC_INT_SHADOW] = + (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0; + + state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = + cpudata->int_window_exit; + state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = + cpudata->nmi_window_exit; } + + CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); + if (flags & NVMM_X64_STATE_FPU) { + memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, + sizeof(state->fpu)); + } + + vmx_vmcs_leave(vcpu); } -static inline bool -vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags) +/* -------------------------------------------------------------------------- */ + +static void +vmx_asid_alloc(struct nvmm_cpu *vcpu) { - uint64_t cr0, cr3, cr4, efer; + struct vmx_cpudata *cpudata = vcpu->cpudata; + size_t i, oct, bit; - if (flags & NVMM_X64_STATE_CRS) { - vmx_vmread(VMCS_GUEST_CR0, &cr0); - if ((cr0 ^ state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { - return true; - } - vmx_vmread(VMCS_GUEST_CR3, &cr3); - if (cr3 != state->crs[NVMM_X64_CR_CR3]) { - return true; - } - vmx_vmread(VMCS_GUEST_CR4, &cr4); - if ((cr4 ^ state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { - return true; - } - } + mutex_enter(&vmx_asidlock); - if (flags & NVMM_X64_STATE_MSRS) { - vmx_vmread(VMCS_GUEST_IA32_EFER, &efer); - if ((efer ^ - state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { - return true; + for (i = 0; i < vmx_maxasid; i++) { + oct = i / 8; + bit = i % 8; + + if (vmx_asidmap[oct] & __BIT(bit)) { + continue; } + + cpudata->asid = i; + + vmx_asidmap[oct] |= __BIT(bit); + vmx_vmwrite(VMCS_VPID, i); + mutex_exit(&vmx_asidlock); + return; } - return false; + mutex_exit(&vmx_asidlock); + + panic("%s: impossible", __func__); } static void -vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) +vmx_asid_free(struct nvmm_cpu *vcpu) { - struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; - struct vmx_cpudata *cpudata = vcpu->cpudata; - struct fxsave *fpustate; - uint64_t ctls1, intstate; + size_t oct, bit; + uint64_t asid; - vmx_vmcs_enter(vcpu); + vmx_vmread(VMCS_VPID, &asid); - if (vmx_state_tlb_flush(state, flags)) { - cpudata->gtlb_want_flush = true; - } + oct = asid / 8; + bit = asid % 8; - if (flags & NVMM_X64_STATE_SEGS) { - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_CS); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_DS); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_ES); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_FS); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GS); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_SS); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GDT); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_IDT); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_LDT); - vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_TR); - } + mutex_enter(&vmx_asidlock); + vmx_asidmap[oct] &= ~__BIT(bit); + mutex_exit(&vmx_asidlock); +} - CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); - if (flags & NVMM_X64_STATE_GPRS) { - memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); +static void +vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata = vcpu->cpudata; + struct vmcs *vmcs = cpudata->vmcs; + struct msr_entry *gmsr = cpudata->gmsr; + extern uint8_t vmx_resume_rip; + uint64_t rev, eptp; - vmx_vmwrite(VMCS_GUEST_RIP, state->gprs[NVMM_X64_GPR_RIP]); - vmx_vmwrite(VMCS_GUEST_RSP, state->gprs[NVMM_X64_GPR_RSP]); - vmx_vmwrite(VMCS_GUEST_RFLAGS, state->gprs[NVMM_X64_GPR_RFLAGS]); - } + rev = vmx_get_revision(); - if (flags & NVMM_X64_STATE_CRS) { - /* These bits are mandatory. */ - state->crs[NVMM_X64_CR_CR4] |= CR4_VMXE; - state->crs[NVMM_X64_CR_CR0] |= CR0_NE; + memset(vmcs, 0, VMCS_SIZE); + vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION); + vmcs->abort = 0; - vmx_vmwrite(VMCS_GUEST_CR0, state->crs[NVMM_X64_CR_CR0]); - cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2]; - vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE? - vmx_vmwrite(VMCS_GUEST_CR4, state->crs[NVMM_X64_CR_CR4]); - cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; + vmx_vmcs_enter(vcpu); - if (vmx_xcr0_mask != 0) { - /* Clear illegal XCR0 bits, set mandatory X87 bit. */ - cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= vmx_xcr0_mask; - cpudata->gxcr0 |= XCR0_X87; - } - } + /* No link pointer. */ + vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF); - CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); - if (flags & NVMM_X64_STATE_DRS) { - memcpy(cpudata->drs, state->drs, sizeof(state->drs)); + /* Install the CTLSs. */ + vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls); + vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls); + vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2); + vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls); + vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls); - cpudata->drs[NVMM_X64_DR_DR6] &= 0xFFFFFFFF; - vmx_vmwrite(VMCS_GUEST_DR7, cpudata->drs[NVMM_X64_DR_DR7]); - } + /* Allow direct access to certain MSRs. */ + memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES, + true, false); + vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa); + + /* + * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This + * includes the L1D_FLUSH MSR, to mitigate L1TF. + */ + gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR; + gmsr[VMX_MSRLIST_STAR].val = 0; + gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR; + gmsr[VMX_MSRLIST_LSTAR].val = 0; + gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR; + gmsr[VMX_MSRLIST_CSTAR].val = 0; + gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK; + gmsr[VMX_MSRLIST_SFMASK].val = 0; + gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE; + gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0; + gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD; + gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH; + vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa); + vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa); + vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr); + vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR); - if (flags & NVMM_X64_STATE_MSRS) { - cpudata->gmsr[VMX_MSRLIST_STAR].val = - state->msrs[NVMM_X64_MSR_STAR]; - cpudata->gmsr[VMX_MSRLIST_LSTAR].val = - state->msrs[NVMM_X64_MSR_LSTAR]; - cpudata->gmsr[VMX_MSRLIST_CSTAR].val = - state->msrs[NVMM_X64_MSR_CSTAR]; - cpudata->gmsr[VMX_MSRLIST_SFMASK].val = - state->msrs[NVMM_X64_MSR_SFMASK]; - cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val = - state->msrs[NVMM_X64_MSR_KERNELGSBASE]; + /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */ + vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET); + vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET); - vmx_vmwrite(VMCS_GUEST_IA32_EFER, - state->msrs[NVMM_X64_MSR_EFER]); - vmx_vmwrite(VMCS_GUEST_IA32_PAT, - state->msrs[NVMM_X64_MSR_PAT]); - vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_CS, - state->msrs[NVMM_X64_MSR_SYSENTER_CS]); - vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_ESP, - state->msrs[NVMM_X64_MSR_SYSENTER_ESP]); - vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_EIP, - state->msrs[NVMM_X64_MSR_SYSENTER_EIP]); + /* Force CR4_VMXE to zero. */ + vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE); - /* ENTRY_CTLS_LONG_MODE must match EFER_LMA. */ - vmx_vmread(VMCS_ENTRY_CTLS, &ctls1); - if (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) { - ctls1 |= ENTRY_CTLS_LONG_MODE; - } else { - ctls1 &= ~ENTRY_CTLS_LONG_MODE; - } - vmx_vmwrite(VMCS_ENTRY_CTLS, ctls1); - } + /* Set the Host state for resuming. */ + vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip); + vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0); + vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0); + vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt); + vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT)); + vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER)); + vmx_vmwrite(VMCS_HOST_CR0, rcr0()); - if (flags & NVMM_X64_STATE_MISC) { - vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate); - intstate &= ~(INT_STATE_STI|INT_STATE_MOVSS); - if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { - intstate |= INT_STATE_MOVSS; - } - vmx_vmwrite(VMCS_GUEST_INTERRUPTIBILITY, intstate); + /* Generate ASID. */ + vmx_asid_alloc(vcpu); - if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { - vmx_event_waitexit_enable(vcpu, false); - } else { - vmx_event_waitexit_disable(vcpu, false); - } + /* Enable Extended Paging, 4-Level. */ + eptp = + __SHIFTIN(vmx_eptp_type, EPTP_TYPE) | + __SHIFTIN(4-1, EPTP_WALKLEN) | + EPTP_FLAGS_AD | + mach->vm->vm_map.pmap->pm_pdirpa[0]; + vmx_vmwrite(VMCS_EPTP, eptp); - if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { - vmx_event_waitexit_enable(vcpu, true); - } else { - vmx_event_waitexit_disable(vcpu, true); - } - } + /* Init IA32_MISC_ENABLE. */ + cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE); + cpudata->gmsr_misc_enable &= + ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN); + cpudata->gmsr_misc_enable |= + (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); - CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); - if (flags & NVMM_X64_STATE_FPU) { - memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, - sizeof(state->fpu)); + /* Must always be set. */ + vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE); + vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE); + cpudata->gxcr0 = XCR0_X87; - fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; - fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; - fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; + /* Init XSAVE header. */ + cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->gfpu.xsh_xcomp_bv = 0; - if (vmx_xcr0_mask != 0) { - /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; - } - } + /* Set guest TSC to zero, more or less. */ + cpudata->tsc_offset = -cpu_counter(); + + /* These MSRs are static. */ + cpudata->star = rdmsr(MSR_STAR); + cpudata->cstar = rdmsr(MSR_CSTAR); + cpudata->sfmask = rdmsr(MSR_SFMASK); vmx_vmcs_leave(vcpu); } -static void -vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) +static int +vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { - struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; - struct vmx_cpudata *cpudata = vcpu->cpudata; - uint64_t intstate; + struct vmx_cpudata *cpudata; + int error; - vmx_vmcs_enter(vcpu); + /* Allocate the VMX cpudata. */ + cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, + roundup(sizeof(*cpudata), PAGE_SIZE), 0, + UVM_KMF_WIRED|UVM_KMF_ZERO); + vcpu->cpudata = cpudata; - if (flags & NVMM_X64_STATE_SEGS) { - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_CS); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_DS); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_ES); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_FS); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GS); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_SS); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GDT); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_IDT); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_LDT); - vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_TR); - } + /* VMCS */ + error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs, + VMCS_NPAGES); + if (error) + goto error; - CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); - if (flags & NVMM_X64_STATE_GPRS) { - memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); + /* MSR Bitmap */ + error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, + MSRBM_NPAGES); + if (error) + goto error; - vmx_vmread(VMCS_GUEST_RIP, &state->gprs[NVMM_X64_GPR_RIP]); - vmx_vmread(VMCS_GUEST_RSP, &state->gprs[NVMM_X64_GPR_RSP]); - vmx_vmread(VMCS_GUEST_RFLAGS, &state->gprs[NVMM_X64_GPR_RFLAGS]); - } + /* Guest MSR List */ + error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1); + if (error) + goto error; - if (flags & NVMM_X64_STATE_CRS) { - vmx_vmread(VMCS_GUEST_CR0, &state->crs[NVMM_X64_CR_CR0]); - state->crs[NVMM_X64_CR_CR2] = cpudata->gcr2; - vmx_vmread(VMCS_GUEST_CR3, &state->crs[NVMM_X64_CR_CR3]); - vmx_vmread(VMCS_GUEST_CR4, &state->crs[NVMM_X64_CR_CR4]); - state->crs[NVMM_X64_CR_CR8] = cpudata->gcr8; - state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; + kcpuset_create(&cpudata->htlb_want_flush, true); - /* Hide VMXE. */ - state->crs[NVMM_X64_CR_CR4] &= ~CR4_VMXE; - } + /* Init the VCPU info. */ + vmx_vcpu_init(mach, vcpu); - CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); - if (flags & NVMM_X64_STATE_DRS) { - memcpy(state->drs, cpudata->drs, sizeof(state->drs)); + return 0; - vmx_vmread(VMCS_GUEST_DR7, &state->drs[NVMM_X64_DR_DR7]); +error: + if (cpudata->vmcs_pa) { + vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, + VMCS_NPAGES); } - - if (flags & NVMM_X64_STATE_MSRS) { - state->msrs[NVMM_X64_MSR_STAR] = - cpudata->gmsr[VMX_MSRLIST_STAR].val; - state->msrs[NVMM_X64_MSR_LSTAR] = - cpudata->gmsr[VMX_MSRLIST_LSTAR].val; - state->msrs[NVMM_X64_MSR_CSTAR] = - cpudata->gmsr[VMX_MSRLIST_CSTAR].val; - state->msrs[NVMM_X64_MSR_SFMASK] = - cpudata->gmsr[VMX_MSRLIST_SFMASK].val; - state->msrs[NVMM_X64_MSR_KERNELGSBASE] = - cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val; - - vmx_vmread(VMCS_GUEST_IA32_EFER, - &state->msrs[NVMM_X64_MSR_EFER]); - vmx_vmread(VMCS_GUEST_IA32_PAT, - &state->msrs[NVMM_X64_MSR_PAT]); - vmx_vmread(VMCS_GUEST_IA32_SYSENTER_CS, - &state->msrs[NVMM_X64_MSR_SYSENTER_CS]); - vmx_vmread(VMCS_GUEST_IA32_SYSENTER_ESP, - &state->msrs[NVMM_X64_MSR_SYSENTER_ESP]); - vmx_vmread(VMCS_GUEST_IA32_SYSENTER_EIP, - &state->msrs[NVMM_X64_MSR_SYSENTER_EIP]); + if (cpudata->msrbm_pa) { + vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, + MSRBM_NPAGES); } - - if (flags & NVMM_X64_STATE_MISC) { - vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate); - state->misc[NVMM_X64_MISC_INT_SHADOW] = - (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0; - - state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = - cpudata->int_window_exit; - state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = - cpudata->nmi_window_exit; + if (cpudata->gmsr_pa) { + vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); } - CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); - if (flags & NVMM_X64_STATE_FPU) { - memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, - sizeof(state->fpu)); - } + kmem_free(cpudata, sizeof(*cpudata)); + return error; +} + +static void +vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata = vcpu->cpudata; + vmx_vmcs_enter(vcpu); + vmx_asid_free(vcpu); vmx_vmcs_leave(vcpu); + + kcpuset_destroy(cpudata->htlb_want_flush); + + vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES); + vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); + vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); } /* -------------------------------------------------------------------------- */ @@ -2747,6 +2735,21 @@ vmx_ident(void) return true; } +static void +vmx_init_asid(uint32_t maxasid) +{ + size_t allocsz; + + mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE); + + vmx_maxasid = maxasid; + allocsz = roundup(maxasid, 8) / 8; + vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP); + + /* ASID 0 is reserved for the host. */ + vmx_asidmap[0] |= __BIT(0); +} + static void vmx_change_cpu(void *arg1, void *arg2) { -- cgit