diff options
| author | maxv <maxv@NetBSD.org> | 2019-02-23 08:19:16 +0000 |
|---|---|---|
| committer | maxv <maxv@NetBSD.org> | 2019-02-23 08:19:16 +0000 |
| commit | e680366d847a7d93bc2ffef6595fddd8699d38d3 (patch) | |
| tree | 6c25b75c038c4f32a0ed4279df8c187b55bd7971 /sys/dev/nvmm | |
| parent | 8df99d236f6b7922d7e95a23cacec253a3cd0ec3 (diff) | |
Reorder the functions, and constify setstate. No functional change.
Diffstat (limited to 'sys/dev/nvmm')
| -rw-r--r-- | sys/dev/nvmm/x86/nvmm_x86_svm.c | 519 | ||||
| -rw-r--r-- | sys/dev/nvmm/x86/nvmm_x86_vmx.c | 541 |
2 files changed, 526 insertions, 534 deletions
diff --git a/sys/dev/nvmm/x86/nvmm_x86_svm.c b/sys/dev/nvmm/x86/nvmm_x86_svm.c index dc47f069ea6..756cd4896cd 100644 --- a/sys/dev/nvmm/x86/nvmm_x86_svm.c +++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $ */ +/* $NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -1468,265 +1468,7 @@ svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) } } -static void -svm_asid_alloc(struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - size_t i, oct, bit; - - mutex_enter(&svm_asidlock); - - for (i = 0; i < svm_maxasid; i++) { - oct = i / 8; - bit = i % 8; - - if (svm_asidmap[oct] & __BIT(bit)) { - continue; - } - - svm_asidmap[oct] |= __BIT(bit); - vmcb->ctrl.guest_asid = i; - mutex_exit(&svm_asidlock); - return; - } - - /* - * No free ASID. Use the last one, which is shared and requires - * special TLB handling. - */ - cpudata->shared_asid = true; - vmcb->ctrl.guest_asid = svm_maxasid - 1; - mutex_exit(&svm_asidlock); -} -static void -svm_asid_free(struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - size_t oct, bit; - - if (cpudata->shared_asid) { - return; - } - - oct = vmcb->ctrl.guest_asid / 8; - bit = vmcb->ctrl.guest_asid % 8; - - mutex_enter(&svm_asidlock); - svm_asidmap[oct] &= ~__BIT(bit); - mutex_exit(&svm_asidlock); -} - -static void -svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - struct vmcb *vmcb = cpudata->vmcb; - - /* Allow reads/writes of Control Registers. */ - vmcb->ctrl.intercept_cr = 0; - - /* Allow reads/writes of Debug Registers. */ - vmcb->ctrl.intercept_dr = 0; - - /* Allow exceptions 0 to 31. */ - vmcb->ctrl.intercept_vec = 0; - - /* - * Allow: - * - SMI [smm interrupts] - * - VINTR [virtual interrupts] - * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] - * - RIDTR [reads of IDTR] - * - RGDTR [reads of GDTR] - * - RLDTR [reads of LDTR] - * - RTR [reads of TR] - * - WIDTR [writes of IDTR] - * - WGDTR [writes of GDTR] - * - WLDTR [writes of LDTR] - * - WTR [writes of TR] - * - RDTSC [rdtsc instruction] - * - PUSHF [pushf instruction] - * - POPF [popf instruction] - * - IRET [iret instruction] - * - INTN [int $n instructions] - * - INVD [invd instruction] - * - PAUSE [pause instruction] - * - INVLPG [invplg instruction] - * - TASKSW [task switches] - * - * Intercept the rest below. - */ - vmcb->ctrl.intercept_misc1 = - VMCB_CTRL_INTERCEPT_INTR | - VMCB_CTRL_INTERCEPT_NMI | - VMCB_CTRL_INTERCEPT_INIT | - VMCB_CTRL_INTERCEPT_RDPMC | - VMCB_CTRL_INTERCEPT_CPUID | - VMCB_CTRL_INTERCEPT_RSM | - VMCB_CTRL_INTERCEPT_HLT | - VMCB_CTRL_INTERCEPT_INVLPGA | - VMCB_CTRL_INTERCEPT_IOIO_PROT | - VMCB_CTRL_INTERCEPT_MSR_PROT | - VMCB_CTRL_INTERCEPT_FERR_FREEZE | - VMCB_CTRL_INTERCEPT_SHUTDOWN; - - /* - * Allow: - * - ICEBP [icebp instruction] - * - WBINVD [wbinvd instruction] - * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] - * - * Intercept the rest below. - */ - vmcb->ctrl.intercept_misc2 = - VMCB_CTRL_INTERCEPT_VMRUN | - VMCB_CTRL_INTERCEPT_VMMCALL | - VMCB_CTRL_INTERCEPT_VMLOAD | - VMCB_CTRL_INTERCEPT_VMSAVE | - VMCB_CTRL_INTERCEPT_STGI | - VMCB_CTRL_INTERCEPT_CLGI | - VMCB_CTRL_INTERCEPT_SKINIT | - VMCB_CTRL_INTERCEPT_RDTSCP | - VMCB_CTRL_INTERCEPT_MONITOR | - VMCB_CTRL_INTERCEPT_MWAIT | - VMCB_CTRL_INTERCEPT_XSETBV; - - /* Intercept all I/O accesses. */ - memset(cpudata->iobm, 0xFF, IOBM_SIZE); - vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; - - /* - * Allow: - * - EFER [read] - * - STAR [read, write] - * - LSTAR [read, write] - * - CSTAR [read, write] - * - SFMASK [read, write] - * - KERNELGSBASE [read, write] - * - SYSENTER_CS [read, write] - * - SYSENTER_ESP [read, write] - * - SYSENTER_EIP [read, write] - * - FSBASE [read, write] - * - GSBASE [read, write] - * - PAT [read, write] - * - TSC [read] - * - * Intercept the rest. - */ - memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); - svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); - vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; - - /* Generate ASID. */ - svm_asid_alloc(vcpu); - - /* Virtual TPR. */ - vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; - - /* Enable Nested Paging. */ - vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; - vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; - - /* Must always be set. */ - vmcb->state.efer = EFER_SVME; - cpudata->gxcr0 = XCR0_X87; - - /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; - cpudata->gfpu.xsh_xcomp_bv = 0; - - /* Set guest TSC to zero, more or less. */ - cpudata->tsc_offset = -cpu_counter(); - - /* These MSRs are static. */ - cpudata->star = rdmsr(MSR_STAR); - cpudata->lstar = rdmsr(MSR_LSTAR); - cpudata->cstar = rdmsr(MSR_CSTAR); - cpudata->sfmask = rdmsr(MSR_SFMASK); -} - -static int -svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata; - int error; - - /* Allocate the SVM cpudata. */ - cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, - UVM_KMF_WIRED|UVM_KMF_ZERO); - vcpu->cpudata = cpudata; - - /* VMCB */ - error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, - VMCB_NPAGES); - if (error) - goto error; - - /* I/O Bitmap */ - error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, - IOBM_NPAGES); - if (error) - goto error; - - /* MSR Bitmap */ - error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, - MSRBM_NPAGES); - if (error) - goto error; - - /* Init the VCPU info. */ - svm_vcpu_init(mach, vcpu); - - return 0; - -error: - if (cpudata->vmcb_pa) { - svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, - VMCB_NPAGES); - } - if (cpudata->iobm_pa) { - svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, - IOBM_NPAGES); - } - if (cpudata->msrbm_pa) { - svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, - MSRBM_NPAGES); - } - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); - return error; -} - -static void -svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct svm_cpudata *cpudata = vcpu->cpudata; - - svm_asid_free(vcpu); - - svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); - svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); - svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); - - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); -} #define SVM_SEG_ATTRIB_TYPE __BITS(4,0) #define SVM_SEG_ATTRIB_DPL __BITS(6,5) @@ -1737,7 +1479,8 @@ svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) #define SVM_SEG_ATTRIB_GRAN __BIT(11) static void -svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) +svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, + struct vmcb_segment *vseg) { vseg->selector = seg->selector; vseg->attrib = @@ -1768,7 +1511,7 @@ svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) } static inline bool -svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, +svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, uint64_t flags) { if (flags & NVMM_X64_STATE_CRS) { @@ -1798,7 +1541,7 @@ svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, static void svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) { - struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; + const struct nvmm_x64_state *state = data; struct svm_cpudata *cpudata = vcpu->cpudata; struct vmcb *vmcb = cpudata->vmcb; struct fxsave *fpustate; @@ -1869,9 +1612,10 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) } if (flags & NVMM_X64_STATE_MSRS) { - /* Bit EFER_SVME is mandatory. */ + /* + * EFER_SVME is mandatory. + */ vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; - vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; @@ -2024,6 +1768,251 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) /* -------------------------------------------------------------------------- */ static void +svm_asid_alloc(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + size_t i, oct, bit; + + mutex_enter(&svm_asidlock); + + for (i = 0; i < svm_maxasid; i++) { + oct = i / 8; + bit = i % 8; + + if (svm_asidmap[oct] & __BIT(bit)) { + continue; + } + + svm_asidmap[oct] |= __BIT(bit); + vmcb->ctrl.guest_asid = i; + mutex_exit(&svm_asidlock); + return; + } + + /* + * No free ASID. Use the last one, which is shared and requires + * special TLB handling. + */ + cpudata->shared_asid = true; + vmcb->ctrl.guest_asid = svm_maxasid - 1; + mutex_exit(&svm_asidlock); +} + +static void +svm_asid_free(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + size_t oct, bit; + + if (cpudata->shared_asid) { + return; + } + + oct = vmcb->ctrl.guest_asid / 8; + bit = vmcb->ctrl.guest_asid % 8; + + mutex_enter(&svm_asidlock); + svm_asidmap[oct] &= ~__BIT(bit); + mutex_exit(&svm_asidlock); +} + +static void +svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + + /* Allow reads/writes of Control Registers. */ + vmcb->ctrl.intercept_cr = 0; + + /* Allow reads/writes of Debug Registers. */ + vmcb->ctrl.intercept_dr = 0; + + /* Allow exceptions 0 to 31. */ + vmcb->ctrl.intercept_vec = 0; + + /* + * Allow: + * - SMI [smm interrupts] + * - VINTR [virtual interrupts] + * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] + * - RIDTR [reads of IDTR] + * - RGDTR [reads of GDTR] + * - RLDTR [reads of LDTR] + * - RTR [reads of TR] + * - WIDTR [writes of IDTR] + * - WGDTR [writes of GDTR] + * - WLDTR [writes of LDTR] + * - WTR [writes of TR] + * - RDTSC [rdtsc instruction] + * - PUSHF [pushf instruction] + * - POPF [popf instruction] + * - IRET [iret instruction] + * - INTN [int $n instructions] + * - INVD [invd instruction] + * - PAUSE [pause instruction] + * - INVLPG [invplg instruction] + * - TASKSW [task switches] + * + * Intercept the rest below. + */ + vmcb->ctrl.intercept_misc1 = + VMCB_CTRL_INTERCEPT_INTR | + VMCB_CTRL_INTERCEPT_NMI | + VMCB_CTRL_INTERCEPT_INIT | + VMCB_CTRL_INTERCEPT_RDPMC | + VMCB_CTRL_INTERCEPT_CPUID | + VMCB_CTRL_INTERCEPT_RSM | + VMCB_CTRL_INTERCEPT_HLT | + VMCB_CTRL_INTERCEPT_INVLPGA | + VMCB_CTRL_INTERCEPT_IOIO_PROT | + VMCB_CTRL_INTERCEPT_MSR_PROT | + VMCB_CTRL_INTERCEPT_FERR_FREEZE | + VMCB_CTRL_INTERCEPT_SHUTDOWN; + + /* + * Allow: + * - ICEBP [icebp instruction] + * - WBINVD [wbinvd instruction] + * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] + * + * Intercept the rest below. + */ + vmcb->ctrl.intercept_misc2 = + VMCB_CTRL_INTERCEPT_VMRUN | + VMCB_CTRL_INTERCEPT_VMMCALL | + VMCB_CTRL_INTERCEPT_VMLOAD | + VMCB_CTRL_INTERCEPT_VMSAVE | + VMCB_CTRL_INTERCEPT_STGI | + VMCB_CTRL_INTERCEPT_CLGI | + VMCB_CTRL_INTERCEPT_SKINIT | + VMCB_CTRL_INTERCEPT_RDTSCP | + VMCB_CTRL_INTERCEPT_MONITOR | + VMCB_CTRL_INTERCEPT_MWAIT | + VMCB_CTRL_INTERCEPT_XSETBV; + + /* Intercept all I/O accesses. */ + memset(cpudata->iobm, 0xFF, IOBM_SIZE); + vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; + + /* Allow direct access to certain MSRs. */ + memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); + svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); + vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; + + /* Generate ASID. */ + svm_asid_alloc(vcpu); + + /* Virtual TPR. */ + vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; + + /* Enable Nested Paging. */ + vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; + vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; + + /* Must always be set. */ + vmcb->state.efer = EFER_SVME; + cpudata->gxcr0 = XCR0_X87; + + /* Init XSAVE header. */ + cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->gfpu.xsh_xcomp_bv = 0; + + /* Set guest TSC to zero, more or less. */ + cpudata->tsc_offset = -cpu_counter(); + + /* These MSRs are static. */ + cpudata->star = rdmsr(MSR_STAR); + cpudata->lstar = rdmsr(MSR_LSTAR); + cpudata->cstar = rdmsr(MSR_CSTAR); + cpudata->sfmask = rdmsr(MSR_SFMASK); +} + +static int +svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata; + int error; + + /* Allocate the SVM cpudata. */ + cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, + roundup(sizeof(*cpudata), PAGE_SIZE), 0, + UVM_KMF_WIRED|UVM_KMF_ZERO); + vcpu->cpudata = cpudata; + + /* VMCB */ + error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, + VMCB_NPAGES); + if (error) + goto error; + + /* I/O Bitmap */ + error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, + IOBM_NPAGES); + if (error) + goto error; + + /* MSR Bitmap */ + error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, + MSRBM_NPAGES); + if (error) + goto error; + + /* Init the VCPU info. */ + svm_vcpu_init(mach, vcpu); + + return 0; + +error: + if (cpudata->vmcb_pa) { + svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, + VMCB_NPAGES); + } + if (cpudata->iobm_pa) { + svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, + IOBM_NPAGES); + } + if (cpudata->msrbm_pa) { + svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, + MSRBM_NPAGES); + } + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); + return error; +} + +static void +svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + + svm_asid_free(vcpu); + + svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); + svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); + svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); + + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); +} + +/* -------------------------------------------------------------------------- */ + +static void svm_tlb_flush(struct pmap *pm) { struct nvmm_machine *mach = pm->pm_data; diff --git a/sys/dev/nvmm/x86/nvmm_x86_vmx.c b/sys/dev/nvmm/x86/nvmm_x86_vmx.c index 6174a7de047..fa6aa7fd05a 100644 --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $ */ +/* $NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -1911,66 +1911,6 @@ vmx_memfree(paddr_t pa, vaddr_t va, size_t npages) /* -------------------------------------------------------------------------- */ static void -vmx_asid_alloc(struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata = vcpu->cpudata; - size_t i, oct, bit; - - mutex_enter(&vmx_asidlock); - - for (i = 0; i < vmx_maxasid; i++) { - oct = i / 8; - bit = i % 8; - - if (vmx_asidmap[oct] & __BIT(bit)) { - continue; - } - - cpudata->asid = i; - - vmx_asidmap[oct] |= __BIT(bit); - vmx_vmwrite(VMCS_VPID, i); - mutex_exit(&vmx_asidlock); - return; - } - - mutex_exit(&vmx_asidlock); - - panic("%s: impossible", __func__); -} - -static void -vmx_asid_free(struct nvmm_cpu *vcpu) -{ - size_t oct, bit; - uint64_t asid; - - vmx_vmread(VMCS_VPID, &asid); - - oct = asid / 8; - bit = asid % 8; - - mutex_enter(&vmx_asidlock); - vmx_asidmap[oct] &= ~__BIT(bit); - mutex_exit(&vmx_asidlock); -} - -static void -vmx_init_asid(uint32_t maxasid) -{ - size_t allocsz; - - mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE); - - vmx_maxasid = maxasid; - allocsz = roundup(maxasid, 8) / 8; - vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP); - - /* ASID 0 is reserved for the host. */ - vmx_asidmap[0] |= __BIT(0); -} - -static void vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) { uint64_t byte; @@ -1996,204 +1936,6 @@ vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) } } -static void -vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata = vcpu->cpudata; - struct vmcs *vmcs = cpudata->vmcs; - struct msr_entry *gmsr = cpudata->gmsr; - extern uint8_t vmx_resume_rip; - uint64_t rev, eptp; - - rev = vmx_get_revision(); - - memset(vmcs, 0, VMCS_SIZE); - vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION); - vmcs->abort = 0; - - vmx_vmcs_enter(vcpu); - - /* No link pointer. */ - vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF); - - /* Install the CTLSs. */ - vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls); - vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls); - vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2); - vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls); - vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls); - - /* Allow direct access to certain MSRs. */ - memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); - vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES, - true, false); - vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa); - - /* - * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This - * includes the L1D_FLUSH MSR, to mitigate L1TF. - */ - gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR; - gmsr[VMX_MSRLIST_STAR].val = 0; - gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR; - gmsr[VMX_MSRLIST_LSTAR].val = 0; - gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR; - gmsr[VMX_MSRLIST_CSTAR].val = 0; - gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK; - gmsr[VMX_MSRLIST_SFMASK].val = 0; - gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE; - gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0; - gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD; - gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH; - vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa); - vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa); - vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr); - vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR); - - /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */ - vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET); - vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET); - - /* Force CR4_VMXE to zero. */ - vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE); - - /* Set the Host state for resuming. */ - vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip); - vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); - vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0); - vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0); - vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0); - vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt); - vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT)); - vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER)); - vmx_vmwrite(VMCS_HOST_CR0, rcr0()); - - /* Generate ASID. */ - vmx_asid_alloc(vcpu); - - /* Enable Extended Paging, 4-Level. */ - eptp = - __SHIFTIN(vmx_eptp_type, EPTP_TYPE) | - __SHIFTIN(4-1, EPTP_WALKLEN) | - EPTP_FLAGS_AD | - mach->vm->vm_map.pmap->pm_pdirpa[0]; - vmx_vmwrite(VMCS_EPTP, eptp); - - /* Init IA32_MISC_ENABLE. */ - cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE); - cpudata->gmsr_misc_enable &= - ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN); - cpudata->gmsr_misc_enable |= - (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); - - /* Must always be set. */ - vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE); - vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE); - cpudata->gxcr0 = XCR0_X87; - - /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; - cpudata->gfpu.xsh_xcomp_bv = 0; - - /* Set guest TSC to zero, more or less. */ - cpudata->tsc_offset = -cpu_counter(); - - /* These MSRs are static. */ - cpudata->star = rdmsr(MSR_STAR); - cpudata->cstar = rdmsr(MSR_CSTAR); - cpudata->sfmask = rdmsr(MSR_SFMASK); - - vmx_vmcs_leave(vcpu); -} - -static int -vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata; - int error; - - /* Allocate the VMX cpudata. */ - cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, - UVM_KMF_WIRED|UVM_KMF_ZERO); - vcpu->cpudata = cpudata; - - /* VMCS */ - error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs, - VMCS_NPAGES); - if (error) - goto error; - - /* MSR Bitmap */ - error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, - MSRBM_NPAGES); - if (error) - goto error; - - /* Guest MSR List */ - error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1); - if (error) - goto error; - - kcpuset_create(&cpudata->htlb_want_flush, true); - - /* Init the VCPU info. */ - vmx_vcpu_init(mach, vcpu); - - return 0; - -error: - if (cpudata->vmcs_pa) { - vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, - VMCS_NPAGES); - } - if (cpudata->msrbm_pa) { - vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, - MSRBM_NPAGES); - } - if (cpudata->gmsr_pa) { - vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); - } - - kmem_free(cpudata, sizeof(*cpudata)); - return error; -} - -static void -vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) -{ - struct vmx_cpudata *cpudata = vcpu->cpudata; - - vmx_vmcs_enter(vcpu); - vmx_asid_free(vcpu); - vmx_vmcs_leave(vcpu); - - kcpuset_destroy(cpudata->htlb_want_flush); - - vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES); - vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); - vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); - uvm_km_free(kernel_map, (vaddr_t)cpudata, - roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); -} - #define VMX_SEG_ATTRIB_TYPE __BITS(4,0) #define VMX_SEG_ATTRIB_DPL __BITS(6,5) #define VMX_SEG_ATTRIB_P __BIT(7) @@ -2204,7 +1946,7 @@ vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) #define VMX_SEG_ATTRIB_UNUSABLE __BIT(16) static void -vmx_vcpu_setstate_seg(struct nvmm_x64_state_seg *segs, int idx) +vmx_vcpu_setstate_seg(const struct nvmm_x64_state_seg *segs, int idx) { uint64_t attrib; @@ -2251,7 +1993,7 @@ vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx) } static inline bool -vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags) +vmx_state_tlb_flush(const struct nvmm_x64_state *state, uint64_t flags) { uint64_t cr0, cr3, cr4, efer; @@ -2284,7 +2026,7 @@ vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags) static void vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) { - struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; + const struct nvmm_x64_state *state = data; struct vmx_cpudata *cpudata = vcpu->cpudata; struct fxsave *fpustate; uint64_t ctls1, intstate; @@ -2318,14 +2060,15 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) } if (flags & NVMM_X64_STATE_CRS) { - /* These bits are mandatory. */ - state->crs[NVMM_X64_CR_CR4] |= CR4_VMXE; - state->crs[NVMM_X64_CR_CR0] |= CR0_NE; - - vmx_vmwrite(VMCS_GUEST_CR0, state->crs[NVMM_X64_CR_CR0]); + /* + * CR0_NE and CR4_VMXE are mandatory. + */ + vmx_vmwrite(VMCS_GUEST_CR0, + state->crs[NVMM_X64_CR_CR0] | CR0_NE); cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2]; vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE? - vmx_vmwrite(VMCS_GUEST_CR4, state->crs[NVMM_X64_CR_CR4]); + vmx_vmwrite(VMCS_GUEST_CR4, + state->crs[NVMM_X64_CR_CR4] | CR4_VMXE); cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; if (vmx_xcr0_mask != 0) { @@ -2513,6 +2256,251 @@ vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) /* -------------------------------------------------------------------------- */ static void +vmx_asid_alloc(struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata = vcpu->cpudata; + size_t i, oct, bit; + + mutex_enter(&vmx_asidlock); + + for (i = 0; i < vmx_maxasid; i++) { + oct = i / 8; + bit = i % 8; + + if (vmx_asidmap[oct] & __BIT(bit)) { + continue; + } + + cpudata->asid = i; + + vmx_asidmap[oct] |= __BIT(bit); + vmx_vmwrite(VMCS_VPID, i); + mutex_exit(&vmx_asidlock); + return; + } + + mutex_exit(&vmx_asidlock); + + panic("%s: impossible", __func__); +} + +static void +vmx_asid_free(struct nvmm_cpu *vcpu) +{ + size_t oct, bit; + uint64_t asid; + + vmx_vmread(VMCS_VPID, &asid); + + oct = asid / 8; + bit = asid % 8; + + mutex_enter(&vmx_asidlock); + vmx_asidmap[oct] &= ~__BIT(bit); + mutex_exit(&vmx_asidlock); +} + +static void +vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata = vcpu->cpudata; + struct vmcs *vmcs = cpudata->vmcs; + struct msr_entry *gmsr = cpudata->gmsr; + extern uint8_t vmx_resume_rip; + uint64_t rev, eptp; + + rev = vmx_get_revision(); + + memset(vmcs, 0, VMCS_SIZE); + vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION); + vmcs->abort = 0; + + vmx_vmcs_enter(vcpu); + + /* No link pointer. */ + vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF); + + /* Install the CTLSs. */ + vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls); + vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls); + vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2); + vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls); + vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls); + + /* Allow direct access to certain MSRs. */ + memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); + vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES, + true, false); + vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa); + + /* + * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This + * includes the L1D_FLUSH MSR, to mitigate L1TF. + */ + gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR; + gmsr[VMX_MSRLIST_STAR].val = 0; + gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR; + gmsr[VMX_MSRLIST_LSTAR].val = 0; + gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR; + gmsr[VMX_MSRLIST_CSTAR].val = 0; + gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK; + gmsr[VMX_MSRLIST_SFMASK].val = 0; + gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE; + gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0; + gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD; + gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH; + vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa); + vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa); + vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr); + vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR); + + /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */ + vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET); + vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET); + + /* Force CR4_VMXE to zero. */ + vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE); + + /* Set the Host state for resuming. */ + vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip); + vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); + vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0); + vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0); + vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0); + vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt); + vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT)); + vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER)); + vmx_vmwrite(VMCS_HOST_CR0, rcr0()); + + /* Generate ASID. */ + vmx_asid_alloc(vcpu); + + /* Enable Extended Paging, 4-Level. */ + eptp = + __SHIFTIN(vmx_eptp_type, EPTP_TYPE) | + __SHIFTIN(4-1, EPTP_WALKLEN) | + EPTP_FLAGS_AD | + mach->vm->vm_map.pmap->pm_pdirpa[0]; + vmx_vmwrite(VMCS_EPTP, eptp); + + /* Init IA32_MISC_ENABLE. */ + cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE); + cpudata->gmsr_misc_enable &= + ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN); + cpudata->gmsr_misc_enable |= + (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); + + /* Must always be set. */ + vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE); + vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE); + cpudata->gxcr0 = XCR0_X87; + + /* Init XSAVE header. */ + cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->gfpu.xsh_xcomp_bv = 0; + + /* Set guest TSC to zero, more or less. */ + cpudata->tsc_offset = -cpu_counter(); + + /* These MSRs are static. */ + cpudata->star = rdmsr(MSR_STAR); + cpudata->cstar = rdmsr(MSR_CSTAR); + cpudata->sfmask = rdmsr(MSR_SFMASK); + + vmx_vmcs_leave(vcpu); +} + +static int +vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata; + int error; + + /* Allocate the VMX cpudata. */ + cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, + roundup(sizeof(*cpudata), PAGE_SIZE), 0, + UVM_KMF_WIRED|UVM_KMF_ZERO); + vcpu->cpudata = cpudata; + + /* VMCS */ + error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs, + VMCS_NPAGES); + if (error) + goto error; + + /* MSR Bitmap */ + error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, + MSRBM_NPAGES); + if (error) + goto error; + + /* Guest MSR List */ + error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1); + if (error) + goto error; + + kcpuset_create(&cpudata->htlb_want_flush, true); + + /* Init the VCPU info. */ + vmx_vcpu_init(mach, vcpu); + + return 0; + +error: + if (cpudata->vmcs_pa) { + vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, + VMCS_NPAGES); + } + if (cpudata->msrbm_pa) { + vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, + MSRBM_NPAGES); + } + if (cpudata->gmsr_pa) { + vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); + } + + kmem_free(cpudata, sizeof(*cpudata)); + return error; +} + +static void +vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) +{ + struct vmx_cpudata *cpudata = vcpu->cpudata; + + vmx_vmcs_enter(vcpu); + vmx_asid_free(vcpu); + vmx_vmcs_leave(vcpu); + + kcpuset_destroy(cpudata->htlb_want_flush); + + vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES); + vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); + vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1); + uvm_km_free(kernel_map, (vaddr_t)cpudata, + roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); +} + +/* -------------------------------------------------------------------------- */ + +static void vmx_tlb_flush(struct pmap *pm) { struct nvmm_machine *mach = pm->pm_data; @@ -2748,6 +2736,21 @@ vmx_ident(void) } static void +vmx_init_asid(uint32_t maxasid) +{ + size_t allocsz; + + mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE); + + vmx_maxasid = maxasid; + allocsz = roundup(maxasid, 8) / 8; + vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP); + + /* ASID 0 is reserved for the host. */ + vmx_asidmap[0] |= __BIT(0); +} + +static void vmx_change_cpu(void *arg1, void *arg2) { struct cpu_info *ci = curcpu(); |
