summaryrefslogtreecommitdiff
path: root/sys/dev
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev')
-rw-r--r--sys/dev/nvmm/x86/nvmm_x86_svm.c519
-rw-r--r--sys/dev/nvmm/x86/nvmm_x86_vmx.c541
2 files changed, 526 insertions, 534 deletions
diff --git a/sys/dev/nvmm/x86/nvmm_x86_svm.c b/sys/dev/nvmm/x86/nvmm_x86_svm.c
index dc47f069ea6..756cd4896cd 100644
--- a/sys/dev/nvmm/x86/nvmm_x86_svm.c
+++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $ */
+/* $NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1468,265 +1468,7 @@ svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write)
}
}
-static void
-svm_asid_alloc(struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
- size_t i, oct, bit;
-
- mutex_enter(&svm_asidlock);
-
- for (i = 0; i < svm_maxasid; i++) {
- oct = i / 8;
- bit = i % 8;
-
- if (svm_asidmap[oct] & __BIT(bit)) {
- continue;
- }
-
- svm_asidmap[oct] |= __BIT(bit);
- vmcb->ctrl.guest_asid = i;
- mutex_exit(&svm_asidlock);
- return;
- }
-
- /*
- * No free ASID. Use the last one, which is shared and requires
- * special TLB handling.
- */
- cpudata->shared_asid = true;
- vmcb->ctrl.guest_asid = svm_maxasid - 1;
- mutex_exit(&svm_asidlock);
-}
-static void
-svm_asid_free(struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
- size_t oct, bit;
-
- if (cpudata->shared_asid) {
- return;
- }
-
- oct = vmcb->ctrl.guest_asid / 8;
- bit = vmcb->ctrl.guest_asid % 8;
-
- mutex_enter(&svm_asidlock);
- svm_asidmap[oct] &= ~__BIT(bit);
- mutex_exit(&svm_asidlock);
-}
-
-static void
-svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
-
- /* Allow reads/writes of Control Registers. */
- vmcb->ctrl.intercept_cr = 0;
-
- /* Allow reads/writes of Debug Registers. */
- vmcb->ctrl.intercept_dr = 0;
-
- /* Allow exceptions 0 to 31. */
- vmcb->ctrl.intercept_vec = 0;
-
- /*
- * Allow:
- * - SMI [smm interrupts]
- * - VINTR [virtual interrupts]
- * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP]
- * - RIDTR [reads of IDTR]
- * - RGDTR [reads of GDTR]
- * - RLDTR [reads of LDTR]
- * - RTR [reads of TR]
- * - WIDTR [writes of IDTR]
- * - WGDTR [writes of GDTR]
- * - WLDTR [writes of LDTR]
- * - WTR [writes of TR]
- * - RDTSC [rdtsc instruction]
- * - PUSHF [pushf instruction]
- * - POPF [popf instruction]
- * - IRET [iret instruction]
- * - INTN [int $n instructions]
- * - INVD [invd instruction]
- * - PAUSE [pause instruction]
- * - INVLPG [invplg instruction]
- * - TASKSW [task switches]
- *
- * Intercept the rest below.
- */
- vmcb->ctrl.intercept_misc1 =
- VMCB_CTRL_INTERCEPT_INTR |
- VMCB_CTRL_INTERCEPT_NMI |
- VMCB_CTRL_INTERCEPT_INIT |
- VMCB_CTRL_INTERCEPT_RDPMC |
- VMCB_CTRL_INTERCEPT_CPUID |
- VMCB_CTRL_INTERCEPT_RSM |
- VMCB_CTRL_INTERCEPT_HLT |
- VMCB_CTRL_INTERCEPT_INVLPGA |
- VMCB_CTRL_INTERCEPT_IOIO_PROT |
- VMCB_CTRL_INTERCEPT_MSR_PROT |
- VMCB_CTRL_INTERCEPT_FERR_FREEZE |
- VMCB_CTRL_INTERCEPT_SHUTDOWN;
-
- /*
- * Allow:
- * - ICEBP [icebp instruction]
- * - WBINVD [wbinvd instruction]
- * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction]
- *
- * Intercept the rest below.
- */
- vmcb->ctrl.intercept_misc2 =
- VMCB_CTRL_INTERCEPT_VMRUN |
- VMCB_CTRL_INTERCEPT_VMMCALL |
- VMCB_CTRL_INTERCEPT_VMLOAD |
- VMCB_CTRL_INTERCEPT_VMSAVE |
- VMCB_CTRL_INTERCEPT_STGI |
- VMCB_CTRL_INTERCEPT_CLGI |
- VMCB_CTRL_INTERCEPT_SKINIT |
- VMCB_CTRL_INTERCEPT_RDTSCP |
- VMCB_CTRL_INTERCEPT_MONITOR |
- VMCB_CTRL_INTERCEPT_MWAIT |
- VMCB_CTRL_INTERCEPT_XSETBV;
-
- /* Intercept all I/O accesses. */
- memset(cpudata->iobm, 0xFF, IOBM_SIZE);
- vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
-
- /*
- * Allow:
- * - EFER [read]
- * - STAR [read, write]
- * - LSTAR [read, write]
- * - CSTAR [read, write]
- * - SFMASK [read, write]
- * - KERNELGSBASE [read, write]
- * - SYSENTER_CS [read, write]
- * - SYSENTER_ESP [read, write]
- * - SYSENTER_EIP [read, write]
- * - FSBASE [read, write]
- * - GSBASE [read, write]
- * - PAT [read, write]
- * - TSC [read]
- *
- * Intercept the rest.
- */
- memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
- vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
-
- /* Generate ASID. */
- svm_asid_alloc(vcpu);
-
- /* Virtual TPR. */
- vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING;
-
- /* Enable Nested Paging. */
- vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP;
- vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0];
-
- /* Must always be set. */
- vmcb->state.efer = EFER_SVME;
- cpudata->gxcr0 = XCR0_X87;
-
- /* Init XSAVE header. */
- cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
- cpudata->gfpu.xsh_xcomp_bv = 0;
-
- /* Set guest TSC to zero, more or less. */
- cpudata->tsc_offset = -cpu_counter();
-
- /* These MSRs are static. */
- cpudata->star = rdmsr(MSR_STAR);
- cpudata->lstar = rdmsr(MSR_LSTAR);
- cpudata->cstar = rdmsr(MSR_CSTAR);
- cpudata->sfmask = rdmsr(MSR_SFMASK);
-}
-
-static int
-svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata;
- int error;
-
- /* Allocate the SVM cpudata. */
- cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
- roundup(sizeof(*cpudata), PAGE_SIZE), 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- vcpu->cpudata = cpudata;
-
- /* VMCB */
- error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
- VMCB_NPAGES);
- if (error)
- goto error;
-
- /* I/O Bitmap */
- error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
- IOBM_NPAGES);
- if (error)
- goto error;
-
- /* MSR Bitmap */
- error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
- MSRBM_NPAGES);
- if (error)
- goto error;
-
- /* Init the VCPU info. */
- svm_vcpu_init(mach, vcpu);
-
- return 0;
-
-error:
- if (cpudata->vmcb_pa) {
- svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
- VMCB_NPAGES);
- }
- if (cpudata->iobm_pa) {
- svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
- IOBM_NPAGES);
- }
- if (cpudata->msrbm_pa) {
- svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
- MSRBM_NPAGES);
- }
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
- return error;
-}
-
-static void
-svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
-
- svm_asid_free(vcpu);
-
- svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
- svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
- svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
-
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
-}
#define SVM_SEG_ATTRIB_TYPE __BITS(4,0)
#define SVM_SEG_ATTRIB_DPL __BITS(6,5)
@@ -1737,7 +1479,8 @@ svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
#define SVM_SEG_ATTRIB_GRAN __BIT(11)
static void
-svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg)
+svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg,
+ struct vmcb_segment *vseg)
{
vseg->selector = seg->selector;
vseg->attrib =
@@ -1768,7 +1511,7 @@ svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg)
}
static inline bool
-svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state,
+svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state,
uint64_t flags)
{
if (flags & NVMM_X64_STATE_CRS) {
@@ -1798,7 +1541,7 @@ svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state,
static void
svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
{
- struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
+ const struct nvmm_x64_state *state = data;
struct svm_cpudata *cpudata = vcpu->cpudata;
struct vmcb *vmcb = cpudata->vmcb;
struct fxsave *fpustate;
@@ -1869,9 +1612,10 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
}
if (flags & NVMM_X64_STATE_MSRS) {
- /* Bit EFER_SVME is mandatory. */
+ /*
+ * EFER_SVME is mandatory.
+ */
vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME;
-
vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR];
vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR];
vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR];
@@ -2024,6 +1768,251 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
/* -------------------------------------------------------------------------- */
static void
+svm_asid_alloc(struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+ size_t i, oct, bit;
+
+ mutex_enter(&svm_asidlock);
+
+ for (i = 0; i < svm_maxasid; i++) {
+ oct = i / 8;
+ bit = i % 8;
+
+ if (svm_asidmap[oct] & __BIT(bit)) {
+ continue;
+ }
+
+ svm_asidmap[oct] |= __BIT(bit);
+ vmcb->ctrl.guest_asid = i;
+ mutex_exit(&svm_asidlock);
+ return;
+ }
+
+ /*
+ * No free ASID. Use the last one, which is shared and requires
+ * special TLB handling.
+ */
+ cpudata->shared_asid = true;
+ vmcb->ctrl.guest_asid = svm_maxasid - 1;
+ mutex_exit(&svm_asidlock);
+}
+
+static void
+svm_asid_free(struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+ size_t oct, bit;
+
+ if (cpudata->shared_asid) {
+ return;
+ }
+
+ oct = vmcb->ctrl.guest_asid / 8;
+ bit = vmcb->ctrl.guest_asid % 8;
+
+ mutex_enter(&svm_asidlock);
+ svm_asidmap[oct] &= ~__BIT(bit);
+ mutex_exit(&svm_asidlock);
+}
+
+static void
+svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+
+ /* Allow reads/writes of Control Registers. */
+ vmcb->ctrl.intercept_cr = 0;
+
+ /* Allow reads/writes of Debug Registers. */
+ vmcb->ctrl.intercept_dr = 0;
+
+ /* Allow exceptions 0 to 31. */
+ vmcb->ctrl.intercept_vec = 0;
+
+ /*
+ * Allow:
+ * - SMI [smm interrupts]
+ * - VINTR [virtual interrupts]
+ * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP]
+ * - RIDTR [reads of IDTR]
+ * - RGDTR [reads of GDTR]
+ * - RLDTR [reads of LDTR]
+ * - RTR [reads of TR]
+ * - WIDTR [writes of IDTR]
+ * - WGDTR [writes of GDTR]
+ * - WLDTR [writes of LDTR]
+ * - WTR [writes of TR]
+ * - RDTSC [rdtsc instruction]
+ * - PUSHF [pushf instruction]
+ * - POPF [popf instruction]
+ * - IRET [iret instruction]
+ * - INTN [int $n instructions]
+ * - INVD [invd instruction]
+ * - PAUSE [pause instruction]
+ * - INVLPG [invplg instruction]
+ * - TASKSW [task switches]
+ *
+ * Intercept the rest below.
+ */
+ vmcb->ctrl.intercept_misc1 =
+ VMCB_CTRL_INTERCEPT_INTR |
+ VMCB_CTRL_INTERCEPT_NMI |
+ VMCB_CTRL_INTERCEPT_INIT |
+ VMCB_CTRL_INTERCEPT_RDPMC |
+ VMCB_CTRL_INTERCEPT_CPUID |
+ VMCB_CTRL_INTERCEPT_RSM |
+ VMCB_CTRL_INTERCEPT_HLT |
+ VMCB_CTRL_INTERCEPT_INVLPGA |
+ VMCB_CTRL_INTERCEPT_IOIO_PROT |
+ VMCB_CTRL_INTERCEPT_MSR_PROT |
+ VMCB_CTRL_INTERCEPT_FERR_FREEZE |
+ VMCB_CTRL_INTERCEPT_SHUTDOWN;
+
+ /*
+ * Allow:
+ * - ICEBP [icebp instruction]
+ * - WBINVD [wbinvd instruction]
+ * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction]
+ *
+ * Intercept the rest below.
+ */
+ vmcb->ctrl.intercept_misc2 =
+ VMCB_CTRL_INTERCEPT_VMRUN |
+ VMCB_CTRL_INTERCEPT_VMMCALL |
+ VMCB_CTRL_INTERCEPT_VMLOAD |
+ VMCB_CTRL_INTERCEPT_VMSAVE |
+ VMCB_CTRL_INTERCEPT_STGI |
+ VMCB_CTRL_INTERCEPT_CLGI |
+ VMCB_CTRL_INTERCEPT_SKINIT |
+ VMCB_CTRL_INTERCEPT_RDTSCP |
+ VMCB_CTRL_INTERCEPT_MONITOR |
+ VMCB_CTRL_INTERCEPT_MWAIT |
+ VMCB_CTRL_INTERCEPT_XSETBV;
+
+ /* Intercept all I/O accesses. */
+ memset(cpudata->iobm, 0xFF, IOBM_SIZE);
+ vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
+
+ /* Allow direct access to certain MSRs. */
+ memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
+ vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
+
+ /* Generate ASID. */
+ svm_asid_alloc(vcpu);
+
+ /* Virtual TPR. */
+ vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING;
+
+ /* Enable Nested Paging. */
+ vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP;
+ vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0];
+
+ /* Must always be set. */
+ vmcb->state.efer = EFER_SVME;
+ cpudata->gxcr0 = XCR0_X87;
+
+ /* Init XSAVE header. */
+ cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
+ cpudata->gfpu.xsh_xcomp_bv = 0;
+
+ /* Set guest TSC to zero, more or less. */
+ cpudata->tsc_offset = -cpu_counter();
+
+ /* These MSRs are static. */
+ cpudata->star = rdmsr(MSR_STAR);
+ cpudata->lstar = rdmsr(MSR_LSTAR);
+ cpudata->cstar = rdmsr(MSR_CSTAR);
+ cpudata->sfmask = rdmsr(MSR_SFMASK);
+}
+
+static int
+svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata;
+ int error;
+
+ /* Allocate the SVM cpudata. */
+ cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
+ roundup(sizeof(*cpudata), PAGE_SIZE), 0,
+ UVM_KMF_WIRED|UVM_KMF_ZERO);
+ vcpu->cpudata = cpudata;
+
+ /* VMCB */
+ error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
+ VMCB_NPAGES);
+ if (error)
+ goto error;
+
+ /* I/O Bitmap */
+ error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
+ IOBM_NPAGES);
+ if (error)
+ goto error;
+
+ /* MSR Bitmap */
+ error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
+ MSRBM_NPAGES);
+ if (error)
+ goto error;
+
+ /* Init the VCPU info. */
+ svm_vcpu_init(mach, vcpu);
+
+ return 0;
+
+error:
+ if (cpudata->vmcb_pa) {
+ svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
+ VMCB_NPAGES);
+ }
+ if (cpudata->iobm_pa) {
+ svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
+ IOBM_NPAGES);
+ }
+ if (cpudata->msrbm_pa) {
+ svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
+ MSRBM_NPAGES);
+ }
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
+ return error;
+}
+
+static void
+svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+
+ svm_asid_free(vcpu);
+
+ svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
+ svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
+ svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
+
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
+}
+
+/* -------------------------------------------------------------------------- */
+
+static void
svm_tlb_flush(struct pmap *pm)
{
struct nvmm_machine *mach = pm->pm_data;
diff --git a/sys/dev/nvmm/x86/nvmm_x86_vmx.c b/sys/dev/nvmm/x86/nvmm_x86_vmx.c
index 6174a7de047..fa6aa7fd05a 100644
--- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c
+++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1911,66 +1911,6 @@ vmx_memfree(paddr_t pa, vaddr_t va, size_t npages)
/* -------------------------------------------------------------------------- */
static void
-vmx_asid_alloc(struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- size_t i, oct, bit;
-
- mutex_enter(&vmx_asidlock);
-
- for (i = 0; i < vmx_maxasid; i++) {
- oct = i / 8;
- bit = i % 8;
-
- if (vmx_asidmap[oct] & __BIT(bit)) {
- continue;
- }
-
- cpudata->asid = i;
-
- vmx_asidmap[oct] |= __BIT(bit);
- vmx_vmwrite(VMCS_VPID, i);
- mutex_exit(&vmx_asidlock);
- return;
- }
-
- mutex_exit(&vmx_asidlock);
-
- panic("%s: impossible", __func__);
-}
-
-static void
-vmx_asid_free(struct nvmm_cpu *vcpu)
-{
- size_t oct, bit;
- uint64_t asid;
-
- vmx_vmread(VMCS_VPID, &asid);
-
- oct = asid / 8;
- bit = asid % 8;
-
- mutex_enter(&vmx_asidlock);
- vmx_asidmap[oct] &= ~__BIT(bit);
- mutex_exit(&vmx_asidlock);
-}
-
-static void
-vmx_init_asid(uint32_t maxasid)
-{
- size_t allocsz;
-
- mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE);
-
- vmx_maxasid = maxasid;
- allocsz = roundup(maxasid, 8) / 8;
- vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP);
-
- /* ASID 0 is reserved for the host. */
- vmx_asidmap[0] |= __BIT(0);
-}
-
-static void
vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write)
{
uint64_t byte;
@@ -1996,204 +1936,6 @@ vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write)
}
}
-static void
-vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- struct vmcs *vmcs = cpudata->vmcs;
- struct msr_entry *gmsr = cpudata->gmsr;
- extern uint8_t vmx_resume_rip;
- uint64_t rev, eptp;
-
- rev = vmx_get_revision();
-
- memset(vmcs, 0, VMCS_SIZE);
- vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION);
- vmcs->abort = 0;
-
- vmx_vmcs_enter(vcpu);
-
- /* No link pointer. */
- vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF);
-
- /* Install the CTLSs. */
- vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls);
- vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls);
- vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2);
- vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls);
- vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls);
-
- /* Allow direct access to certain MSRs. */
- memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES,
- true, false);
- vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa);
-
- /*
- * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This
- * includes the L1D_FLUSH MSR, to mitigate L1TF.
- */
- gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR;
- gmsr[VMX_MSRLIST_STAR].val = 0;
- gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR;
- gmsr[VMX_MSRLIST_LSTAR].val = 0;
- gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR;
- gmsr[VMX_MSRLIST_CSTAR].val = 0;
- gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK;
- gmsr[VMX_MSRLIST_SFMASK].val = 0;
- gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE;
- gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0;
- gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD;
- gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH;
- vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa);
- vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa);
- vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr);
- vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR);
-
- /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */
- vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET);
- vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET);
-
- /* Force CR4_VMXE to zero. */
- vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE);
-
- /* Set the Host state for resuming. */
- vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip);
- vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0);
- vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0);
- vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt);
- vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT));
- vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER));
- vmx_vmwrite(VMCS_HOST_CR0, rcr0());
-
- /* Generate ASID. */
- vmx_asid_alloc(vcpu);
-
- /* Enable Extended Paging, 4-Level. */
- eptp =
- __SHIFTIN(vmx_eptp_type, EPTP_TYPE) |
- __SHIFTIN(4-1, EPTP_WALKLEN) |
- EPTP_FLAGS_AD |
- mach->vm->vm_map.pmap->pm_pdirpa[0];
- vmx_vmwrite(VMCS_EPTP, eptp);
-
- /* Init IA32_MISC_ENABLE. */
- cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE);
- cpudata->gmsr_misc_enable &=
- ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN);
- cpudata->gmsr_misc_enable |=
- (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL);
-
- /* Must always be set. */
- vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE);
- vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE);
- cpudata->gxcr0 = XCR0_X87;
-
- /* Init XSAVE header. */
- cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
- cpudata->gfpu.xsh_xcomp_bv = 0;
-
- /* Set guest TSC to zero, more or less. */
- cpudata->tsc_offset = -cpu_counter();
-
- /* These MSRs are static. */
- cpudata->star = rdmsr(MSR_STAR);
- cpudata->cstar = rdmsr(MSR_CSTAR);
- cpudata->sfmask = rdmsr(MSR_SFMASK);
-
- vmx_vmcs_leave(vcpu);
-}
-
-static int
-vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata;
- int error;
-
- /* Allocate the VMX cpudata. */
- cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map,
- roundup(sizeof(*cpudata), PAGE_SIZE), 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- vcpu->cpudata = cpudata;
-
- /* VMCS */
- error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs,
- VMCS_NPAGES);
- if (error)
- goto error;
-
- /* MSR Bitmap */
- error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
- MSRBM_NPAGES);
- if (error)
- goto error;
-
- /* Guest MSR List */
- error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1);
- if (error)
- goto error;
-
- kcpuset_create(&cpudata->htlb_want_flush, true);
-
- /* Init the VCPU info. */
- vmx_vcpu_init(mach, vcpu);
-
- return 0;
-
-error:
- if (cpudata->vmcs_pa) {
- vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs,
- VMCS_NPAGES);
- }
- if (cpudata->msrbm_pa) {
- vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
- MSRBM_NPAGES);
- }
- if (cpudata->gmsr_pa) {
- vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
- }
-
- kmem_free(cpudata, sizeof(*cpudata));
- return error;
-}
-
-static void
-vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata = vcpu->cpudata;
-
- vmx_vmcs_enter(vcpu);
- vmx_asid_free(vcpu);
- vmx_vmcs_leave(vcpu);
-
- kcpuset_destroy(cpudata->htlb_want_flush);
-
- vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES);
- vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
- vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
-}
-
#define VMX_SEG_ATTRIB_TYPE __BITS(4,0)
#define VMX_SEG_ATTRIB_DPL __BITS(6,5)
#define VMX_SEG_ATTRIB_P __BIT(7)
@@ -2204,7 +1946,7 @@ vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
#define VMX_SEG_ATTRIB_UNUSABLE __BIT(16)
static void
-vmx_vcpu_setstate_seg(struct nvmm_x64_state_seg *segs, int idx)
+vmx_vcpu_setstate_seg(const struct nvmm_x64_state_seg *segs, int idx)
{
uint64_t attrib;
@@ -2251,7 +1993,7 @@ vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx)
}
static inline bool
-vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags)
+vmx_state_tlb_flush(const struct nvmm_x64_state *state, uint64_t flags)
{
uint64_t cr0, cr3, cr4, efer;
@@ -2284,7 +2026,7 @@ vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags)
static void
vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
{
- struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
+ const struct nvmm_x64_state *state = data;
struct vmx_cpudata *cpudata = vcpu->cpudata;
struct fxsave *fpustate;
uint64_t ctls1, intstate;
@@ -2318,14 +2060,15 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
}
if (flags & NVMM_X64_STATE_CRS) {
- /* These bits are mandatory. */
- state->crs[NVMM_X64_CR_CR4] |= CR4_VMXE;
- state->crs[NVMM_X64_CR_CR0] |= CR0_NE;
-
- vmx_vmwrite(VMCS_GUEST_CR0, state->crs[NVMM_X64_CR_CR0]);
+ /*
+ * CR0_NE and CR4_VMXE are mandatory.
+ */
+ vmx_vmwrite(VMCS_GUEST_CR0,
+ state->crs[NVMM_X64_CR_CR0] | CR0_NE);
cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2];
vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE?
- vmx_vmwrite(VMCS_GUEST_CR4, state->crs[NVMM_X64_CR_CR4]);
+ vmx_vmwrite(VMCS_GUEST_CR4,
+ state->crs[NVMM_X64_CR_CR4] | CR4_VMXE);
cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8];
if (vmx_xcr0_mask != 0) {
@@ -2513,6 +2256,251 @@ vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
/* -------------------------------------------------------------------------- */
static void
+vmx_asid_alloc(struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ size_t i, oct, bit;
+
+ mutex_enter(&vmx_asidlock);
+
+ for (i = 0; i < vmx_maxasid; i++) {
+ oct = i / 8;
+ bit = i % 8;
+
+ if (vmx_asidmap[oct] & __BIT(bit)) {
+ continue;
+ }
+
+ cpudata->asid = i;
+
+ vmx_asidmap[oct] |= __BIT(bit);
+ vmx_vmwrite(VMCS_VPID, i);
+ mutex_exit(&vmx_asidlock);
+ return;
+ }
+
+ mutex_exit(&vmx_asidlock);
+
+ panic("%s: impossible", __func__);
+}
+
+static void
+vmx_asid_free(struct nvmm_cpu *vcpu)
+{
+ size_t oct, bit;
+ uint64_t asid;
+
+ vmx_vmread(VMCS_VPID, &asid);
+
+ oct = asid / 8;
+ bit = asid % 8;
+
+ mutex_enter(&vmx_asidlock);
+ vmx_asidmap[oct] &= ~__BIT(bit);
+ mutex_exit(&vmx_asidlock);
+}
+
+static void
+vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ struct vmcs *vmcs = cpudata->vmcs;
+ struct msr_entry *gmsr = cpudata->gmsr;
+ extern uint8_t vmx_resume_rip;
+ uint64_t rev, eptp;
+
+ rev = vmx_get_revision();
+
+ memset(vmcs, 0, VMCS_SIZE);
+ vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION);
+ vmcs->abort = 0;
+
+ vmx_vmcs_enter(vcpu);
+
+ /* No link pointer. */
+ vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF);
+
+ /* Install the CTLSs. */
+ vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls);
+ vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls);
+ vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2);
+ vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls);
+ vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls);
+
+ /* Allow direct access to certain MSRs. */
+ memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES,
+ true, false);
+ vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa);
+
+ /*
+ * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This
+ * includes the L1D_FLUSH MSR, to mitigate L1TF.
+ */
+ gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR;
+ gmsr[VMX_MSRLIST_STAR].val = 0;
+ gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR;
+ gmsr[VMX_MSRLIST_LSTAR].val = 0;
+ gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR;
+ gmsr[VMX_MSRLIST_CSTAR].val = 0;
+ gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK;
+ gmsr[VMX_MSRLIST_SFMASK].val = 0;
+ gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE;
+ gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0;
+ gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD;
+ gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH;
+ vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa);
+ vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa);
+ vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr);
+ vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR);
+
+ /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */
+ vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET);
+ vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET);
+
+ /* Force CR4_VMXE to zero. */
+ vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE);
+
+ /* Set the Host state for resuming. */
+ vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip);
+ vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0);
+ vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0);
+ vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt);
+ vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT));
+ vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER));
+ vmx_vmwrite(VMCS_HOST_CR0, rcr0());
+
+ /* Generate ASID. */
+ vmx_asid_alloc(vcpu);
+
+ /* Enable Extended Paging, 4-Level. */
+ eptp =
+ __SHIFTIN(vmx_eptp_type, EPTP_TYPE) |
+ __SHIFTIN(4-1, EPTP_WALKLEN) |
+ EPTP_FLAGS_AD |
+ mach->vm->vm_map.pmap->pm_pdirpa[0];
+ vmx_vmwrite(VMCS_EPTP, eptp);
+
+ /* Init IA32_MISC_ENABLE. */
+ cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE);
+ cpudata->gmsr_misc_enable &=
+ ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN);
+ cpudata->gmsr_misc_enable |=
+ (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL);
+
+ /* Must always be set. */
+ vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE);
+ vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE);
+ cpudata->gxcr0 = XCR0_X87;
+
+ /* Init XSAVE header. */
+ cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
+ cpudata->gfpu.xsh_xcomp_bv = 0;
+
+ /* Set guest TSC to zero, more or less. */
+ cpudata->tsc_offset = -cpu_counter();
+
+ /* These MSRs are static. */
+ cpudata->star = rdmsr(MSR_STAR);
+ cpudata->cstar = rdmsr(MSR_CSTAR);
+ cpudata->sfmask = rdmsr(MSR_SFMASK);
+
+ vmx_vmcs_leave(vcpu);
+}
+
+static int
+vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata;
+ int error;
+
+ /* Allocate the VMX cpudata. */
+ cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map,
+ roundup(sizeof(*cpudata), PAGE_SIZE), 0,
+ UVM_KMF_WIRED|UVM_KMF_ZERO);
+ vcpu->cpudata = cpudata;
+
+ /* VMCS */
+ error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs,
+ VMCS_NPAGES);
+ if (error)
+ goto error;
+
+ /* MSR Bitmap */
+ error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
+ MSRBM_NPAGES);
+ if (error)
+ goto error;
+
+ /* Guest MSR List */
+ error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1);
+ if (error)
+ goto error;
+
+ kcpuset_create(&cpudata->htlb_want_flush, true);
+
+ /* Init the VCPU info. */
+ vmx_vcpu_init(mach, vcpu);
+
+ return 0;
+
+error:
+ if (cpudata->vmcs_pa) {
+ vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs,
+ VMCS_NPAGES);
+ }
+ if (cpudata->msrbm_pa) {
+ vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
+ MSRBM_NPAGES);
+ }
+ if (cpudata->gmsr_pa) {
+ vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
+ }
+
+ kmem_free(cpudata, sizeof(*cpudata));
+ return error;
+}
+
+static void
+vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+
+ vmx_vmcs_enter(vcpu);
+ vmx_asid_free(vcpu);
+ vmx_vmcs_leave(vcpu);
+
+ kcpuset_destroy(cpudata->htlb_want_flush);
+
+ vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES);
+ vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
+ vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
+}
+
+/* -------------------------------------------------------------------------- */
+
+static void
vmx_tlb_flush(struct pmap *pm)
{
struct nvmm_machine *mach = pm->pm_data;
@@ -2748,6 +2736,21 @@ vmx_ident(void)
}
static void
+vmx_init_asid(uint32_t maxasid)
+{
+ size_t allocsz;
+
+ mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE);
+
+ vmx_maxasid = maxasid;
+ allocsz = roundup(maxasid, 8) / 8;
+ vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP);
+
+ /* ASID 0 is reserved for the host. */
+ vmx_asidmap[0] |= __BIT(0);
+}
+
+static void
vmx_change_cpu(void *arg1, void *arg2)
{
struct cpu_info *ci = curcpu();