diff options
| author | maxv <maxv@NetBSD.org> | 2019-05-29 16:54:41 +0000 |
|---|---|---|
| committer | maxv <maxv@NetBSD.org> | 2019-05-29 16:54:41 +0000 |
| commit | a237bd4282967c978ed60e7e57bb8bb8fdd72ee1 (patch) | |
| tree | 870adc3d088a7a29de4f1d6ca9b2258efa279404 /sys | |
| parent | 6b6e0205bc185e9bbddc453d2be4aa22700e57df (diff) | |
Add PCID support in SVS. This avoids TLB flushes during kernel<->user
transitions, which greatly reduces the performance penalty introduced by
SVS.
We use two ASIDs, 0 (kern) and 1 (user), and use invpcid to flush pages
in both ASIDs.
The read-only machdep.svs.pcid={0,1} sysctl is added, and indicates whether
SVS+PCID is in use.
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/arch/amd64/amd64/cpufunc.S | 31 | ||||
| -rw-r--r-- | sys/arch/amd64/amd64/genassym.cf | 10 | ||||
| -rw-r--r-- | sys/arch/amd64/include/pmap.h | 3 | ||||
| -rw-r--r-- | sys/arch/x86/include/cpufunc.h | 26 | ||||
| -rw-r--r-- | sys/arch/x86/include/pmap.h | 5 | ||||
| -rw-r--r-- | sys/arch/x86/include/specialreg.h | 9 | ||||
| -rw-r--r-- | sys/arch/x86/x86/cpu.c | 10 | ||||
| -rw-r--r-- | sys/arch/x86/x86/svs.c | 42 | ||||
| -rw-r--r-- | sys/arch/x86/x86/x86_machdep.c | 12 |
9 files changed, 129 insertions, 19 deletions
diff --git a/sys/arch/amd64/amd64/cpufunc.S b/sys/arch/amd64/amd64/cpufunc.S index 5a162e5c24a..0461bdee687 100644 --- a/sys/arch/amd64/amd64/cpufunc.S +++ b/sys/arch/amd64/amd64/cpufunc.S @@ -1,4 +1,4 @@ -/* $NetBSD: cpufunc.S,v 1.40 2019/05/19 08:17:02 maxv Exp $ */ +/* $NetBSD: cpufunc.S,v 1.41 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc. @@ -41,6 +41,7 @@ #include <machine/segments.h> #include "opt_xen.h" +#include "opt_svs.h" #include "assym.h" @@ -65,6 +66,17 @@ END(x86_mfence) #ifndef XENPV ENTRY(invlpg) +#ifdef SVS + movb _C_LABEL(svs_pcid),%al + testb %al,%al + jz 1f + pushq %rdi + pushq $PMAP_PCID_USER + movq $INVPCID_ADDRESS,%rax + invpcid (%rsp),%rax + addq $16,%rsp +1: /* FALLTHROUGH */ +#endif invlpg (%rdi) ret END(invlpg) @@ -112,7 +124,7 @@ END(ltr) ENTRY(tlbflushg) movq %cr4, %rax testq $CR4_PGE, %rax - jz 1f + jz tlbflush movq %rax, %rdx andq $~CR4_PGE, %rdx movq %rdx, %cr4 @@ -121,8 +133,19 @@ ENTRY(tlbflushg) END(tlbflushg) ENTRY(tlbflush) -1: - movq %cr3, %rax +#ifdef SVS + movb _C_LABEL(svs_pcid),%al + testb %al,%al + jz 1f + xorq %rax,%rax + pushq %rax + pushq %rax + movq $INVPCID_ALL_NONGLOBAL,%rax + invpcid (%rsp),%rax + addq $16,%rsp + ret +#endif +1: movq %cr3, %rax movq %rax, %cr3 ret END(tlbflush) diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf index 9014f0bf63d..168cae5cb8b 100644 --- a/sys/arch/amd64/amd64/genassym.cf +++ b/sys/arch/amd64/amd64/genassym.cf @@ -1,4 +1,4 @@ -# $NetBSD: genassym.cf,v 1.75 2019/05/27 17:32:36 maxv Exp $ +# $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $ # # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -276,6 +276,14 @@ define ACPI_SUSPEND_CR3 offsetof(struct cpu_info, ci_suspend_cr3) define ACPI_SUSPEND_CR4 offsetof(struct cpu_info, ci_suspend_cr4) define ACPI_SUSPEND_CR8 offsetof(struct cpu_info, ci_suspend_cr8) +define INVPCID_ADDRESS INVPCID_ADDRESS +define INVPCID_CONTEXT INVPCID_CONTEXT +define INVPCID_ALL INVPCID_ALL +define INVPCID_ALL_NONGLOBAL INVPCID_ALL_NONGLOBAL + +define PMAP_PCID_USER PMAP_PCID_USER +define PMAP_PCID_KERN PMAP_PCID_KERN + if NIOAPIC > 0 define PIC_IOAPIC offsetof(struct pic, pic_ioapic) define IOAPIC_SC_REG offsetof(struct ioapic_softc, sc_reg) diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h index c1b2e7fda22..b0b1d921d8e 100644 --- a/sys/arch/amd64/include/pmap.h +++ b/sys/arch/amd64/include/pmap.h @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.h,v 1.60 2019/03/09 09:09:56 maxv Exp $ */ +/* $NetBSD: pmap.h,v 1.61 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -179,6 +179,7 @@ void svs_lwp_switch(struct lwp *, struct lwp *); void svs_pdir_switch(struct pmap *); void svs_init(void); extern bool svs_enabled; +extern bool svs_pcid; #include <x86/pmap.h> diff --git a/sys/arch/x86/include/cpufunc.h b/sys/arch/x86/include/cpufunc.h index e562b397100..70f3a794949 100644 --- a/sys/arch/x86/include/cpufunc.h +++ b/sys/arch/x86/include/cpufunc.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpufunc.h,v 1.30 2019/05/11 13:40:26 christos Exp $ */ +/* $NetBSD: cpufunc.h,v 1.31 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc. @@ -65,6 +65,30 @@ void invlpg(vaddr_t); void wbinvd(void); void breakpoint(void); +#define INVPCID_ADDRESS 0 +#define INVPCID_CONTEXT 1 +#define INVPCID_ALL 2 +#define INVPCID_ALL_NONGLOBAL 3 + +static inline void +invpcid(register_t op, uint64_t pcid, vaddr_t va) +{ + struct { + uint64_t pcid; + uint64_t addr; + } desc = { + .pcid = pcid, + .addr = va + }; + + asm volatile ( + "invpcid %[desc],%[op]" + : + : [desc] "m" (desc), [op] "r" (op) + : "memory" + ); +} + static inline uint64_t rdtsc(void) { diff --git a/sys/arch/x86/include/pmap.h b/sys/arch/x86/include/pmap.h index 01b86e5c900..bb4f05bb644 100644 --- a/sys/arch/x86/include/pmap.h +++ b/sys/arch/x86/include/pmap.h @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.h,v 1.100 2019/03/10 16:30:01 maxv Exp $ */ +/* $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -210,6 +210,9 @@ struct pcpu_area { extern struct pcpu_area *pcpuarea; +#define PMAP_PCID_KERN 0 +#define PMAP_PCID_USER 1 + /* * pmap data structures: see pmap.c for details of locking. */ diff --git a/sys/arch/x86/include/specialreg.h b/sys/arch/x86/include/specialreg.h index b3a1b9f06f8..4f8c4cca6db 100644 --- a/sys/arch/x86/include/specialreg.h +++ b/sys/arch/x86/include/specialreg.h @@ -1,4 +1,4 @@ -/* $NetBSD: specialreg.h,v 1.146 2019/05/18 08:17:39 maxv Exp $ */ +/* $NetBSD: specialreg.h,v 1.147 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 2014-2019 The NetBSD Foundation, Inc. @@ -88,6 +88,13 @@ #define CCR1_RPL 0x01 /* enables RPLSET and RPLVAL# pins */ /* + * CR3 + */ +#define CR3_PCID __BITS(11,0) +#define CR3_PA __BITS(62,12) +#define CR3_NO_TLB_FLUSH __BIT(63) + +/* * CR4 */ #define CR4_VME 0x00000001 /* virtual 8086 mode extension enable */ diff --git a/sys/arch/x86/x86/cpu.c b/sys/arch/x86/x86/cpu.c index dc8ff2fb490..829021c4b93 100644 --- a/sys/arch/x86/x86/cpu.c +++ b/sys/arch/x86/x86/cpu.c @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $ */ +/* $NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 2000-2012 NetBSD Foundation, Inc. @@ -62,7 +62,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $"); #include "opt_ddb.h" #include "opt_mpbios.h" /* for MPDEBUG */ @@ -613,6 +613,12 @@ cpu_init(struct cpu_info *ci) if (cpu_feature[5] & CPUID_SEF_SMAP) cr4 |= CR4_SMAP; +#ifdef SVS + /* If PCID is supported, enable it */ + if (svs_pcid) + cr4 |= CR4_PCIDE; +#endif + if (cr4) { cr4 |= rcr4(); lcr4(cr4); diff --git a/sys/arch/x86/x86/svs.c b/sys/arch/x86/x86/svs.c index a4f74b6e25a..2eb43bcb09c 100644 --- a/sys/arch/x86/x86/svs.c +++ b/sys/arch/x86/x86/svs.c @@ -1,4 +1,4 @@ -/* $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $ */ +/* $NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $ */ /* * Copyright (c) 2018-2019 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $"); #include "opt_svs.h" @@ -228,6 +228,10 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $"); */ bool svs_enabled __read_mostly = false; +bool svs_pcid __read_mostly = false; + +static uint64_t svs_pcid_kcr3 __read_mostly; +static uint64_t svs_pcid_ucr3 __read_mostly; struct svs_utls { paddr_t kpdirpa; @@ -405,6 +409,19 @@ svs_utls_init(struct cpu_info *ci) } static void +svs_pcid_init(struct cpu_info *ci) +{ + if (!svs_pcid) { + return; + } + + svs_pcid_ucr3 = __SHIFTIN(PMAP_PCID_USER, CR3_PCID) | CR3_NO_TLB_FLUSH; + svs_pcid_kcr3 = __SHIFTIN(PMAP_PCID_KERN, CR3_PCID) | CR3_NO_TLB_FLUSH; + + ci->ci_svs_updirpa |= svs_pcid_ucr3; +} + +static void svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global) { size_t i, n; @@ -454,6 +471,8 @@ cpu_svs_init(struct cpu_info *ci) svs_rsp0_init(ci); svs_utls_init(ci); + + svs_pcid_init(ci); } void @@ -523,11 +542,14 @@ svs_lwp_switch(struct lwp *oldlwp, struct lwp *newlwp) utls->scratch = 0; /* - * Enter the user rsp0. We don't need to flush the TLB here, since - * the user page tables are not loaded. + * Enter the user rsp0. If we're using PCID we must flush the user VA, + * if we aren't it will be flushed during the next CR3 reload. */ pte = ci->ci_svs_rsp0_pte; *pte = L1_BASE[pl1_i(va)]; + if (svs_pcid) { + invpcid(INVPCID_ADDRESS, PMAP_PCID_USER, ci->ci_svs_rsp0); + } } static inline pt_entry_t @@ -558,7 +580,7 @@ svs_pdir_switch(struct pmap *pmap) /* Update the info in the UTLS page */ utls = (struct svs_utls *)ci->ci_svs_utls; - utls->kpdirpa = pmap_pdirpa(pmap, 0); + utls->kpdirpa = pmap_pdirpa(pmap, 0) | svs_pcid_kcr3; mutex_enter(&ci->ci_svs_mtx); @@ -569,6 +591,10 @@ svs_pdir_switch(struct pmap *pmap) } mutex_exit(&ci->ci_svs_mtx); + + if (svs_pcid) { + invpcid(INVPCID_CONTEXT, PMAP_PCID_USER, 0); + } } static void @@ -641,5 +667,11 @@ svs_init(void) } } + if ((cpu_info_primary.ci_feat_val[1] & CPUID2_PCID) && + (cpu_info_primary.ci_feat_val[5] & CPUID_SEF_INVPCID)) { + svs_pcid = true; + lcr4(rcr4() | CR4_PCIDE); + } + svs_enable(); } diff --git a/sys/arch/x86/x86/x86_machdep.c b/sys/arch/x86/x86/x86_machdep.c index 9a8cf613948..3229849d254 100644 --- a/sys/arch/x86/x86/x86_machdep.c +++ b/sys/arch/x86/x86/x86_machdep.c @@ -1,4 +1,4 @@ -/* $NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $ */ +/* $NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $"); #include "opt_modular.h" #include "opt_physmem.h" @@ -1293,12 +1293,18 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup") CTLTYPE_NODE, "svs", NULL, NULL, 0, NULL, 0, CTL_MACHDEP, CTL_CREATE); - sysctl_createv(clog, 0, &svs_rnode, &svs_rnode, + sysctl_createv(clog, 0, &svs_rnode, NULL, CTLFLAG_PERMANENT, CTLTYPE_BOOL, "enabled", SYSCTL_DESCR("Whether the kernel uses SVS"), NULL, 0, &svs_enabled, 0, CTL_CREATE, CTL_EOL); + sysctl_createv(clog, 0, &svs_rnode, NULL, + CTLFLAG_PERMANENT, + CTLTYPE_BOOL, "pcid", + SYSCTL_DESCR("Whether SVS uses PCID"), + NULL, 0, &svs_pcid, 0, + CTL_CREATE, CTL_EOL); #endif #ifndef XEN |
