summaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
authormaxv <maxv@NetBSD.org>2019-05-29 16:54:41 +0000
committermaxv <maxv@NetBSD.org>2019-05-29 16:54:41 +0000
commita237bd4282967c978ed60e7e57bb8bb8fdd72ee1 (patch)
tree870adc3d088a7a29de4f1d6ca9b2258efa279404 /sys
parent6b6e0205bc185e9bbddc453d2be4aa22700e57df (diff)
Add PCID support in SVS. This avoids TLB flushes during kernel<->user
transitions, which greatly reduces the performance penalty introduced by SVS. We use two ASIDs, 0 (kern) and 1 (user), and use invpcid to flush pages in both ASIDs. The read-only machdep.svs.pcid={0,1} sysctl is added, and indicates whether SVS+PCID is in use.
Diffstat (limited to 'sys')
-rw-r--r--sys/arch/amd64/amd64/cpufunc.S31
-rw-r--r--sys/arch/amd64/amd64/genassym.cf10
-rw-r--r--sys/arch/amd64/include/pmap.h3
-rw-r--r--sys/arch/x86/include/cpufunc.h26
-rw-r--r--sys/arch/x86/include/pmap.h5
-rw-r--r--sys/arch/x86/include/specialreg.h9
-rw-r--r--sys/arch/x86/x86/cpu.c10
-rw-r--r--sys/arch/x86/x86/svs.c42
-rw-r--r--sys/arch/x86/x86/x86_machdep.c12
9 files changed, 129 insertions, 19 deletions
diff --git a/sys/arch/amd64/amd64/cpufunc.S b/sys/arch/amd64/amd64/cpufunc.S
index 5a162e5c24a..0461bdee687 100644
--- a/sys/arch/amd64/amd64/cpufunc.S
+++ b/sys/arch/amd64/amd64/cpufunc.S
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.40 2019/05/19 08:17:02 maxv Exp $ */
+/* $NetBSD: cpufunc.S,v 1.41 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -41,6 +41,7 @@
#include <machine/segments.h>
#include "opt_xen.h"
+#include "opt_svs.h"
#include "assym.h"
@@ -65,6 +66,17 @@ END(x86_mfence)
#ifndef XENPV
ENTRY(invlpg)
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ pushq %rdi
+ pushq $PMAP_PCID_USER
+ movq $INVPCID_ADDRESS,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+1: /* FALLTHROUGH */
+#endif
invlpg (%rdi)
ret
END(invlpg)
@@ -112,7 +124,7 @@ END(ltr)
ENTRY(tlbflushg)
movq %cr4, %rax
testq $CR4_PGE, %rax
- jz 1f
+ jz tlbflush
movq %rax, %rdx
andq $~CR4_PGE, %rdx
movq %rdx, %cr4
@@ -121,8 +133,19 @@ ENTRY(tlbflushg)
END(tlbflushg)
ENTRY(tlbflush)
-1:
- movq %cr3, %rax
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ xorq %rax,%rax
+ pushq %rax
+ pushq %rax
+ movq $INVPCID_ALL_NONGLOBAL,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+ ret
+#endif
+1: movq %cr3, %rax
movq %rax, %cr3
ret
END(tlbflush)
diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf
index 9014f0bf63d..168cae5cb8b 100644
--- a/sys/arch/amd64/amd64/genassym.cf
+++ b/sys/arch/amd64/amd64/genassym.cf
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.75 2019/05/27 17:32:36 maxv Exp $
+# $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -276,6 +276,14 @@ define ACPI_SUSPEND_CR3 offsetof(struct cpu_info, ci_suspend_cr3)
define ACPI_SUSPEND_CR4 offsetof(struct cpu_info, ci_suspend_cr4)
define ACPI_SUSPEND_CR8 offsetof(struct cpu_info, ci_suspend_cr8)
+define INVPCID_ADDRESS INVPCID_ADDRESS
+define INVPCID_CONTEXT INVPCID_CONTEXT
+define INVPCID_ALL INVPCID_ALL
+define INVPCID_ALL_NONGLOBAL INVPCID_ALL_NONGLOBAL
+
+define PMAP_PCID_USER PMAP_PCID_USER
+define PMAP_PCID_KERN PMAP_PCID_KERN
+
if NIOAPIC > 0
define PIC_IOAPIC offsetof(struct pic, pic_ioapic)
define IOAPIC_SC_REG offsetof(struct ioapic_softc, sc_reg)
diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h
index c1b2e7fda22..b0b1d921d8e 100644
--- a/sys/arch/amd64/include/pmap.h
+++ b/sys/arch/amd64/include/pmap.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.60 2019/03/09 09:09:56 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.61 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -179,6 +179,7 @@ void svs_lwp_switch(struct lwp *, struct lwp *);
void svs_pdir_switch(struct pmap *);
void svs_init(void);
extern bool svs_enabled;
+extern bool svs_pcid;
#include <x86/pmap.h>
diff --git a/sys/arch/x86/include/cpufunc.h b/sys/arch/x86/include/cpufunc.h
index e562b397100..70f3a794949 100644
--- a/sys/arch/x86/include/cpufunc.h
+++ b/sys/arch/x86/include/cpufunc.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.30 2019/05/11 13:40:26 christos Exp $ */
+/* $NetBSD: cpufunc.h,v 1.31 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
@@ -65,6 +65,30 @@ void invlpg(vaddr_t);
void wbinvd(void);
void breakpoint(void);
+#define INVPCID_ADDRESS 0
+#define INVPCID_CONTEXT 1
+#define INVPCID_ALL 2
+#define INVPCID_ALL_NONGLOBAL 3
+
+static inline void
+invpcid(register_t op, uint64_t pcid, vaddr_t va)
+{
+ struct {
+ uint64_t pcid;
+ uint64_t addr;
+ } desc = {
+ .pcid = pcid,
+ .addr = va
+ };
+
+ asm volatile (
+ "invpcid %[desc],%[op]"
+ :
+ : [desc] "m" (desc), [op] "r" (op)
+ : "memory"
+ );
+}
+
static inline uint64_t
rdtsc(void)
{
diff --git a/sys/arch/x86/include/pmap.h b/sys/arch/x86/include/pmap.h
index 01b86e5c900..bb4f05bb644 100644
--- a/sys/arch/x86/include/pmap.h
+++ b/sys/arch/x86/include/pmap.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.100 2019/03/10 16:30:01 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -210,6 +210,9 @@ struct pcpu_area {
extern struct pcpu_area *pcpuarea;
+#define PMAP_PCID_KERN 0
+#define PMAP_PCID_USER 1
+
/*
* pmap data structures: see pmap.c for details of locking.
*/
diff --git a/sys/arch/x86/include/specialreg.h b/sys/arch/x86/include/specialreg.h
index b3a1b9f06f8..4f8c4cca6db 100644
--- a/sys/arch/x86/include/specialreg.h
+++ b/sys/arch/x86/include/specialreg.h
@@ -1,4 +1,4 @@
-/* $NetBSD: specialreg.h,v 1.146 2019/05/18 08:17:39 maxv Exp $ */
+/* $NetBSD: specialreg.h,v 1.147 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2014-2019 The NetBSD Foundation, Inc.
@@ -88,6 +88,13 @@
#define CCR1_RPL 0x01 /* enables RPLSET and RPLVAL# pins */
/*
+ * CR3
+ */
+#define CR3_PCID __BITS(11,0)
+#define CR3_PA __BITS(62,12)
+#define CR3_NO_TLB_FLUSH __BIT(63)
+
+/*
* CR4
*/
#define CR4_VME 0x00000001 /* virtual 8086 mode extension enable */
diff --git a/sys/arch/x86/x86/cpu.c b/sys/arch/x86/x86/cpu.c
index dc8ff2fb490..829021c4b93 100644
--- a/sys/arch/x86/x86/cpu.c
+++ b/sys/arch/x86/x86/cpu.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -613,6 +613,12 @@ cpu_init(struct cpu_info *ci)
if (cpu_feature[5] & CPUID_SEF_SMAP)
cr4 |= CR4_SMAP;
+#ifdef SVS
+ /* If PCID is supported, enable it */
+ if (svs_pcid)
+ cr4 |= CR4_PCIDE;
+#endif
+
if (cr4) {
cr4 |= rcr4();
lcr4(cr4);
diff --git a/sys/arch/x86/x86/svs.c b/sys/arch/x86/x86/svs.c
index a4f74b6e25a..2eb43bcb09c 100644
--- a/sys/arch/x86/x86/svs.c
+++ b/sys/arch/x86/x86/svs.c
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $");
#include "opt_svs.h"
@@ -228,6 +228,10 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
*/
bool svs_enabled __read_mostly = false;
+bool svs_pcid __read_mostly = false;
+
+static uint64_t svs_pcid_kcr3 __read_mostly;
+static uint64_t svs_pcid_ucr3 __read_mostly;
struct svs_utls {
paddr_t kpdirpa;
@@ -405,6 +409,19 @@ svs_utls_init(struct cpu_info *ci)
}
static void
+svs_pcid_init(struct cpu_info *ci)
+{
+ if (!svs_pcid) {
+ return;
+ }
+
+ svs_pcid_ucr3 = __SHIFTIN(PMAP_PCID_USER, CR3_PCID) | CR3_NO_TLB_FLUSH;
+ svs_pcid_kcr3 = __SHIFTIN(PMAP_PCID_KERN, CR3_PCID) | CR3_NO_TLB_FLUSH;
+
+ ci->ci_svs_updirpa |= svs_pcid_ucr3;
+}
+
+static void
svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global)
{
size_t i, n;
@@ -454,6 +471,8 @@ cpu_svs_init(struct cpu_info *ci)
svs_rsp0_init(ci);
svs_utls_init(ci);
+
+ svs_pcid_init(ci);
}
void
@@ -523,11 +542,14 @@ svs_lwp_switch(struct lwp *oldlwp, struct lwp *newlwp)
utls->scratch = 0;
/*
- * Enter the user rsp0. We don't need to flush the TLB here, since
- * the user page tables are not loaded.
+ * Enter the user rsp0. If we're using PCID we must flush the user VA,
+ * if we aren't it will be flushed during the next CR3 reload.
*/
pte = ci->ci_svs_rsp0_pte;
*pte = L1_BASE[pl1_i(va)];
+ if (svs_pcid) {
+ invpcid(INVPCID_ADDRESS, PMAP_PCID_USER, ci->ci_svs_rsp0);
+ }
}
static inline pt_entry_t
@@ -558,7 +580,7 @@ svs_pdir_switch(struct pmap *pmap)
/* Update the info in the UTLS page */
utls = (struct svs_utls *)ci->ci_svs_utls;
- utls->kpdirpa = pmap_pdirpa(pmap, 0);
+ utls->kpdirpa = pmap_pdirpa(pmap, 0) | svs_pcid_kcr3;
mutex_enter(&ci->ci_svs_mtx);
@@ -569,6 +591,10 @@ svs_pdir_switch(struct pmap *pmap)
}
mutex_exit(&ci->ci_svs_mtx);
+
+ if (svs_pcid) {
+ invpcid(INVPCID_CONTEXT, PMAP_PCID_USER, 0);
+ }
}
static void
@@ -641,5 +667,11 @@ svs_init(void)
}
}
+ if ((cpu_info_primary.ci_feat_val[1] & CPUID2_PCID) &&
+ (cpu_info_primary.ci_feat_val[5] & CPUID_SEF_INVPCID)) {
+ svs_pcid = true;
+ lcr4(rcr4() | CR4_PCIDE);
+ }
+
svs_enable();
}
diff --git a/sys/arch/x86/x86/x86_machdep.c b/sys/arch/x86/x86/x86_machdep.c
index 9a8cf613948..3229849d254 100644
--- a/sys/arch/x86/x86/x86_machdep.c
+++ b/sys/arch/x86/x86/x86_machdep.c
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1293,12 +1293,18 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
CTLTYPE_NODE, "svs", NULL,
NULL, 0, NULL, 0,
CTL_MACHDEP, CTL_CREATE);
- sysctl_createv(clog, 0, &svs_rnode, &svs_rnode,
+ sysctl_createv(clog, 0, &svs_rnode, NULL,
CTLFLAG_PERMANENT,
CTLTYPE_BOOL, "enabled",
SYSCTL_DESCR("Whether the kernel uses SVS"),
NULL, 0, &svs_enabled, 0,
CTL_CREATE, CTL_EOL);
+ sysctl_createv(clog, 0, &svs_rnode, NULL,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_BOOL, "pcid",
+ SYSCTL_DESCR("Whether SVS uses PCID"),
+ NULL, 0, &svs_pcid, 0,
+ CTL_CREATE, CTL_EOL);
#endif
#ifndef XEN