summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorad <ad@NetBSD.org>2019-12-02 23:22:43 +0000
committerad <ad@NetBSD.org>2019-12-02 23:22:43 +0000
commitd2caeeabd083fd6954c8fce5e65d064008d4d094 (patch)
tree34fd023ca9e26eec3e5c0f0a1ccaf0e3fa6e3aca
parentcbb526bc2b0c84b14ec0c0d2ba19d000c4eaf422 (diff)
Take the basic CPU topology information we already collect, and use it
to make circular lists of CPU siblings in the same core, and in the same package. Nothing fancy, just enough to have a bit of fun in the scheduler trying out different tactics.
-rw-r--r--sys/arch/aarch64/aarch64/cpufunc.c19
-rw-r--r--sys/arch/arm/arm32/arm32_boot.c17
-rw-r--r--sys/arch/arm/arm32/cpu.c17
-rw-r--r--sys/arch/mips/mips/cpu_subr.c9
-rw-r--r--sys/arch/x86/x86/cpu_topology.c31
-rw-r--r--sys/kern/init_main.c7
-rw-r--r--sys/kern/kern_cpu.c118
-rw-r--r--sys/sys/cpu.h4
-rw-r--r--sys/sys/cpu_data.h19
9 files changed, 192 insertions, 49 deletions
diff --git a/sys/arch/aarch64/aarch64/cpufunc.c b/sys/arch/aarch64/aarch64/cpufunc.c
index 857f58f2bc6..06b59ee645f 100644
--- a/sys/arch/aarch64/aarch64/cpufunc.c
+++ b/sys/arch/aarch64/aarch64/cpufunc.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $ */
+/* $NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
@@ -29,13 +29,13 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/kmem.h>
+#include <sys/cpu.h>
-#include <aarch64/cpu.h>
#include <aarch64/cpufunc.h>
u_int cputype; /* compat arm */
@@ -93,12 +93,15 @@ aarch64_gettopology(struct cpu_info * const ci, uint64_t mpidr)
{
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
}
diff --git a/sys/arch/arm/arm32/arm32_boot.c b/sys/arch/arm/arm32/arm32_boot.c
index 4693d6810fa..9d498c1ae27 100644
--- a/sys/arch/arm/arm32/arm32_boot.c
+++ b/sys/arch/arm/arm32/arm32_boot.c
@@ -1,4 +1,4 @@
-/* $NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $ */
+/* $NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved.
@@ -122,7 +122,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $");
+__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $");
#include "opt_arm_debug.h"
#include "opt_cputypes.h"
@@ -358,12 +358,15 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in
uint32_t mpidr = armreg_mpidr_read();
ci->ci_mpidr = mpidr;
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
ci->ci_arm_cpuid = cpu_idnum();
diff --git a/sys/arch/arm/arm32/cpu.c b/sys/arch/arm/arm32/cpu.c
index c0d5b1ed584..d79088ba4d3 100644
--- a/sys/arch/arm/arm32/cpu.c
+++ b/sys/arch/arm/arm32/cpu.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $ */
+/* $NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 1995 Mark Brinicombe.
@@ -46,7 +46,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/conf.h>
@@ -111,12 +111,15 @@ cpu_attach(device_t dv, cpuid_t id)
ci->ci_mpidr = mpidr;
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
#endif
} else {
diff --git a/sys/arch/mips/mips/cpu_subr.c b/sys/arch/mips/mips/cpu_subr.c
index 85fb15b9d9b..cc1b5cf3a2f 100644
--- a/sys/arch/mips/mips/cpu_subr.c
+++ b/sys/arch/mips/mips/cpu_subr.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $ */
+/* $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2010, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $");
#include "opt_cputype.h"
#include "opt_ddb.h"
@@ -182,9 +182,6 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id,
KASSERT(cpu_id != 0);
ci->ci_cpuid = cpu_id;
ci->ci_pmap_kern_segtab = &pmap_kern_segtab,
- ci->ci_package_id = cpu_package_id;
- ci->ci_core_id = cpu_core_id;
- ci->ci_smt_id = cpu_smt_id;
ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq;
ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq;
ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz;
@@ -192,6 +189,8 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id,
ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip;
ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count;
+ cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id);
+
pmap_md_alloc_ephemeral_address_space(ci);
mi_cpu_attach(ci);
diff --git a/sys/arch/x86/x86/cpu_topology.c b/sys/arch/x86/x86/cpu_topology.c
index 0763ccc8bbf..ac817630d81 100644
--- a/sys/arch/x86/x86/cpu_topology.c
+++ b/sys/arch/x86/x86/cpu_topology.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $ */
+/* $NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2009 Mindaugas Rasiukevicius <rmind at NetBSD org>,
@@ -36,13 +36,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/bitops.h>
+#include <sys/cpu.h>
#include <machine/specialreg.h>
-#include <machine/cpu.h>
#include <x86/cpufunc.h>
#include <x86/cputypes.h>
@@ -55,23 +55,28 @@ x86_cpu_topology(struct cpu_info *ci)
u_int core_max; /* Core per package */
int n, cpu_family, apic_id, smt_bits, core_bits = 0;
uint32_t descs[4];
+ int package_id, core_id, smt_id;
apic_id = ci->ci_initapicid;
cpu_family = CPUID_TO_FAMILY(ci->ci_signature);
/* Initial values. */
- ci->ci_package_id = apic_id;
- ci->ci_core_id = 0;
- ci->ci_smt_id = 0;
+ package_id = apic_id;
+ core_id = 0;
+ smt_id = 0;
switch (cpu_vendor) {
case CPUVENDOR_INTEL:
- if (cpu_family < 6)
+ if (cpu_family < 6) {
+ cpu_topology_set(ci, package_id, core_id, smt_id);
return;
+ }
break;
case CPUVENDOR_AMD:
- if (cpu_family < 0xf)
+ if (cpu_family < 0xf) {
+ cpu_topology_set(ci, package_id, core_id, smt_id);
return;
+ }
break;
default:
return;
@@ -164,16 +169,18 @@ x86_cpu_topology(struct cpu_info *ci)
if (smt_bits + core_bits) {
if (smt_bits + core_bits < sizeof(apic_id) * NBBY)
- ci->ci_package_id = apic_id >> (smt_bits + core_bits);
+ package_id = apic_id >> (smt_bits + core_bits);
else
- ci->ci_package_id = 0;
+ package_id = 0;
}
if (core_bits) {
u_int core_mask = __BITS(smt_bits, smt_bits + core_bits - 1);
- ci->ci_core_id = __SHIFTOUT(apic_id, core_mask);
+ core_id = __SHIFTOUT(apic_id, core_mask);
}
if (smt_bits) {
u_int smt_mask = __BITS(0, smt_bits - 1);
- ci->ci_smt_id = __SHIFTOUT(apic_id, smt_mask);
+ smt_id = __SHIFTOUT(apic_id, smt_mask);
}
+
+ cpu_topology_set(ci, package_id, core_id, smt_id);
}
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 28d535bdda4..cf93314864d 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/* $NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $ */
+/* $NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $");
#include "opt_ddb.h"
#include "opt_inet.h"
@@ -788,6 +788,9 @@ configure2(void)
struct cpu_info *ci;
int s;
+ /* Fix up CPU topology info, which has all been collected by now. */
+ cpu_topology_init();
+
/*
* Now that we've found all the hardware, start the real time
* and statistics clocks.
diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c
index 450d2a9a844..fafe364cc9f 100644
--- a/sys/kern/kern_cpu.c
+++ b/sys/kern/kern_cpu.c
@@ -1,7 +1,7 @@
-/* $NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $ */
/*-
- * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $");
#include "opt_cpu_ucode.h"
@@ -119,6 +119,7 @@ kmutex_t cpu_lock __cacheline_aligned;
int ncpu __read_mostly;
int ncpuonline __read_mostly;
bool mp_online __read_mostly;
+static bool cpu_topology_present __read_mostly;
/* An array of CPUs. There are ncpu entries. */
struct cpu_info **cpu_infos __read_mostly;
@@ -587,6 +588,117 @@ cpu_softintr_p(void)
return (curlwp->l_pflag & LP_INTR) != 0;
}
+/*
+ * Collect CPU topology information as each CPU is attached. This can be
+ * called early during boot, so we need to be careful what we do.
+ */
+void
+cpu_topology_set(struct cpu_info *ci, int package_id, int core_id, int smt_id)
+{
+
+ cpu_topology_present = true;
+ ci->ci_package_id = package_id;
+ ci->ci_core_id = core_id;
+ ci->ci_smt_id = smt_id;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_ncore_cpus = 1;
+}
+
+/*
+ * Fake up toplogy info if we have none, or if what we got was bogus.
+ */
+static void
+cpu_topology_fake(void)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ ci->ci_package_id = cpu_index(ci);
+ ci->ci_core_id = 0;
+ ci->ci_smt_id = 0;
+ ci->ci_ncore_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ }
+}
+
+/*
+ * Fix up basic CPU topology info. Right now that means attach each CPU to
+ * circular lists of its siblings in the same core, and in the same package.
+ */
+void
+cpu_topology_init(void)
+{
+ CPU_INFO_ITERATOR cii, cii2;
+ struct cpu_info *ci, *ci2, *ci3;
+
+ if (!cpu_topology_present) {
+ cpu_topology_fake();
+ return;
+ }
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ ci->ci_ncore_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ }
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ for (CPU_INFO_FOREACH(cii2, ci2)) {
+ /* Avoid bad things happening. */
+ if (ci2->ci_package_id == ci->ci_package_id &&
+ ci2->ci_core_id == ci->ci_core_id &&
+ ci2->ci_smt_id == ci->ci_smt_id &&
+ ci2 != ci) {
+ printf("cpu_topology_init: info bogus, "
+ "faking it\n");
+ cpu_topology_fake();
+ return;
+ }
+ if (ci2 == ci ||
+ ci2->ci_package_id != ci->ci_package_id) {
+ continue;
+ }
+ /*
+ * Find CPUs in the same core. Walk to the end of
+ * the existing circular list and append.
+ */
+ if (ci->ci_ncore_cpus == 1 &&
+ ci->ci_core_id == ci2->ci_core_id) {
+ for (ci3 = ci2;; ci3 = ci3->ci_core_cpus) {
+ ci3->ci_ncore_cpus++;
+ if (ci3->ci_core_cpus == ci2) {
+ break;
+ }
+ }
+ ci->ci_core_cpus = ci2;
+ ci3->ci_core_cpus = ci;
+ ci->ci_ncore_cpus = ci3->ci_ncore_cpus;
+ }
+ /* Same, but for package. */
+ if (ci->ci_npackage_cpus == 1) {
+ for (ci3 = ci2;; ci3 = ci3->ci_package_cpus) {
+ ci3->ci_npackage_cpus++;
+ if (ci3->ci_package_cpus == ci2) {
+ break;
+ }
+ }
+ ci->ci_package_cpus = ci2;
+ ci3->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = ci3->ci_npackage_cpus;
+ }
+ if (ci->ci_ncore_cpus > 1 && ci->ci_npackage_cpus > 1) {
+ break;
+ }
+ }
+ }
+}
+
#ifdef CPU_UCODE
int
cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname)
diff --git a/sys/sys/cpu.h b/sys/sys/cpu.h
index f9818b35768..2494b97b8d0 100644
--- a/sys/sys/cpu.h
+++ b/sys/sys/cpu.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.44 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.45 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2007 YAMAMOTO Takashi,
@@ -90,6 +90,8 @@ bool cpu_kpreempt_disabled(void);
int cpu_lwp_setprivate(struct lwp *, void *);
void cpu_intr_redistribute(void);
u_int cpu_intr_count(struct cpu_info *);
+void cpu_topology_set(struct cpu_info *, int, int, int);
+void cpu_topology_init(void);
#endif
#ifdef _KERNEL
diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h
index cb686273835..6cf81f77066 100644
--- a/sys/sys/cpu_data.h
+++ b/sys/sys/cpu_data.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_data.h,v 1.40 2019/12/01 15:34:47 ad Exp $ */
+/* $NetBSD: cpu_data.h,v 1.41 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -71,6 +71,15 @@ struct cpu_data {
uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */
struct schedstate_percpu cpu_schedstate; /* scheduler state */
+ /* Basic topology information. May be fake. */
+ cpuid_t cpu_package_id;
+ cpuid_t cpu_core_id;
+ cpuid_t cpu_smt_id;
+ u_int cpu_npackage_cpus;
+ u_int cpu_ncore_cpus;
+ struct cpu_info *cpu_package_cpus; /* sibling CPUs in package */
+ struct cpu_info *cpu_core_cpus; /* sibling CPUs in core */
+
/*
* This section is mostly CPU-private.
*/
@@ -101,9 +110,7 @@ struct cpu_data {
int64_t cpu_cc_skew; /* counter skew vs cpu0 */
char cpu_name[8]; /* eg, "cpu4" */
kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */
- cpuid_t cpu_package_id;
- cpuid_t cpu_core_id;
- cpuid_t cpu_smt_id;
+
struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT];
};
@@ -124,6 +131,10 @@ struct cpu_data {
#define ci_package_id ci_data.cpu_package_id
#define ci_core_id ci_data.cpu_core_id
#define ci_smt_id ci_data.cpu_smt_id
+#define ci_npackage_cpus ci_data.cpu_npackage_cpus
+#define ci_ncore_cpus ci_data.cpu_ncore_cpus
+#define ci_package_cpus ci_data.cpu_package_cpus
+#define ci_core_cpus ci_data.cpu_core_cpus
void mi_cpu_init(void);
int mi_cpu_attach(struct cpu_info *);