summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--sys/arch/aarch64/aarch64/cpufunc.c19
-rw-r--r--sys/arch/arm/arm32/arm32_boot.c17
-rw-r--r--sys/arch/arm/arm32/cpu.c17
-rw-r--r--sys/arch/mips/mips/cpu_subr.c9
-rw-r--r--sys/arch/x86/x86/cpu_topology.c31
-rw-r--r--sys/kern/init_main.c7
-rw-r--r--sys/kern/kern_cpu.c118
-rw-r--r--sys/sys/cpu.h4
-rw-r--r--sys/sys/cpu_data.h19
9 files changed, 192 insertions, 49 deletions
diff --git a/sys/arch/aarch64/aarch64/cpufunc.c b/sys/arch/aarch64/aarch64/cpufunc.c
index 857f58f2bc6..06b59ee645f 100644
--- a/sys/arch/aarch64/aarch64/cpufunc.c
+++ b/sys/arch/aarch64/aarch64/cpufunc.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $ */
+/* $NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
@@ -29,13 +29,13 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/kmem.h>
+#include <sys/cpu.h>
-#include <aarch64/cpu.h>
#include <aarch64/cpufunc.h>
u_int cputype; /* compat arm */
@@ -93,12 +93,15 @@ aarch64_gettopology(struct cpu_info * const ci, uint64_t mpidr)
{
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
}
diff --git a/sys/arch/arm/arm32/arm32_boot.c b/sys/arch/arm/arm32/arm32_boot.c
index 4693d6810fa..9d498c1ae27 100644
--- a/sys/arch/arm/arm32/arm32_boot.c
+++ b/sys/arch/arm/arm32/arm32_boot.c
@@ -1,4 +1,4 @@
-/* $NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $ */
+/* $NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved.
@@ -122,7 +122,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $");
+__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $");
#include "opt_arm_debug.h"
#include "opt_cputypes.h"
@@ -358,12 +358,15 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in
uint32_t mpidr = armreg_mpidr_read();
ci->ci_mpidr = mpidr;
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
ci->ci_arm_cpuid = cpu_idnum();
diff --git a/sys/arch/arm/arm32/cpu.c b/sys/arch/arm/arm32/cpu.c
index c0d5b1ed584..d79088ba4d3 100644
--- a/sys/arch/arm/arm32/cpu.c
+++ b/sys/arch/arm/arm32/cpu.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $ */
+/* $NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $ */
/*
* Copyright (c) 1995 Mark Brinicombe.
@@ -46,7 +46,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/conf.h>
@@ -111,12 +111,15 @@ cpu_attach(device_t dv, cpuid_t id)
ci->ci_mpidr = mpidr;
if (mpidr & MPIDR_MT) {
- ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF2),
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0));
} else {
- ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0);
- ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1);
+ cpu_topology_set(ci,
+ __SHIFTOUT(mpidr, MPIDR_AFF1),
+ __SHIFTOUT(mpidr, MPIDR_AFF0),
+ 0);
}
#endif
} else {
diff --git a/sys/arch/mips/mips/cpu_subr.c b/sys/arch/mips/mips/cpu_subr.c
index 85fb15b9d9b..cc1b5cf3a2f 100644
--- a/sys/arch/mips/mips/cpu_subr.c
+++ b/sys/arch/mips/mips/cpu_subr.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $ */
+/* $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2010, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $");
#include "opt_cputype.h"
#include "opt_ddb.h"
@@ -182,9 +182,6 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id,
KASSERT(cpu_id != 0);
ci->ci_cpuid = cpu_id;
ci->ci_pmap_kern_segtab = &pmap_kern_segtab,
- ci->ci_package_id = cpu_package_id;
- ci->ci_core_id = cpu_core_id;
- ci->ci_smt_id = cpu_smt_id;
ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq;
ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq;
ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz;
@@ -192,6 +189,8 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id,
ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip;
ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count;
+ cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id);
+
pmap_md_alloc_ephemeral_address_space(ci);
mi_cpu_attach(ci);
diff --git a/sys/arch/x86/x86/cpu_topology.c b/sys/arch/x86/x86/cpu_topology.c
index 0763ccc8bbf..ac817630d81 100644
--- a/sys/arch/x86/x86/cpu_topology.c
+++ b/sys/arch/x86/x86/cpu_topology.c
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $ */
+/* $NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2009 Mindaugas Rasiukevicius <rmind at NetBSD org>,
@@ -36,13 +36,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $");
#include <sys/param.h>
#include <sys/bitops.h>
+#include <sys/cpu.h>
#include <machine/specialreg.h>
-#include <machine/cpu.h>
#include <x86/cpufunc.h>
#include <x86/cputypes.h>
@@ -55,23 +55,28 @@ x86_cpu_topology(struct cpu_info *ci)
u_int core_max; /* Core per package */
int n, cpu_family, apic_id, smt_bits, core_bits = 0;
uint32_t descs[4];
+ int package_id, core_id, smt_id;
apic_id = ci->ci_initapicid;
cpu_family = CPUID_TO_FAMILY(ci->ci_signature);
/* Initial values. */
- ci->ci_package_id = apic_id;
- ci->ci_core_id = 0;
- ci->ci_smt_id = 0;
+ package_id = apic_id;
+ core_id = 0;
+ smt_id = 0;
switch (cpu_vendor) {
case CPUVENDOR_INTEL:
- if (cpu_family < 6)
+ if (cpu_family < 6) {
+ cpu_topology_set(ci, package_id, core_id, smt_id);
return;
+ }
break;
case CPUVENDOR_AMD:
- if (cpu_family < 0xf)
+ if (cpu_family < 0xf) {
+ cpu_topology_set(ci, package_id, core_id, smt_id);
return;
+ }
break;
default:
return;
@@ -164,16 +169,18 @@ x86_cpu_topology(struct cpu_info *ci)
if (smt_bits + core_bits) {
if (smt_bits + core_bits < sizeof(apic_id) * NBBY)
- ci->ci_package_id = apic_id >> (smt_bits + core_bits);
+ package_id = apic_id >> (smt_bits + core_bits);
else
- ci->ci_package_id = 0;
+ package_id = 0;
}
if (core_bits) {
u_int core_mask = __BITS(smt_bits, smt_bits + core_bits - 1);
- ci->ci_core_id = __SHIFTOUT(apic_id, core_mask);
+ core_id = __SHIFTOUT(apic_id, core_mask);
}
if (smt_bits) {
u_int smt_mask = __BITS(0, smt_bits - 1);
- ci->ci_smt_id = __SHIFTOUT(apic_id, smt_mask);
+ smt_id = __SHIFTOUT(apic_id, smt_mask);
}
+
+ cpu_topology_set(ci, package_id, core_id, smt_id);
}
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 28d535bdda4..cf93314864d 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/* $NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $ */
+/* $NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $");
#include "opt_ddb.h"
#include "opt_inet.h"
@@ -788,6 +788,9 @@ configure2(void)
struct cpu_info *ci;
int s;
+ /* Fix up CPU topology info, which has all been collected by now. */
+ cpu_topology_init();
+
/*
* Now that we've found all the hardware, start the real time
* and statistics clocks.
diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c
index 450d2a9a844..fafe364cc9f 100644
--- a/sys/kern/kern_cpu.c
+++ b/sys/kern/kern_cpu.c
@@ -1,7 +1,7 @@
-/* $NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $ */
/*-
- * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $");
#include "opt_cpu_ucode.h"
@@ -119,6 +119,7 @@ kmutex_t cpu_lock __cacheline_aligned;
int ncpu __read_mostly;
int ncpuonline __read_mostly;
bool mp_online __read_mostly;
+static bool cpu_topology_present __read_mostly;
/* An array of CPUs. There are ncpu entries. */
struct cpu_info **cpu_infos __read_mostly;
@@ -587,6 +588,117 @@ cpu_softintr_p(void)
return (curlwp->l_pflag & LP_INTR) != 0;
}
+/*
+ * Collect CPU topology information as each CPU is attached. This can be
+ * called early during boot, so we need to be careful what we do.
+ */
+void
+cpu_topology_set(struct cpu_info *ci, int package_id, int core_id, int smt_id)
+{
+
+ cpu_topology_present = true;
+ ci->ci_package_id = package_id;
+ ci->ci_core_id = core_id;
+ ci->ci_smt_id = smt_id;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_ncore_cpus = 1;
+}
+
+/*
+ * Fake up toplogy info if we have none, or if what we got was bogus.
+ */
+static void
+cpu_topology_fake(void)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ ci->ci_package_id = cpu_index(ci);
+ ci->ci_core_id = 0;
+ ci->ci_smt_id = 0;
+ ci->ci_ncore_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ }
+}
+
+/*
+ * Fix up basic CPU topology info. Right now that means attach each CPU to
+ * circular lists of its siblings in the same core, and in the same package.
+ */
+void
+cpu_topology_init(void)
+{
+ CPU_INFO_ITERATOR cii, cii2;
+ struct cpu_info *ci, *ci2, *ci3;
+
+ if (!cpu_topology_present) {
+ cpu_topology_fake();
+ return;
+ }
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ ci->ci_ncore_cpus = 1;
+ ci->ci_core_cpus = ci;
+ ci->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = 1;
+ }
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ for (CPU_INFO_FOREACH(cii2, ci2)) {
+ /* Avoid bad things happening. */
+ if (ci2->ci_package_id == ci->ci_package_id &&
+ ci2->ci_core_id == ci->ci_core_id &&
+ ci2->ci_smt_id == ci->ci_smt_id &&
+ ci2 != ci) {
+ printf("cpu_topology_init: info bogus, "
+ "faking it\n");
+ cpu_topology_fake();
+ return;
+ }
+ if (ci2 == ci ||
+ ci2->ci_package_id != ci->ci_package_id) {
+ continue;
+ }
+ /*
+ * Find CPUs in the same core. Walk to the end of
+ * the existing circular list and append.
+ */
+ if (ci->ci_ncore_cpus == 1 &&
+ ci->ci_core_id == ci2->ci_core_id) {
+ for (ci3 = ci2;; ci3 = ci3->ci_core_cpus) {
+ ci3->ci_ncore_cpus++;
+ if (ci3->ci_core_cpus == ci2) {
+ break;
+ }
+ }
+ ci->ci_core_cpus = ci2;
+ ci3->ci_core_cpus = ci;
+ ci->ci_ncore_cpus = ci3->ci_ncore_cpus;
+ }
+ /* Same, but for package. */
+ if (ci->ci_npackage_cpus == 1) {
+ for (ci3 = ci2;; ci3 = ci3->ci_package_cpus) {
+ ci3->ci_npackage_cpus++;
+ if (ci3->ci_package_cpus == ci2) {
+ break;
+ }
+ }
+ ci->ci_package_cpus = ci2;
+ ci3->ci_package_cpus = ci;
+ ci->ci_npackage_cpus = ci3->ci_npackage_cpus;
+ }
+ if (ci->ci_ncore_cpus > 1 && ci->ci_npackage_cpus > 1) {
+ break;
+ }
+ }
+ }
+}
+
#ifdef CPU_UCODE
int
cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname)
diff --git a/sys/sys/cpu.h b/sys/sys/cpu.h
index f9818b35768..2494b97b8d0 100644
--- a/sys/sys/cpu.h
+++ b/sys/sys/cpu.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.44 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.45 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2007 YAMAMOTO Takashi,
@@ -90,6 +90,8 @@ bool cpu_kpreempt_disabled(void);
int cpu_lwp_setprivate(struct lwp *, void *);
void cpu_intr_redistribute(void);
u_int cpu_intr_count(struct cpu_info *);
+void cpu_topology_set(struct cpu_info *, int, int, int);
+void cpu_topology_init(void);
#endif
#ifdef _KERNEL
diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h
index cb686273835..6cf81f77066 100644
--- a/sys/sys/cpu_data.h
+++ b/sys/sys/cpu_data.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_data.h,v 1.40 2019/12/01 15:34:47 ad Exp $ */
+/* $NetBSD: cpu_data.h,v 1.41 2019/12/02 23:22:43 ad Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -71,6 +71,15 @@ struct cpu_data {
uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */
struct schedstate_percpu cpu_schedstate; /* scheduler state */
+ /* Basic topology information. May be fake. */
+ cpuid_t cpu_package_id;
+ cpuid_t cpu_core_id;
+ cpuid_t cpu_smt_id;
+ u_int cpu_npackage_cpus;
+ u_int cpu_ncore_cpus;
+ struct cpu_info *cpu_package_cpus; /* sibling CPUs in package */
+ struct cpu_info *cpu_core_cpus; /* sibling CPUs in core */
+
/*
* This section is mostly CPU-private.
*/
@@ -101,9 +110,7 @@ struct cpu_data {
int64_t cpu_cc_skew; /* counter skew vs cpu0 */
char cpu_name[8]; /* eg, "cpu4" */
kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */
- cpuid_t cpu_package_id;
- cpuid_t cpu_core_id;
- cpuid_t cpu_smt_id;
+
struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT];
};
@@ -124,6 +131,10 @@ struct cpu_data {
#define ci_package_id ci_data.cpu_package_id
#define ci_core_id ci_data.cpu_core_id
#define ci_smt_id ci_data.cpu_smt_id
+#define ci_npackage_cpus ci_data.cpu_npackage_cpus
+#define ci_ncore_cpus ci_data.cpu_ncore_cpus
+#define ci_package_cpus ci_data.cpu_package_cpus
+#define ci_core_cpus ci_data.cpu_core_cpus
void mi_cpu_init(void);
int mi_cpu_attach(struct cpu_info *);