diff options
| -rw-r--r-- | sys/arch/aarch64/aarch64/cpufunc.c | 19 | ||||
| -rw-r--r-- | sys/arch/arm/arm32/arm32_boot.c | 17 | ||||
| -rw-r--r-- | sys/arch/arm/arm32/cpu.c | 17 | ||||
| -rw-r--r-- | sys/arch/mips/mips/cpu_subr.c | 9 | ||||
| -rw-r--r-- | sys/arch/x86/x86/cpu_topology.c | 31 | ||||
| -rw-r--r-- | sys/kern/init_main.c | 7 | ||||
| -rw-r--r-- | sys/kern/kern_cpu.c | 118 | ||||
| -rw-r--r-- | sys/sys/cpu.h | 4 | ||||
| -rw-r--r-- | sys/sys/cpu_data.h | 19 |
9 files changed, 192 insertions, 49 deletions
diff --git a/sys/arch/aarch64/aarch64/cpufunc.c b/sys/arch/aarch64/aarch64/cpufunc.c index 857f58f2bc6..06b59ee645f 100644 --- a/sys/arch/aarch64/aarch64/cpufunc.c +++ b/sys/arch/aarch64/aarch64/cpufunc.c @@ -1,4 +1,4 @@ -/* $NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $ */ +/* $NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> @@ -29,13 +29,13 @@ #include "opt_multiprocessor.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/types.h> #include <sys/kmem.h> +#include <sys/cpu.h> -#include <aarch64/cpu.h> #include <aarch64/cpufunc.h> u_int cputype; /* compat arm */ @@ -93,12 +93,15 @@ aarch64_gettopology(struct cpu_info * const ci, uint64_t mpidr) { if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } } diff --git a/sys/arch/arm/arm32/arm32_boot.c b/sys/arch/arm/arm32/arm32_boot.c index 4693d6810fa..9d498c1ae27 100644 --- a/sys/arch/arm/arm32/arm32_boot.c +++ b/sys/arch/arm/arm32/arm32_boot.c @@ -1,4 +1,4 @@ -/* $NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $ */ +/* $NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. @@ -122,7 +122,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $"); +__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $"); #include "opt_arm_debug.h" #include "opt_cputypes.h" @@ -358,12 +358,15 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in uint32_t mpidr = armreg_mpidr_read(); ci->ci_mpidr = mpidr; if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } ci->ci_arm_cpuid = cpu_idnum(); diff --git a/sys/arch/arm/arm32/cpu.c b/sys/arch/arm/arm32/cpu.c index c0d5b1ed584..d79088ba4d3 100644 --- a/sys/arch/arm/arm32/cpu.c +++ b/sys/arch/arm/arm32/cpu.c @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $ */ +/* $NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 1995 Mark Brinicombe. @@ -46,7 +46,7 @@ #include "opt_multiprocessor.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/conf.h> @@ -111,12 +111,15 @@ cpu_attach(device_t dv, cpuid_t id) ci->ci_mpidr = mpidr; if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } #endif } else { diff --git a/sys/arch/mips/mips/cpu_subr.c b/sys/arch/mips/mips/cpu_subr.c index 85fb15b9d9b..cc1b5cf3a2f 100644 --- a/sys/arch/mips/mips/cpu_subr.c +++ b/sys/arch/mips/mips/cpu_subr.c @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $ */ +/* $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2010, 2019 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $"); #include "opt_cputype.h" #include "opt_ddb.h" @@ -182,9 +182,6 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id, KASSERT(cpu_id != 0); ci->ci_cpuid = cpu_id; ci->ci_pmap_kern_segtab = &pmap_kern_segtab, - ci->ci_package_id = cpu_package_id; - ci->ci_core_id = cpu_core_id; - ci->ci_smt_id = cpu_smt_id; ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq; ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq; ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz; @@ -192,6 +189,8 @@ cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id, ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip; ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count; + cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id); + pmap_md_alloc_ephemeral_address_space(ci); mi_cpu_attach(ci); diff --git a/sys/arch/x86/x86/cpu_topology.c b/sys/arch/x86/x86/cpu_topology.c index 0763ccc8bbf..ac817630d81 100644 --- a/sys/arch/x86/x86/cpu_topology.c +++ b/sys/arch/x86/x86/cpu_topology.c @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $ */ +/* $NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2009 Mindaugas Rasiukevicius <rmind at NetBSD org>, @@ -36,13 +36,13 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/bitops.h> +#include <sys/cpu.h> #include <machine/specialreg.h> -#include <machine/cpu.h> #include <x86/cpufunc.h> #include <x86/cputypes.h> @@ -55,23 +55,28 @@ x86_cpu_topology(struct cpu_info *ci) u_int core_max; /* Core per package */ int n, cpu_family, apic_id, smt_bits, core_bits = 0; uint32_t descs[4]; + int package_id, core_id, smt_id; apic_id = ci->ci_initapicid; cpu_family = CPUID_TO_FAMILY(ci->ci_signature); /* Initial values. */ - ci->ci_package_id = apic_id; - ci->ci_core_id = 0; - ci->ci_smt_id = 0; + package_id = apic_id; + core_id = 0; + smt_id = 0; switch (cpu_vendor) { case CPUVENDOR_INTEL: - if (cpu_family < 6) + if (cpu_family < 6) { + cpu_topology_set(ci, package_id, core_id, smt_id); return; + } break; case CPUVENDOR_AMD: - if (cpu_family < 0xf) + if (cpu_family < 0xf) { + cpu_topology_set(ci, package_id, core_id, smt_id); return; + } break; default: return; @@ -164,16 +169,18 @@ x86_cpu_topology(struct cpu_info *ci) if (smt_bits + core_bits) { if (smt_bits + core_bits < sizeof(apic_id) * NBBY) - ci->ci_package_id = apic_id >> (smt_bits + core_bits); + package_id = apic_id >> (smt_bits + core_bits); else - ci->ci_package_id = 0; + package_id = 0; } if (core_bits) { u_int core_mask = __BITS(smt_bits, smt_bits + core_bits - 1); - ci->ci_core_id = __SHIFTOUT(apic_id, core_mask); + core_id = __SHIFTOUT(apic_id, core_mask); } if (smt_bits) { u_int smt_mask = __BITS(0, smt_bits - 1); - ci->ci_smt_id = __SHIFTOUT(apic_id, smt_mask); + smt_id = __SHIFTOUT(apic_id, smt_mask); } + + cpu_topology_set(ci, package_id, core_id, smt_id); } diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 28d535bdda4..cf93314864d 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -1,4 +1,4 @@ -/* $NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $ */ +/* $NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc. @@ -97,7 +97,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $"); #include "opt_ddb.h" #include "opt_inet.h" @@ -788,6 +788,9 @@ configure2(void) struct cpu_info *ci; int s; + /* Fix up CPU topology info, which has all been collected by now. */ + cpu_topology_init(); + /* * Now that we've found all the hardware, start the real time * and statistics clocks. diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c index 450d2a9a844..fafe364cc9f 100644 --- a/sys/kern/kern_cpu.c +++ b/sys/kern/kern_cpu.c @@ -1,7 +1,7 @@ -/* $NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $ */ +/* $NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $ */ /*- - * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. + * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -56,7 +56,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $"); #include "opt_cpu_ucode.h" @@ -119,6 +119,7 @@ kmutex_t cpu_lock __cacheline_aligned; int ncpu __read_mostly; int ncpuonline __read_mostly; bool mp_online __read_mostly; +static bool cpu_topology_present __read_mostly; /* An array of CPUs. There are ncpu entries. */ struct cpu_info **cpu_infos __read_mostly; @@ -587,6 +588,117 @@ cpu_softintr_p(void) return (curlwp->l_pflag & LP_INTR) != 0; } +/* + * Collect CPU topology information as each CPU is attached. This can be + * called early during boot, so we need to be careful what we do. + */ +void +cpu_topology_set(struct cpu_info *ci, int package_id, int core_id, int smt_id) +{ + + cpu_topology_present = true; + ci->ci_package_id = package_id; + ci->ci_core_id = core_id; + ci->ci_smt_id = smt_id; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_ncore_cpus = 1; +} + +/* + * Fake up toplogy info if we have none, or if what we got was bogus. + */ +static void +cpu_topology_fake(void) +{ + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + for (CPU_INFO_FOREACH(cii, ci)) { + ci->ci_package_id = cpu_index(ci); + ci->ci_core_id = 0; + ci->ci_smt_id = 0; + ci->ci_ncore_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + } +} + +/* + * Fix up basic CPU topology info. Right now that means attach each CPU to + * circular lists of its siblings in the same core, and in the same package. + */ +void +cpu_topology_init(void) +{ + CPU_INFO_ITERATOR cii, cii2; + struct cpu_info *ci, *ci2, *ci3; + + if (!cpu_topology_present) { + cpu_topology_fake(); + return; + } + + for (CPU_INFO_FOREACH(cii, ci)) { + ci->ci_ncore_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + } + + for (CPU_INFO_FOREACH(cii, ci)) { + for (CPU_INFO_FOREACH(cii2, ci2)) { + /* Avoid bad things happening. */ + if (ci2->ci_package_id == ci->ci_package_id && + ci2->ci_core_id == ci->ci_core_id && + ci2->ci_smt_id == ci->ci_smt_id && + ci2 != ci) { + printf("cpu_topology_init: info bogus, " + "faking it\n"); + cpu_topology_fake(); + return; + } + if (ci2 == ci || + ci2->ci_package_id != ci->ci_package_id) { + continue; + } + /* + * Find CPUs in the same core. Walk to the end of + * the existing circular list and append. + */ + if (ci->ci_ncore_cpus == 1 && + ci->ci_core_id == ci2->ci_core_id) { + for (ci3 = ci2;; ci3 = ci3->ci_core_cpus) { + ci3->ci_ncore_cpus++; + if (ci3->ci_core_cpus == ci2) { + break; + } + } + ci->ci_core_cpus = ci2; + ci3->ci_core_cpus = ci; + ci->ci_ncore_cpus = ci3->ci_ncore_cpus; + } + /* Same, but for package. */ + if (ci->ci_npackage_cpus == 1) { + for (ci3 = ci2;; ci3 = ci3->ci_package_cpus) { + ci3->ci_npackage_cpus++; + if (ci3->ci_package_cpus == ci2) { + break; + } + } + ci->ci_package_cpus = ci2; + ci3->ci_package_cpus = ci; + ci->ci_npackage_cpus = ci3->ci_npackage_cpus; + } + if (ci->ci_ncore_cpus > 1 && ci->ci_npackage_cpus > 1) { + break; + } + } + } +} + #ifdef CPU_UCODE int cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) diff --git a/sys/sys/cpu.h b/sys/sys/cpu.h index f9818b35768..2494b97b8d0 100644 --- a/sys/sys/cpu.h +++ b/sys/sys/cpu.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.h,v 1.44 2019/11/23 19:42:52 ad Exp $ */ +/* $NetBSD: cpu.h,v 1.45 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2007 YAMAMOTO Takashi, @@ -90,6 +90,8 @@ bool cpu_kpreempt_disabled(void); int cpu_lwp_setprivate(struct lwp *, void *); void cpu_intr_redistribute(void); u_int cpu_intr_count(struct cpu_info *); +void cpu_topology_set(struct cpu_info *, int, int, int); +void cpu_topology_init(void); #endif #ifdef _KERNEL diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h index cb686273835..6cf81f77066 100644 --- a/sys/sys/cpu_data.h +++ b/sys/sys/cpu_data.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_data.h,v 1.40 2019/12/01 15:34:47 ad Exp $ */ +/* $NetBSD: cpu_data.h,v 1.41 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc. @@ -71,6 +71,15 @@ struct cpu_data { uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */ struct schedstate_percpu cpu_schedstate; /* scheduler state */ + /* Basic topology information. May be fake. */ + cpuid_t cpu_package_id; + cpuid_t cpu_core_id; + cpuid_t cpu_smt_id; + u_int cpu_npackage_cpus; + u_int cpu_ncore_cpus; + struct cpu_info *cpu_package_cpus; /* sibling CPUs in package */ + struct cpu_info *cpu_core_cpus; /* sibling CPUs in core */ + /* * This section is mostly CPU-private. */ @@ -101,9 +110,7 @@ struct cpu_data { int64_t cpu_cc_skew; /* counter skew vs cpu0 */ char cpu_name[8]; /* eg, "cpu4" */ kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */ - cpuid_t cpu_package_id; - cpuid_t cpu_core_id; - cpuid_t cpu_smt_id; + struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT]; }; @@ -124,6 +131,10 @@ struct cpu_data { #define ci_package_id ci_data.cpu_package_id #define ci_core_id ci_data.cpu_core_id #define ci_smt_id ci_data.cpu_smt_id +#define ci_npackage_cpus ci_data.cpu_npackage_cpus +#define ci_ncore_cpus ci_data.cpu_ncore_cpus +#define ci_package_cpus ci_data.cpu_package_cpus +#define ci_core_cpus ci_data.cpu_core_cpus void mi_cpu_init(void); int mi_cpu_attach(struct cpu_info *); |
