diff options
| -rw-r--r-- | share/man/man9/heartbeat.9 | 169 | ||||
| -rw-r--r-- | sys/kern/files.kern | 5 | ||||
| -rw-r--r-- | sys/kern/init_main.c | 14 | ||||
| -rw-r--r-- | sys/kern/kern_clock.c | 13 | ||||
| -rw-r--r-- | sys/kern/kern_cpu.c | 14 | ||||
| -rw-r--r-- | sys/kern/kern_heartbeat.c | 656 | ||||
| -rw-r--r-- | sys/sys/cpu_data.h | 11 | ||||
| -rw-r--r-- | sys/sys/heartbeat.h | 53 |
8 files changed, 927 insertions, 8 deletions
diff --git a/share/man/man9/heartbeat.9 b/share/man/man9/heartbeat.9 new file mode 100644 index 00000000000..740c43e09f3 --- /dev/null +++ b/share/man/man9/heartbeat.9 @@ -0,0 +1,169 @@ +.\" $NetBSD: heartbeat.9,v 1.1 2023/07/07 12:34:49 riastradh Exp $ +.\" +.\" Copyright (c) 2023 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.Dd July 6, 2023 +.Dt HEARTBEAT 9 +.Os +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh NAME +.Nm heartbeat +.Nd periodic checks to ensure CPUs are making progress +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh SYNOPSIS +.Fd "options HEARTBEAT" +.Fd "options HEARTBEAT_MAX_PERIOD_DEFAULT=15" +.\" +.In sys/heartbeat.h +.\" +.Ft void +.Fn heartbeat_start void +.Ft void +.Fn heartbeat void +.Ft void +.Fn heartbeat_suspend void +.Ft void +.Fn heartbeat_resume void +.Fd "#ifdef DDB" +.Ft void +.Fn heartbeat_dump void +.Fd "#endif" +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh DESCRIPTION +The +.Nm +subsystem verifies that soft interrupts +.Pq Xr softint 9 +and the system +.Xr timecounter 9 +are making progress over time, and panics if they appear stuck. +.Pp +The number of seconds before +.Nm +panics without progress is controlled by the sysctl knob +.Li kern.heartbeat.max_period , +which defaults to 15. +If set to zero, heartbeat checks are disabled. +.Pp +The periodic hardware timer interrupt handler calls +.Fn heartbeat +every tick on each CPU. +Once per second +.Po +i.e., every +.Xr hz 9 +ticks +.Pc , +.Fn heartbeat +schedules a soft interrupt at priority +.Dv SOFTINT_CLOCK +to advance the current CPU's view of +.Xr time_uptime 9 . +.Pp +.Fn heartbeat +checks whether +.Xr time_uptime 9 +has changed, to see if either the +.Xr timecounter 9 +or soft intrrupts on the current CPU are stuck. +If it hasn't advanced within +.Li kern.heartbeat.max_period +seconds worth of ticks, or if it has updated and the current CPU's view +of it hasn't been updated by more than +.Li kern.heartbeat.max_period +seconds, then +.Fn heartbeat +panics. +.Pp +.Fn heartbeat +also checks whether the next online CPU has advanced its view of +.Xr time_uptime 9 , +to see if soft interrupts +.Pq including Xr callout 9 +on that CPU are stuck. +If it hasn't updated within +.Li kern.heartbeat.max_period +seconds, +.Fn heartbeat +sends an +.Xr ipi 9 +to panic on that CPU. +If that CPU has not acknowledged the +.Xr ipi 9 +within one second, +.Fn heartbeat +panics on the current CPU instead. +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh FUNCTIONS +.Bl -tag -width Fn +.It Fn heartbeat +Check for timecounter and soft interrupt progress on this CPU and on +another CPU, and schedule a soft interrupt to advance this CPU's view +of timecounter progress. +.Pp +Called by +.Xr hardclock 9 +periodically. +.It Fn heartbeat_dump +Print all the heartbeat counter, uptime cache, and uptime cache +timestamp (in units of heartbeats) to the console. +.Pp +Can be invoked from +.Xr ddb 9 +by +.Ql call heartbeat_dump . +.It Fn heartbeat_resume +Resume heartbeat monitoring of the current CPU. +.Pp +Called after a CPU has started running but before it has been +marked online. +.It Fn heartbeat_start +Start monitoring heartbeats systemwide. +.Pp +Called by +.Xr main 9 +as soon as soft interrupts can be established. +.It Fn heartbeat_suspend +Suspend heartbeat monitoring of the current CPU. +.Pp +Called after the current CPU has been marked offline but before it has +stopped running. +.El +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh CODE REFERENCES +The +.Nm +subsystem is implemented in +.Pa sys/kern/kern_heartbeat.c . +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh SEE ALSO +.Xr wdogctl 8 , +.Xr swwdog 4 +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh HISTORY +The +.Nm +subsystem first appeared in +.Nx 11.0 . diff --git a/sys/kern/files.kern b/sys/kern/files.kern index 03eb05ec258..3ebde63dfac 100644 --- a/sys/kern/files.kern +++ b/sys/kern/files.kern @@ -1,13 +1,15 @@ -# $NetBSD: files.kern,v 1.57 2021/09/19 15:51:27 thorpej Exp $ +# $NetBSD: files.kern,v 1.58 2023/07/07 12:34:50 riastradh Exp $ # # kernel sources # define kern: cprng_fast, machdep, uvm +defflag opt_heartbeat.h HEARTBEAT defflag opt_kern.h KERN defflag opt_script.h SETUIDSCRIPTS FDSCRIPTS defflag KASLR defparam opt_cnmagic.h CNMAGIC +defparam heartbeat.h HEARTBEAT_MAX_PERIOD_DEFAULT file conf/debugsyms.c kern file conf/param.c kern @@ -48,6 +50,7 @@ file kern/kern_exec.c kern file kern/kern_exit.c kern file kern/kern_fork.c kern file kern/kern_idle.c kern +file kern/kern_heartbeat.c kern & heartbeat file kern/kern_hook.c kern file kern/kern_kthread.c kern file kern/kern_ktrace.c ktrace diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index f0428ad570f..1d2dbd742fe 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -1,4 +1,4 @@ -/* $NetBSD: init_main.c,v 1.541 2022/10/26 23:20:47 riastradh Exp $ */ +/* $NetBSD: init_main.c,v 1.542 2023/07/07 12:34:50 riastradh Exp $ */ /*- * Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc. @@ -97,10 +97,11 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.541 2022/10/26 23:20:47 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.542 2023/07/07 12:34:50 riastradh Exp $"); #include "opt_cnmagic.h" #include "opt_ddb.h" +#include "opt_heartbeat.h" #include "opt_inet.h" #include "opt_ipsec.h" #include "opt_modular.h" @@ -199,6 +200,7 @@ extern void *_binary_splash_image_end; #include <sys/cprng.h> #include <sys/psref.h> #include <sys/radixtree.h> +#include <sys/heartbeat.h> #include <sys/syscall.h> #include <sys/syscallargs.h> @@ -557,6 +559,14 @@ main(void) /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */ cprng_fast_init(); +#ifdef HEARTBEAT + /* + * Now that softints can be established, start monitoring + * system heartbeat on all CPUs. + */ + heartbeat_start(); +#endif + ssp_init(); ubc_init(); /* must be after autoconfig */ diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 53fa4ed7554..791301f8d6a 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh Exp $ */ +/* $NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $ */ /*- * Copyright (c) 2000, 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -69,11 +69,12 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $"); #ifdef _KERNEL_OPT #include "opt_dtrace.h" #include "opt_gprof.h" +#include "opt_heartbeat.h" #include "opt_multiprocessor.h" #endif @@ -92,6 +93,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh E #include <sys/cpu.h> #include <sys/atomic.h> #include <sys/rndsource.h> +#include <sys/heartbeat.h> #ifdef GPROF #include <sys/gmon.h> @@ -335,6 +337,13 @@ hardclock(struct clockframe *frame) tc_ticktock(); } +#ifdef HEARTBEAT + /* + * Make sure the CPUs and timecounter are making progress. + */ + heartbeat(); +#endif + /* * Update real-time timeout queue. */ diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c index 0ac53351e36..d082994d266 100644 --- a/sys/kern/kern_cpu.c +++ b/sys/kern/kern_cpu.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $ */ +/* $NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $ */ /*- * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc. @@ -60,10 +60,11 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $"); #ifdef _KERNEL_OPT #include "opt_cpu_ucode.h" +#include "opt_heartbeat.h" #endif #include <sys/param.h> @@ -85,6 +86,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $"); #include <sys/namei.h> #include <sys/callout.h> #include <sys/pcu.h> +#include <sys/heartbeat.h> #include <uvm/uvm_extern.h> @@ -377,6 +379,10 @@ fail: s = splsched(); spc->spc_flags &= ~SPCF_OFFLINE; splx(s); + +#ifdef HEARTBEAT + heartbeat_suspend(); +#endif } static void @@ -385,6 +391,10 @@ cpu_xc_online(struct cpu_info *ci, void *unused) struct schedstate_percpu *spc; int s; +#ifdef HEARTBEAT + heartbeat_resume(); +#endif + spc = &ci->ci_schedstate; s = splsched(); spc->spc_flags &= ~SPCF_OFFLINE; diff --git a/sys/kern/kern_heartbeat.c b/sys/kern/kern_heartbeat.c new file mode 100644 index 00000000000..732e13ade75 --- /dev/null +++ b/sys/kern/kern_heartbeat.c @@ -0,0 +1,656 @@ +/* $NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $ */ + +/*- + * Copyright (c) 2023 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * heartbeat(9) -- periodic checks to ensure CPUs are making progress + * + * Manual tests to run when changing this file. Magic numbers are for + * evbarm; adjust for other platforms. Tests involving cpuctl + * online/offline assume a 2-CPU system -- for full testing on a >2-CPU + * system, offline all but one CPU. + * + * 1. cpuctl offline 0 + * sleep 20 + * cpuctl online 0 + * + * 2. cpuctl offline 1 + * sleep 20 + * cpuctl online 1 + * + * 3. cpuctl offline 0 + * sysctl -w kern.heartbeat.max_period=5 + * sleep 10 + * sysctl -w kern.heartbeat.max_period=0 + * sleep 10 + * sysctl -w kern.heartbeat.max_period=5 + * sleep 10 + * cpuctl online 0 + * + * 4. sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK + * # verify system panics after 15sec + * + * 5. sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED + * # verify system panics after 15sec + * + * 6. cpuctl offline 0 + * sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK + * # verify system panics after 15sec + * + * 7. cpuctl offline 0 + * sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=5 # IPL_VM + * # verify system panics after 15sec + * + * # Not this -- IPL_SCHED and IPL_HIGH spinout on a single CPU + * # require a hardware watchdog timer. + * #cpuctl offline 0 + * #sysctl -w debug.crashme_enable + * #sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED + * # hope watchdog timer kicks in + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(0, "$NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $"); + +#ifdef _KERNEL_OPT +#include "opt_ddb.h" +#include "opt_heartbeat.h" +#endif + +#include "heartbeat.h" + +#include <sys/param.h> +#include <sys/types.h> + +#include <sys/atomic.h> +#include <sys/cpu.h> +#include <sys/errno.h> +#include <sys/heartbeat.h> +#include <sys/ipi.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <sys/systm.h> +#include <sys/xcall.h> + +#ifdef DDB +#include <ddb/ddb.h> +#endif + +/* + * Global state. + * + * heartbeat_lock serializes access to heartbeat_max_period_secs + * and heartbeat_max_period_ticks. Two separate variables so we + * can avoid multiplication or division in the heartbeat routine. + * + * heartbeat_sih is stable after initialization in + * heartbeat_start. + */ +kmutex_t heartbeat_lock __cacheline_aligned; +unsigned heartbeat_max_period_secs __read_mostly; +unsigned heartbeat_max_period_ticks __read_mostly; + +void *heartbeat_sih __read_mostly; + +/* + * heartbeat_suspend() + * + * Suspend heartbeat monitoring of the current CPU. + * + * Called after the current CPU has been marked offline but before + * it has stopped running. Caller must have preemption disabled. + */ +void +heartbeat_suspend(void) +{ + + KASSERT(kpreempt_disabled()); + + /* + * Nothing to do -- we just check the SPCF_OFFLINE flag. + */ +} + +/* + * heartbeat_resume() + * + * Resume heartbeat monitoring of the current CPU. + * + * Called after the current CPU has started running but before it + * has been marked online. Also used internally when starting up + * heartbeat monitoring at boot or when the maximum period is set + * from zero to nonzero. Caller must have preemption disabled. + */ +void +heartbeat_resume(void) +{ + struct cpu_info *ci = curcpu(); + int s; + + KASSERT(kpreempt_disabled()); + + /* + * Block heartbeats while we reset the state so we don't + * spuriously think we had a heart attack in the middle of + * resetting the count and the uptime stamp. + */ + s = splsched(); + ci->ci_heartbeat_count = 0; + ci->ci_heartbeat_uptime_cache = atomic_load_relaxed(&time_uptime); + ci->ci_heartbeat_uptime_stamp = 0; + splx(s); +} + +/* + * heartbeat_reset_xc(a, b) + * + * Cross-call handler to reset heartbeat state just prior to + * enabling heartbeat checks. + */ +static void +heartbeat_reset_xc(void *a, void *b) +{ + + heartbeat_resume(); +} + +/* + * set_max_period(max_period) + * + * Set the maximum period, in seconds, for heartbeat checks. + * + * - If max_period is zero, disable them. + * + * - If the max period was zero and max_period is nonzero, ensure + * all CPUs' heartbeat uptime caches are up-to-date before + * re-enabling them. + * + * max_period must be below UINT_MAX/4/hz to avoid arithmetic + * overflow and give room for slop. + * + * Caller must hold heartbeat_lock. + */ +static void +set_max_period(unsigned max_period) +{ + + KASSERTMSG(max_period <= UINT_MAX/4/hz, + "max_period=%u must not exceed UINT_MAX/4/hz=%u (hz=%u)", + max_period, UINT_MAX/4/hz, hz); + KASSERT(mutex_owned(&heartbeat_lock)); + + /* + * If we're enabling heartbeat checks, make sure we have a + * reasonably up-to-date time_uptime cache on all CPUs so we + * don't think we had an instant heart attack. + */ + if (heartbeat_max_period_secs == 0 && max_period != 0) + xc_wait(xc_broadcast(0, &heartbeat_reset_xc, NULL, NULL)); + + /* + * Once the heartbeat state has been updated on all (online) + * CPUs, set the period. At this point, heartbeat checks can + * begin. + */ + atomic_store_relaxed(&heartbeat_max_period_secs, max_period); + atomic_store_relaxed(&heartbeat_max_period_ticks, max_period*hz); +} + +/* + * heartbeat_max_period_ticks(SYSCTLFN_ARGS) + * + * Sysctl handler for sysctl kern.heartbeat.max_period. Verifies + * it lies within a reasonable interval and sets it. + */ +static int +heartbeat_max_period_sysctl(SYSCTLFN_ARGS) +{ + struct sysctlnode node; + unsigned max_period; + int error; + + mutex_enter(&heartbeat_lock); + + max_period = heartbeat_max_period_secs; + node = *rnode; + node.sysctl_data = &max_period; + error = sysctl_lookup(SYSCTLFN_CALL(&node)); + if (error || newp == NULL) + goto out; + + /* + * Ensure there's plenty of slop between heartbeats. + */ + if (max_period > UINT_MAX/4/hz) { + error = EOVERFLOW; + goto out; + } + + /* + * Success! Set the period. This enables heartbeat checks if + * we went from zero period to nonzero period, or disables them + * if the other way around. + */ + set_max_period(max_period); + error = 0; + +out: mutex_exit(&heartbeat_lock); + return error; +} + +/* + * sysctl_heartbeat_setup() + * + * Set up the kern.heartbeat.* sysctl subtree. + */ +SYSCTL_SETUP(sysctl_heartbeat_setup, "sysctl kern.heartbeat setup") +{ + const struct sysctlnode *rnode; + int error; + + mutex_init(&heartbeat_lock, MUTEX_DEFAULT, IPL_NONE); + + /* kern.heartbeat */ + error = sysctl_createv(NULL, 0, NULL, &rnode, + CTLFLAG_PERMANENT, + CTLTYPE_NODE, "heartbeat", + SYSCTL_DESCR("Kernel heartbeat parameters"), + NULL, 0, NULL, 0, + CTL_KERN, CTL_CREATE, CTL_EOL); + if (error) { + printf("%s: failed to create kern.heartbeat: %d\n", + __func__, error); + return; + } + + /* kern.heartbeat.max_period */ + error = sysctl_createv(NULL, 0, &rnode, NULL, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, + CTLTYPE_INT, "max_period", + SYSCTL_DESCR("Max seconds between heartbeats before panic"), + &heartbeat_max_period_sysctl, 0, NULL, 0, + CTL_CREATE, CTL_EOL); + if (error) { + printf("%s: failed to create kern.heartbeat.max_period: %d\n", + __func__, error); + return; + } +} + +/* + * heartbeat_intr(cookie) + * + * Soft interrupt handler to update the local CPU's view of the + * system uptime. This runs at the same priority level as + * callouts, so if callouts are stuck on this CPU, it won't run, + * and eventually another CPU will notice that this one is stuck. + * + * Don't do spl* here -- keep it to a minimum so if anything goes + * wrong we don't end up with hard interrupts blocked and unable + * to detect a missed heartbeat. + */ +static void +heartbeat_intr(void *cookie) +{ + unsigned count = atomic_load_relaxed(&curcpu()->ci_heartbeat_count); + unsigned uptime = atomic_load_relaxed(&time_uptime); + + atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_stamp, count); + atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_cache, uptime); +} + +/* + * heartbeat_start() + * + * Start system heartbeat monitoring. + */ +void +heartbeat_start(void) +{ + const unsigned max_period = HEARTBEAT_MAX_PERIOD_DEFAULT; + + /* + * Establish a softint so we can schedule it once ready. This + * should be at the lowest softint priority level so that we + * ensure all softint priorities are making progress. + */ + heartbeat_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, + &heartbeat_intr, NULL); + + /* + * Now that the softint is established, kick off heartbeat + * monitoring with the default period. This will initialize + * the per-CPU state to an up-to-date cache of time_uptime. + */ + mutex_enter(&heartbeat_lock); + set_max_period(max_period); + mutex_exit(&heartbeat_lock); +} + +/* + * defibrillator(cookie) + * + * IPI handler for defibrillation. If the CPU's heart has stopped + * beating normally, but the CPU can still execute things, + * acknowledge the IPI to the doctor and then panic so we at least + * get a stack trace from whatever the current CPU is stuck doing, + * if not a core dump. + * + * (This metaphor is a little stretched, since defibrillation is + * usually administered when the heart is beating errattically but + * hasn't stopped, and causes the heart to stop temporarily, and + * one hopes it is not fatal. But we're (software) engineers, so + * we can stretch metaphors like silly putty in a blender.) + */ +static void +defibrillator(void *cookie) +{ + bool *ack = cookie; + + atomic_store_relaxed(ack, true); + panic("%s[%d %s]: heart stopped beating", cpu_name(curcpu()), + curlwp->l_lid, + curlwp->l_name ? curlwp->l_name : curproc->p_comm); +} + +/* + * defibrillate(ci, unsigned d) + * + * The patient CPU ci's heart has stopped beating after d seconds. + * Force the patient CPU ci to panic, or panic on this CPU if the + * patient CPU doesn't respond within 1sec. + */ +static void __noinline +defibrillate(struct cpu_info *ci, unsigned d) +{ + bool ack = false; + ipi_msg_t msg = { + .func = &defibrillator, + .arg = &ack, + }; + unsigned countdown = 1000; /* 1sec */ + + KASSERT(kpreempt_disabled()); + + /* + * First notify the console that the patient CPU's heart seems + * to have stopped beating. + */ + printf("%s: found %s heart stopped beating after %u seconds\n", + cpu_name(curcpu()), cpu_name(ci), d); + + /* + * Next, give the patient CPU a chance to panic, so we get a + * stack trace on that CPU even if we don't get a crash dump. + */ + ipi_unicast(&msg, ci); + + /* + * Busy-wait up to 1sec for the patient CPU to print a stack + * trace and panic. If the patient CPU acknowledges the IPI, + * or if we're panicking anyway, just give up and stop here -- + * the system is coming down soon and we should avoid getting + * in the way. + */ + while (countdown --> 0) { + if (atomic_load_relaxed(&ack) || + atomic_load_relaxed(&panicstr) != NULL) + return; + DELAY(1000); /* 1ms */ + } + + /* + * The patient CPU failed to acknowledge the panic request. + * Panic now; with any luck, we'll get a crash dump. + */ + panic("%s: found %s heart stopped beating and unresponsive", + cpu_name(curcpu()), cpu_name(ci)); +} + +/* + * select_patient() + * + * Select another CPU to check the heartbeat of. Returns NULL if + * there are no other online CPUs. Never returns curcpu(). + * Caller must have kpreemption disabled. + */ +static struct cpu_info * +select_patient(void) +{ + CPU_INFO_ITERATOR cii; + struct cpu_info *first = NULL, *patient = NULL, *ci; + bool passedcur = false; + + KASSERT(kpreempt_disabled()); + + /* + * In the iteration order of all CPUs, find the next online CPU + * after curcpu(), or the first online one if curcpu() is last + * in the iteration order. + */ + for (CPU_INFO_FOREACH(cii, ci)) { + if (ci->ci_schedstate.spc_flags & SPCF_OFFLINE) + continue; + if (passedcur) { + /* + * (...|curcpu()|ci|...) + * + * Found the patient right after curcpu(). + */ + KASSERT(patient != ci); + patient = ci; + break; + } + if (ci == curcpu()) { + /* + * (...|prev|ci=curcpu()|next|...) + * + * Note that we want next (or first, if there's + * nothing after curcpu()). + */ + passedcur = true; + continue; + } + if (first == NULL) { + /* + * (ci|...|curcpu()|...) + * + * Record ci as first in case there's nothing + * after curcpu(). + */ + first = ci; + continue; + } + } + + /* + * If we hit the end, wrap around to the beginning. + */ + if (patient == NULL) { + KASSERT(passedcur); + patient = first; + } + + return patient; +} + +/* + * heartbeat() + * + * 1. Count a heartbeat on the local CPU. + * + * 2. Panic if the system uptime doesn't seem to have advanced in + * a while. + * + * 3. Panic if the soft interrupt on this CPU hasn't advanced the + * local view of the system uptime. + * + * 4. Schedule the soft interrupt to advance the local view of the + * system uptime. + * + * 5. Select another CPU to check the heartbeat of. + * + * 6. Panic if the other CPU hasn't advanced its view of the + * system uptime in a while. + */ +void +heartbeat(void) +{ + unsigned period_ticks, period_secs; + unsigned count, uptime, cache, stamp, d; + struct cpu_info *patient; + + KASSERT(kpreempt_disabled()); + + period_ticks = atomic_load_relaxed(&heartbeat_max_period_ticks); + period_secs = atomic_load_relaxed(&heartbeat_max_period_secs); + if (__predict_false(period_ticks == 0) || + __predict_false(period_secs == 0) || + __predict_false(curcpu()->ci_schedstate.spc_flags & SPCF_OFFLINE)) + return; + + /* + * Count a heartbeat on this CPU. + */ + count = curcpu()->ci_heartbeat_count++; + + /* + * If the uptime hasn't changed, make sure that we haven't + * counted too many of our own heartbeats since the uptime last + * changed, and stop here -- we only do the cross-CPU work once + * per second. + */ + uptime = atomic_load_relaxed(&time_uptime); + cache = atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_cache); + if (__predict_true(cache == uptime)) { + /* + * Timecounter hasn't advanced by more than a second. + * Make sure the timecounter isn't stuck according to + * our heartbeats. + * + * Our own heartbeat count can't roll back, and + * time_uptime should be updated before it wraps + * around, so d should never go negative; hence no + * check for d < UINT_MAX/2. + */ + stamp = + atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_stamp); + d = count - stamp; + if (__predict_false(d > period_ticks)) { + panic("%s: time has not advanced in %u heartbeats", + cpu_name(curcpu()), d); + } + return; + } + + /* + * If the uptime has changed, make sure that it hasn't changed + * so much that softints must be stuck on this CPU. Since + * time_uptime is monotonic, this can't go negative, hence no + * check for d < UINT_MAX/2. + * + * This uses the hard timer interrupt handler on the current + * CPU to ensure soft interrupts at all priority levels have + * made progress. + */ + d = uptime - cache; + if (__predict_false(d > period_secs)) { + panic("%s: softints stuck for %u seconds", + cpu_name(curcpu()), d); + } + + /* + * Schedule a softint to update our cache of the system uptime + * so the next call to heartbeat, on this or another CPU, can + * detect progress on this one. + */ + softint_schedule(heartbeat_sih); + + /* + * Select a patient to check the heartbeat of. If there's no + * other online CPU, nothing to do. + */ + patient = select_patient(); + if (patient == NULL) + return; + + /* + * Verify that time is advancing on the patient CPU. If the + * delta exceeds UINT_MAX/2, that means it is already ahead by + * a little on the other CPU, and the subtraction went + * negative, which is OK. If the CPU has been + * offlined since we selected it, no worries. + * + * This uses the current CPU to ensure the other CPU has made + * progress, even if the other CPU's hard timer interrupt + * handler is stuck for some reason. + * + * XXX Maybe confirm it hasn't gone negative by more than + * max_period? + */ + d = uptime - atomic_load_relaxed(&patient->ci_heartbeat_uptime_cache); + if (__predict_false(d > period_secs) && + __predict_false(d < UINT_MAX/2) && + ((patient->ci_schedstate.spc_flags & SPCF_OFFLINE) == 0)) + defibrillate(patient, d); +} + +/* + * heartbeat_dump() + * + * Print the heartbeat data of all CPUs. Can be called from ddb. + */ +#ifdef DDB +static unsigned +db_read_unsigned(const unsigned *p) +{ + unsigned x; + + db_read_bytes((db_addr_t)p, sizeof(x), (char *)&x); + + return x; +} + +void +heartbeat_dump(void) +{ + struct cpu_info *ci; + + db_printf("Heartbeats:\n"); + for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) { + db_printf("cpu%u: count %u uptime %u stamp %u\n", + db_read_unsigned(&ci->ci_index), + db_read_unsigned(&ci->ci_heartbeat_count), + db_read_unsigned(&ci->ci_heartbeat_uptime_cache), + db_read_unsigned(&ci->ci_heartbeat_uptime_stamp)); + } +} +#endif diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h index 4c2c6fe3d76..80341ff55fb 100644 --- a/sys/sys/cpu_data.h +++ b/sys/sys/cpu_data.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_data.h,v 1.52 2020/06/14 21:41:42 ad Exp $ */ +/* $NetBSD: cpu_data.h,v 1.53 2023/07/07 12:34:50 riastradh Exp $ */ /*- * Copyright (c) 2004, 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. @@ -188,6 +188,11 @@ struct cpu_data { kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */ struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT]; int64_t cpu_counts[CPU_COUNT_MAX];/* per-CPU counts */ + + unsigned cpu_heartbeat_count; /* # of heartbeats */ + unsigned cpu_heartbeat_uptime_cache; /* last time_uptime */ + unsigned cpu_heartbeat_uptime_stamp; /* heartbeats since + * uptime changed */ }; #define ci_schedstate ci_data.cpu_schedstate @@ -216,6 +221,10 @@ struct cpu_data { #define ci_faultrng ci_data.cpu_faultrng #define ci_counts ci_data.cpu_counts +#define ci_heartbeat_count ci_data.cpu_heartbeat_count +#define ci_heartbeat_uptime_cache ci_data.cpu_heartbeat_uptime_cache +#define ci_heartbeat_uptime_stamp ci_data.cpu_heartbeat_uptime_stamp + #define cpu_nsyscall cpu_counts[CPU_COUNT_NSYSCALL] #define cpu_ntrap cpu_counts[CPU_COUNT_NTRAP] #define cpu_nswtch cpu_counts[CPU_COUNT_NSWTCH] diff --git a/sys/sys/heartbeat.h b/sys/sys/heartbeat.h new file mode 100644 index 00000000000..ef32feaf5ec --- /dev/null +++ b/sys/sys/heartbeat.h @@ -0,0 +1,53 @@ +/* $NetBSD: heartbeat.h,v 1.1 2023/07/07 12:34:50 riastradh Exp $ */ + +/*- + * Copyright (c) 2023 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_HEARTBEAT_H +#define _SYS_HEARTBEAT_H + +#ifndef _KERNEL +#error No user-serviceable parts in this heart. +#endif + +#ifdef _KERNEL_OPT +#include "opt_heartbeat.h" +#endif + +#ifdef HEARTBEAT + +void heartbeat_start(void); + +void heartbeat(void); + +void heartbeat_suspend(void); +void heartbeat_resume(void); + +void heartbeat_dump(void); + +#endif + +#endif /* _SYS_HEARTBEAT_H */ |
