summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--share/man/man9/heartbeat.9169
-rw-r--r--sys/kern/files.kern5
-rw-r--r--sys/kern/init_main.c14
-rw-r--r--sys/kern/kern_clock.c13
-rw-r--r--sys/kern/kern_cpu.c14
-rw-r--r--sys/kern/kern_heartbeat.c656
-rw-r--r--sys/sys/cpu_data.h11
-rw-r--r--sys/sys/heartbeat.h53
8 files changed, 927 insertions, 8 deletions
diff --git a/share/man/man9/heartbeat.9 b/share/man/man9/heartbeat.9
new file mode 100644
index 00000000000..740c43e09f3
--- /dev/null
+++ b/share/man/man9/heartbeat.9
@@ -0,0 +1,169 @@
+.\" $NetBSD: heartbeat.9,v 1.1 2023/07/07 12:34:49 riastradh Exp $
+.\"
+.\" Copyright (c) 2023 The NetBSD Foundation, Inc.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.Dd July 6, 2023
+.Dt HEARTBEAT 9
+.Os
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh NAME
+.Nm heartbeat
+.Nd periodic checks to ensure CPUs are making progress
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh SYNOPSIS
+.Fd "options HEARTBEAT"
+.Fd "options HEARTBEAT_MAX_PERIOD_DEFAULT=15"
+.\"
+.In sys/heartbeat.h
+.\"
+.Ft void
+.Fn heartbeat_start void
+.Ft void
+.Fn heartbeat void
+.Ft void
+.Fn heartbeat_suspend void
+.Ft void
+.Fn heartbeat_resume void
+.Fd "#ifdef DDB"
+.Ft void
+.Fn heartbeat_dump void
+.Fd "#endif"
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh DESCRIPTION
+The
+.Nm
+subsystem verifies that soft interrupts
+.Pq Xr softint 9
+and the system
+.Xr timecounter 9
+are making progress over time, and panics if they appear stuck.
+.Pp
+The number of seconds before
+.Nm
+panics without progress is controlled by the sysctl knob
+.Li kern.heartbeat.max_period ,
+which defaults to 15.
+If set to zero, heartbeat checks are disabled.
+.Pp
+The periodic hardware timer interrupt handler calls
+.Fn heartbeat
+every tick on each CPU.
+Once per second
+.Po
+i.e., every
+.Xr hz 9
+ticks
+.Pc ,
+.Fn heartbeat
+schedules a soft interrupt at priority
+.Dv SOFTINT_CLOCK
+to advance the current CPU's view of
+.Xr time_uptime 9 .
+.Pp
+.Fn heartbeat
+checks whether
+.Xr time_uptime 9
+has changed, to see if either the
+.Xr timecounter 9
+or soft intrrupts on the current CPU are stuck.
+If it hasn't advanced within
+.Li kern.heartbeat.max_period
+seconds worth of ticks, or if it has updated and the current CPU's view
+of it hasn't been updated by more than
+.Li kern.heartbeat.max_period
+seconds, then
+.Fn heartbeat
+panics.
+.Pp
+.Fn heartbeat
+also checks whether the next online CPU has advanced its view of
+.Xr time_uptime 9 ,
+to see if soft interrupts
+.Pq including Xr callout 9
+on that CPU are stuck.
+If it hasn't updated within
+.Li kern.heartbeat.max_period
+seconds,
+.Fn heartbeat
+sends an
+.Xr ipi 9
+to panic on that CPU.
+If that CPU has not acknowledged the
+.Xr ipi 9
+within one second,
+.Fn heartbeat
+panics on the current CPU instead.
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh FUNCTIONS
+.Bl -tag -width Fn
+.It Fn heartbeat
+Check for timecounter and soft interrupt progress on this CPU and on
+another CPU, and schedule a soft interrupt to advance this CPU's view
+of timecounter progress.
+.Pp
+Called by
+.Xr hardclock 9
+periodically.
+.It Fn heartbeat_dump
+Print all the heartbeat counter, uptime cache, and uptime cache
+timestamp (in units of heartbeats) to the console.
+.Pp
+Can be invoked from
+.Xr ddb 9
+by
+.Ql call heartbeat_dump .
+.It Fn heartbeat_resume
+Resume heartbeat monitoring of the current CPU.
+.Pp
+Called after a CPU has started running but before it has been
+marked online.
+.It Fn heartbeat_start
+Start monitoring heartbeats systemwide.
+.Pp
+Called by
+.Xr main 9
+as soon as soft interrupts can be established.
+.It Fn heartbeat_suspend
+Suspend heartbeat monitoring of the current CPU.
+.Pp
+Called after the current CPU has been marked offline but before it has
+stopped running.
+.El
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh CODE REFERENCES
+The
+.Nm
+subsystem is implemented in
+.Pa sys/kern/kern_heartbeat.c .
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh SEE ALSO
+.Xr wdogctl 8 ,
+.Xr swwdog 4
+.\"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.Sh HISTORY
+The
+.Nm
+subsystem first appeared in
+.Nx 11.0 .
diff --git a/sys/kern/files.kern b/sys/kern/files.kern
index 03eb05ec258..3ebde63dfac 100644
--- a/sys/kern/files.kern
+++ b/sys/kern/files.kern
@@ -1,13 +1,15 @@
-# $NetBSD: files.kern,v 1.57 2021/09/19 15:51:27 thorpej Exp $
+# $NetBSD: files.kern,v 1.58 2023/07/07 12:34:50 riastradh Exp $
#
# kernel sources
#
define kern: cprng_fast, machdep, uvm
+defflag opt_heartbeat.h HEARTBEAT
defflag opt_kern.h KERN
defflag opt_script.h SETUIDSCRIPTS FDSCRIPTS
defflag KASLR
defparam opt_cnmagic.h CNMAGIC
+defparam heartbeat.h HEARTBEAT_MAX_PERIOD_DEFAULT
file conf/debugsyms.c kern
file conf/param.c kern
@@ -48,6 +50,7 @@ file kern/kern_exec.c kern
file kern/kern_exit.c kern
file kern/kern_fork.c kern
file kern/kern_idle.c kern
+file kern/kern_heartbeat.c kern & heartbeat
file kern/kern_hook.c kern
file kern/kern_kthread.c kern
file kern/kern_ktrace.c ktrace
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index f0428ad570f..1d2dbd742fe 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/* $NetBSD: init_main.c,v 1.541 2022/10/26 23:20:47 riastradh Exp $ */
+/* $NetBSD: init_main.c,v 1.542 2023/07/07 12:34:50 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -97,10 +97,11 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.541 2022/10/26 23:20:47 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.542 2023/07/07 12:34:50 riastradh Exp $");
#include "opt_cnmagic.h"
#include "opt_ddb.h"
+#include "opt_heartbeat.h"
#include "opt_inet.h"
#include "opt_ipsec.h"
#include "opt_modular.h"
@@ -199,6 +200,7 @@ extern void *_binary_splash_image_end;
#include <sys/cprng.h>
#include <sys/psref.h>
#include <sys/radixtree.h>
+#include <sys/heartbeat.h>
#include <sys/syscall.h>
#include <sys/syscallargs.h>
@@ -557,6 +559,14 @@ main(void)
/* Once all CPUs are detected, initialize the per-CPU cprng_fast. */
cprng_fast_init();
+#ifdef HEARTBEAT
+ /*
+ * Now that softints can be established, start monitoring
+ * system heartbeat on all CPUs.
+ */
+ heartbeat_start();
+#endif
+
ssp_init();
ubc_init(); /* must be after autoconfig */
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 53fa4ed7554..791301f8d6a 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh Exp $ */
+/* $NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $ */
/*-
* Copyright (c) 2000, 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -69,11 +69,12 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_dtrace.h"
#include "opt_gprof.h"
+#include "opt_heartbeat.h"
#include "opt_multiprocessor.h"
#endif
@@ -92,6 +93,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.149 2023/06/30 21:42:05 riastradh E
#include <sys/cpu.h>
#include <sys/atomic.h>
#include <sys/rndsource.h>
+#include <sys/heartbeat.h>
#ifdef GPROF
#include <sys/gmon.h>
@@ -335,6 +337,13 @@ hardclock(struct clockframe *frame)
tc_ticktock();
}
+#ifdef HEARTBEAT
+ /*
+ * Make sure the CPUs and timecounter are making progress.
+ */
+ heartbeat();
+#endif
+
/*
* Update real-time timeout queue.
*/
diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c
index 0ac53351e36..d082994d266 100644
--- a/sys/kern/kern_cpu.c
+++ b/sys/kern/kern_cpu.c
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
@@ -60,10 +60,11 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.95 2023/07/07 12:34:50 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_cpu_ucode.h"
+#include "opt_heartbeat.h"
#endif
#include <sys/param.h>
@@ -85,6 +86,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.94 2023/02/26 07:13:55 skrll Exp $");
#include <sys/namei.h>
#include <sys/callout.h>
#include <sys/pcu.h>
+#include <sys/heartbeat.h>
#include <uvm/uvm_extern.h>
@@ -377,6 +379,10 @@ fail:
s = splsched();
spc->spc_flags &= ~SPCF_OFFLINE;
splx(s);
+
+#ifdef HEARTBEAT
+ heartbeat_suspend();
+#endif
}
static void
@@ -385,6 +391,10 @@ cpu_xc_online(struct cpu_info *ci, void *unused)
struct schedstate_percpu *spc;
int s;
+#ifdef HEARTBEAT
+ heartbeat_resume();
+#endif
+
spc = &ci->ci_schedstate;
s = splsched();
spc->spc_flags &= ~SPCF_OFFLINE;
diff --git a/sys/kern/kern_heartbeat.c b/sys/kern/kern_heartbeat.c
new file mode 100644
index 00000000000..732e13ade75
--- /dev/null
+++ b/sys/kern/kern_heartbeat.c
@@ -0,0 +1,656 @@
+/* $NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2023 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * heartbeat(9) -- periodic checks to ensure CPUs are making progress
+ *
+ * Manual tests to run when changing this file. Magic numbers are for
+ * evbarm; adjust for other platforms. Tests involving cpuctl
+ * online/offline assume a 2-CPU system -- for full testing on a >2-CPU
+ * system, offline all but one CPU.
+ *
+ * 1. cpuctl offline 0
+ * sleep 20
+ * cpuctl online 0
+ *
+ * 2. cpuctl offline 1
+ * sleep 20
+ * cpuctl online 1
+ *
+ * 3. cpuctl offline 0
+ * sysctl -w kern.heartbeat.max_period=5
+ * sleep 10
+ * sysctl -w kern.heartbeat.max_period=0
+ * sleep 10
+ * sysctl -w kern.heartbeat.max_period=5
+ * sleep 10
+ * cpuctl online 0
+ *
+ * 4. sysctl -w debug.crashme_enable=1
+ * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK
+ * # verify system panics after 15sec
+ *
+ * 5. sysctl -w debug.crashme_enable=1
+ * sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED
+ * # verify system panics after 15sec
+ *
+ * 6. cpuctl offline 0
+ * sysctl -w debug.crashme_enable=1
+ * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK
+ * # verify system panics after 15sec
+ *
+ * 7. cpuctl offline 0
+ * sysctl -w debug.crashme_enable=1
+ * sysctl -w debug.crashme.spl_spinout=5 # IPL_VM
+ * # verify system panics after 15sec
+ *
+ * # Not this -- IPL_SCHED and IPL_HIGH spinout on a single CPU
+ * # require a hardware watchdog timer.
+ * #cpuctl offline 0
+ * #sysctl -w debug.crashme_enable
+ * #sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED
+ * # hope watchdog timer kicks in
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $");
+
+#ifdef _KERNEL_OPT
+#include "opt_ddb.h"
+#include "opt_heartbeat.h"
+#endif
+
+#include "heartbeat.h"
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <sys/atomic.h>
+#include <sys/cpu.h>
+#include <sys/errno.h>
+#include <sys/heartbeat.h>
+#include <sys/ipi.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/xcall.h>
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
+
+/*
+ * Global state.
+ *
+ * heartbeat_lock serializes access to heartbeat_max_period_secs
+ * and heartbeat_max_period_ticks. Two separate variables so we
+ * can avoid multiplication or division in the heartbeat routine.
+ *
+ * heartbeat_sih is stable after initialization in
+ * heartbeat_start.
+ */
+kmutex_t heartbeat_lock __cacheline_aligned;
+unsigned heartbeat_max_period_secs __read_mostly;
+unsigned heartbeat_max_period_ticks __read_mostly;
+
+void *heartbeat_sih __read_mostly;
+
+/*
+ * heartbeat_suspend()
+ *
+ * Suspend heartbeat monitoring of the current CPU.
+ *
+ * Called after the current CPU has been marked offline but before
+ * it has stopped running. Caller must have preemption disabled.
+ */
+void
+heartbeat_suspend(void)
+{
+
+ KASSERT(kpreempt_disabled());
+
+ /*
+ * Nothing to do -- we just check the SPCF_OFFLINE flag.
+ */
+}
+
+/*
+ * heartbeat_resume()
+ *
+ * Resume heartbeat monitoring of the current CPU.
+ *
+ * Called after the current CPU has started running but before it
+ * has been marked online. Also used internally when starting up
+ * heartbeat monitoring at boot or when the maximum period is set
+ * from zero to nonzero. Caller must have preemption disabled.
+ */
+void
+heartbeat_resume(void)
+{
+ struct cpu_info *ci = curcpu();
+ int s;
+
+ KASSERT(kpreempt_disabled());
+
+ /*
+ * Block heartbeats while we reset the state so we don't
+ * spuriously think we had a heart attack in the middle of
+ * resetting the count and the uptime stamp.
+ */
+ s = splsched();
+ ci->ci_heartbeat_count = 0;
+ ci->ci_heartbeat_uptime_cache = atomic_load_relaxed(&time_uptime);
+ ci->ci_heartbeat_uptime_stamp = 0;
+ splx(s);
+}
+
+/*
+ * heartbeat_reset_xc(a, b)
+ *
+ * Cross-call handler to reset heartbeat state just prior to
+ * enabling heartbeat checks.
+ */
+static void
+heartbeat_reset_xc(void *a, void *b)
+{
+
+ heartbeat_resume();
+}
+
+/*
+ * set_max_period(max_period)
+ *
+ * Set the maximum period, in seconds, for heartbeat checks.
+ *
+ * - If max_period is zero, disable them.
+ *
+ * - If the max period was zero and max_period is nonzero, ensure
+ * all CPUs' heartbeat uptime caches are up-to-date before
+ * re-enabling them.
+ *
+ * max_period must be below UINT_MAX/4/hz to avoid arithmetic
+ * overflow and give room for slop.
+ *
+ * Caller must hold heartbeat_lock.
+ */
+static void
+set_max_period(unsigned max_period)
+{
+
+ KASSERTMSG(max_period <= UINT_MAX/4/hz,
+ "max_period=%u must not exceed UINT_MAX/4/hz=%u (hz=%u)",
+ max_period, UINT_MAX/4/hz, hz);
+ KASSERT(mutex_owned(&heartbeat_lock));
+
+ /*
+ * If we're enabling heartbeat checks, make sure we have a
+ * reasonably up-to-date time_uptime cache on all CPUs so we
+ * don't think we had an instant heart attack.
+ */
+ if (heartbeat_max_period_secs == 0 && max_period != 0)
+ xc_wait(xc_broadcast(0, &heartbeat_reset_xc, NULL, NULL));
+
+ /*
+ * Once the heartbeat state has been updated on all (online)
+ * CPUs, set the period. At this point, heartbeat checks can
+ * begin.
+ */
+ atomic_store_relaxed(&heartbeat_max_period_secs, max_period);
+ atomic_store_relaxed(&heartbeat_max_period_ticks, max_period*hz);
+}
+
+/*
+ * heartbeat_max_period_ticks(SYSCTLFN_ARGS)
+ *
+ * Sysctl handler for sysctl kern.heartbeat.max_period. Verifies
+ * it lies within a reasonable interval and sets it.
+ */
+static int
+heartbeat_max_period_sysctl(SYSCTLFN_ARGS)
+{
+ struct sysctlnode node;
+ unsigned max_period;
+ int error;
+
+ mutex_enter(&heartbeat_lock);
+
+ max_period = heartbeat_max_period_secs;
+ node = *rnode;
+ node.sysctl_data = &max_period;
+ error = sysctl_lookup(SYSCTLFN_CALL(&node));
+ if (error || newp == NULL)
+ goto out;
+
+ /*
+ * Ensure there's plenty of slop between heartbeats.
+ */
+ if (max_period > UINT_MAX/4/hz) {
+ error = EOVERFLOW;
+ goto out;
+ }
+
+ /*
+ * Success! Set the period. This enables heartbeat checks if
+ * we went from zero period to nonzero period, or disables them
+ * if the other way around.
+ */
+ set_max_period(max_period);
+ error = 0;
+
+out: mutex_exit(&heartbeat_lock);
+ return error;
+}
+
+/*
+ * sysctl_heartbeat_setup()
+ *
+ * Set up the kern.heartbeat.* sysctl subtree.
+ */
+SYSCTL_SETUP(sysctl_heartbeat_setup, "sysctl kern.heartbeat setup")
+{
+ const struct sysctlnode *rnode;
+ int error;
+
+ mutex_init(&heartbeat_lock, MUTEX_DEFAULT, IPL_NONE);
+
+ /* kern.heartbeat */
+ error = sysctl_createv(NULL, 0, NULL, &rnode,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_NODE, "heartbeat",
+ SYSCTL_DESCR("Kernel heartbeat parameters"),
+ NULL, 0, NULL, 0,
+ CTL_KERN, CTL_CREATE, CTL_EOL);
+ if (error) {
+ printf("%s: failed to create kern.heartbeat: %d\n",
+ __func__, error);
+ return;
+ }
+
+ /* kern.heartbeat.max_period */
+ error = sysctl_createv(NULL, 0, &rnode, NULL,
+ CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
+ CTLTYPE_INT, "max_period",
+ SYSCTL_DESCR("Max seconds between heartbeats before panic"),
+ &heartbeat_max_period_sysctl, 0, NULL, 0,
+ CTL_CREATE, CTL_EOL);
+ if (error) {
+ printf("%s: failed to create kern.heartbeat.max_period: %d\n",
+ __func__, error);
+ return;
+ }
+}
+
+/*
+ * heartbeat_intr(cookie)
+ *
+ * Soft interrupt handler to update the local CPU's view of the
+ * system uptime. This runs at the same priority level as
+ * callouts, so if callouts are stuck on this CPU, it won't run,
+ * and eventually another CPU will notice that this one is stuck.
+ *
+ * Don't do spl* here -- keep it to a minimum so if anything goes
+ * wrong we don't end up with hard interrupts blocked and unable
+ * to detect a missed heartbeat.
+ */
+static void
+heartbeat_intr(void *cookie)
+{
+ unsigned count = atomic_load_relaxed(&curcpu()->ci_heartbeat_count);
+ unsigned uptime = atomic_load_relaxed(&time_uptime);
+
+ atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_stamp, count);
+ atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_cache, uptime);
+}
+
+/*
+ * heartbeat_start()
+ *
+ * Start system heartbeat monitoring.
+ */
+void
+heartbeat_start(void)
+{
+ const unsigned max_period = HEARTBEAT_MAX_PERIOD_DEFAULT;
+
+ /*
+ * Establish a softint so we can schedule it once ready. This
+ * should be at the lowest softint priority level so that we
+ * ensure all softint priorities are making progress.
+ */
+ heartbeat_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE,
+ &heartbeat_intr, NULL);
+
+ /*
+ * Now that the softint is established, kick off heartbeat
+ * monitoring with the default period. This will initialize
+ * the per-CPU state to an up-to-date cache of time_uptime.
+ */
+ mutex_enter(&heartbeat_lock);
+ set_max_period(max_period);
+ mutex_exit(&heartbeat_lock);
+}
+
+/*
+ * defibrillator(cookie)
+ *
+ * IPI handler for defibrillation. If the CPU's heart has stopped
+ * beating normally, but the CPU can still execute things,
+ * acknowledge the IPI to the doctor and then panic so we at least
+ * get a stack trace from whatever the current CPU is stuck doing,
+ * if not a core dump.
+ *
+ * (This metaphor is a little stretched, since defibrillation is
+ * usually administered when the heart is beating errattically but
+ * hasn't stopped, and causes the heart to stop temporarily, and
+ * one hopes it is not fatal. But we're (software) engineers, so
+ * we can stretch metaphors like silly putty in a blender.)
+ */
+static void
+defibrillator(void *cookie)
+{
+ bool *ack = cookie;
+
+ atomic_store_relaxed(ack, true);
+ panic("%s[%d %s]: heart stopped beating", cpu_name(curcpu()),
+ curlwp->l_lid,
+ curlwp->l_name ? curlwp->l_name : curproc->p_comm);
+}
+
+/*
+ * defibrillate(ci, unsigned d)
+ *
+ * The patient CPU ci's heart has stopped beating after d seconds.
+ * Force the patient CPU ci to panic, or panic on this CPU if the
+ * patient CPU doesn't respond within 1sec.
+ */
+static void __noinline
+defibrillate(struct cpu_info *ci, unsigned d)
+{
+ bool ack = false;
+ ipi_msg_t msg = {
+ .func = &defibrillator,
+ .arg = &ack,
+ };
+ unsigned countdown = 1000; /* 1sec */
+
+ KASSERT(kpreempt_disabled());
+
+ /*
+ * First notify the console that the patient CPU's heart seems
+ * to have stopped beating.
+ */
+ printf("%s: found %s heart stopped beating after %u seconds\n",
+ cpu_name(curcpu()), cpu_name(ci), d);
+
+ /*
+ * Next, give the patient CPU a chance to panic, so we get a
+ * stack trace on that CPU even if we don't get a crash dump.
+ */
+ ipi_unicast(&msg, ci);
+
+ /*
+ * Busy-wait up to 1sec for the patient CPU to print a stack
+ * trace and panic. If the patient CPU acknowledges the IPI,
+ * or if we're panicking anyway, just give up and stop here --
+ * the system is coming down soon and we should avoid getting
+ * in the way.
+ */
+ while (countdown --> 0) {
+ if (atomic_load_relaxed(&ack) ||
+ atomic_load_relaxed(&panicstr) != NULL)
+ return;
+ DELAY(1000); /* 1ms */
+ }
+
+ /*
+ * The patient CPU failed to acknowledge the panic request.
+ * Panic now; with any luck, we'll get a crash dump.
+ */
+ panic("%s: found %s heart stopped beating and unresponsive",
+ cpu_name(curcpu()), cpu_name(ci));
+}
+
+/*
+ * select_patient()
+ *
+ * Select another CPU to check the heartbeat of. Returns NULL if
+ * there are no other online CPUs. Never returns curcpu().
+ * Caller must have kpreemption disabled.
+ */
+static struct cpu_info *
+select_patient(void)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *first = NULL, *patient = NULL, *ci;
+ bool passedcur = false;
+
+ KASSERT(kpreempt_disabled());
+
+ /*
+ * In the iteration order of all CPUs, find the next online CPU
+ * after curcpu(), or the first online one if curcpu() is last
+ * in the iteration order.
+ */
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ if (ci->ci_schedstate.spc_flags & SPCF_OFFLINE)
+ continue;
+ if (passedcur) {
+ /*
+ * (...|curcpu()|ci|...)
+ *
+ * Found the patient right after curcpu().
+ */
+ KASSERT(patient != ci);
+ patient = ci;
+ break;
+ }
+ if (ci == curcpu()) {
+ /*
+ * (...|prev|ci=curcpu()|next|...)
+ *
+ * Note that we want next (or first, if there's
+ * nothing after curcpu()).
+ */
+ passedcur = true;
+ continue;
+ }
+ if (first == NULL) {
+ /*
+ * (ci|...|curcpu()|...)
+ *
+ * Record ci as first in case there's nothing
+ * after curcpu().
+ */
+ first = ci;
+ continue;
+ }
+ }
+
+ /*
+ * If we hit the end, wrap around to the beginning.
+ */
+ if (patient == NULL) {
+ KASSERT(passedcur);
+ patient = first;
+ }
+
+ return patient;
+}
+
+/*
+ * heartbeat()
+ *
+ * 1. Count a heartbeat on the local CPU.
+ *
+ * 2. Panic if the system uptime doesn't seem to have advanced in
+ * a while.
+ *
+ * 3. Panic if the soft interrupt on this CPU hasn't advanced the
+ * local view of the system uptime.
+ *
+ * 4. Schedule the soft interrupt to advance the local view of the
+ * system uptime.
+ *
+ * 5. Select another CPU to check the heartbeat of.
+ *
+ * 6. Panic if the other CPU hasn't advanced its view of the
+ * system uptime in a while.
+ */
+void
+heartbeat(void)
+{
+ unsigned period_ticks, period_secs;
+ unsigned count, uptime, cache, stamp, d;
+ struct cpu_info *patient;
+
+ KASSERT(kpreempt_disabled());
+
+ period_ticks = atomic_load_relaxed(&heartbeat_max_period_ticks);
+ period_secs = atomic_load_relaxed(&heartbeat_max_period_secs);
+ if (__predict_false(period_ticks == 0) ||
+ __predict_false(period_secs == 0) ||
+ __predict_false(curcpu()->ci_schedstate.spc_flags & SPCF_OFFLINE))
+ return;
+
+ /*
+ * Count a heartbeat on this CPU.
+ */
+ count = curcpu()->ci_heartbeat_count++;
+
+ /*
+ * If the uptime hasn't changed, make sure that we haven't
+ * counted too many of our own heartbeats since the uptime last
+ * changed, and stop here -- we only do the cross-CPU work once
+ * per second.
+ */
+ uptime = atomic_load_relaxed(&time_uptime);
+ cache = atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_cache);
+ if (__predict_true(cache == uptime)) {
+ /*
+ * Timecounter hasn't advanced by more than a second.
+ * Make sure the timecounter isn't stuck according to
+ * our heartbeats.
+ *
+ * Our own heartbeat count can't roll back, and
+ * time_uptime should be updated before it wraps
+ * around, so d should never go negative; hence no
+ * check for d < UINT_MAX/2.
+ */
+ stamp =
+ atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_stamp);
+ d = count - stamp;
+ if (__predict_false(d > period_ticks)) {
+ panic("%s: time has not advanced in %u heartbeats",
+ cpu_name(curcpu()), d);
+ }
+ return;
+ }
+
+ /*
+ * If the uptime has changed, make sure that it hasn't changed
+ * so much that softints must be stuck on this CPU. Since
+ * time_uptime is monotonic, this can't go negative, hence no
+ * check for d < UINT_MAX/2.
+ *
+ * This uses the hard timer interrupt handler on the current
+ * CPU to ensure soft interrupts at all priority levels have
+ * made progress.
+ */
+ d = uptime - cache;
+ if (__predict_false(d > period_secs)) {
+ panic("%s: softints stuck for %u seconds",
+ cpu_name(curcpu()), d);
+ }
+
+ /*
+ * Schedule a softint to update our cache of the system uptime
+ * so the next call to heartbeat, on this or another CPU, can
+ * detect progress on this one.
+ */
+ softint_schedule(heartbeat_sih);
+
+ /*
+ * Select a patient to check the heartbeat of. If there's no
+ * other online CPU, nothing to do.
+ */
+ patient = select_patient();
+ if (patient == NULL)
+ return;
+
+ /*
+ * Verify that time is advancing on the patient CPU. If the
+ * delta exceeds UINT_MAX/2, that means it is already ahead by
+ * a little on the other CPU, and the subtraction went
+ * negative, which is OK. If the CPU has been
+ * offlined since we selected it, no worries.
+ *
+ * This uses the current CPU to ensure the other CPU has made
+ * progress, even if the other CPU's hard timer interrupt
+ * handler is stuck for some reason.
+ *
+ * XXX Maybe confirm it hasn't gone negative by more than
+ * max_period?
+ */
+ d = uptime - atomic_load_relaxed(&patient->ci_heartbeat_uptime_cache);
+ if (__predict_false(d > period_secs) &&
+ __predict_false(d < UINT_MAX/2) &&
+ ((patient->ci_schedstate.spc_flags & SPCF_OFFLINE) == 0))
+ defibrillate(patient, d);
+}
+
+/*
+ * heartbeat_dump()
+ *
+ * Print the heartbeat data of all CPUs. Can be called from ddb.
+ */
+#ifdef DDB
+static unsigned
+db_read_unsigned(const unsigned *p)
+{
+ unsigned x;
+
+ db_read_bytes((db_addr_t)p, sizeof(x), (char *)&x);
+
+ return x;
+}
+
+void
+heartbeat_dump(void)
+{
+ struct cpu_info *ci;
+
+ db_printf("Heartbeats:\n");
+ for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) {
+ db_printf("cpu%u: count %u uptime %u stamp %u\n",
+ db_read_unsigned(&ci->ci_index),
+ db_read_unsigned(&ci->ci_heartbeat_count),
+ db_read_unsigned(&ci->ci_heartbeat_uptime_cache),
+ db_read_unsigned(&ci->ci_heartbeat_uptime_stamp));
+ }
+}
+#endif
diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h
index 4c2c6fe3d76..80341ff55fb 100644
--- a/sys/sys/cpu_data.h
+++ b/sys/sys/cpu_data.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_data.h,v 1.52 2020/06/14 21:41:42 ad Exp $ */
+/* $NetBSD: cpu_data.h,v 1.53 2023/07/07 12:34:50 riastradh Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -188,6 +188,11 @@ struct cpu_data {
kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */
struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT];
int64_t cpu_counts[CPU_COUNT_MAX];/* per-CPU counts */
+
+ unsigned cpu_heartbeat_count; /* # of heartbeats */
+ unsigned cpu_heartbeat_uptime_cache; /* last time_uptime */
+ unsigned cpu_heartbeat_uptime_stamp; /* heartbeats since
+ * uptime changed */
};
#define ci_schedstate ci_data.cpu_schedstate
@@ -216,6 +221,10 @@ struct cpu_data {
#define ci_faultrng ci_data.cpu_faultrng
#define ci_counts ci_data.cpu_counts
+#define ci_heartbeat_count ci_data.cpu_heartbeat_count
+#define ci_heartbeat_uptime_cache ci_data.cpu_heartbeat_uptime_cache
+#define ci_heartbeat_uptime_stamp ci_data.cpu_heartbeat_uptime_stamp
+
#define cpu_nsyscall cpu_counts[CPU_COUNT_NSYSCALL]
#define cpu_ntrap cpu_counts[CPU_COUNT_NTRAP]
#define cpu_nswtch cpu_counts[CPU_COUNT_NSWTCH]
diff --git a/sys/sys/heartbeat.h b/sys/sys/heartbeat.h
new file mode 100644
index 00000000000..ef32feaf5ec
--- /dev/null
+++ b/sys/sys/heartbeat.h
@@ -0,0 +1,53 @@
+/* $NetBSD: heartbeat.h,v 1.1 2023/07/07 12:34:50 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2023 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SYS_HEARTBEAT_H
+#define _SYS_HEARTBEAT_H
+
+#ifndef _KERNEL
+#error No user-serviceable parts in this heart.
+#endif
+
+#ifdef _KERNEL_OPT
+#include "opt_heartbeat.h"
+#endif
+
+#ifdef HEARTBEAT
+
+void heartbeat_start(void);
+
+void heartbeat(void);
+
+void heartbeat_suspend(void);
+void heartbeat_resume(void);
+
+void heartbeat_dump(void);
+
+#endif
+
+#endif /* _SYS_HEARTBEAT_H */