diff options
Diffstat (limited to 'sys/kern/kern_heartbeat.c')
| -rw-r--r-- | sys/kern/kern_heartbeat.c | 656 |
1 files changed, 656 insertions, 0 deletions
diff --git a/sys/kern/kern_heartbeat.c b/sys/kern/kern_heartbeat.c new file mode 100644 index 00000000000..732e13ade75 --- /dev/null +++ b/sys/kern/kern_heartbeat.c @@ -0,0 +1,656 @@ +/* $NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $ */ + +/*- + * Copyright (c) 2023 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * heartbeat(9) -- periodic checks to ensure CPUs are making progress + * + * Manual tests to run when changing this file. Magic numbers are for + * evbarm; adjust for other platforms. Tests involving cpuctl + * online/offline assume a 2-CPU system -- for full testing on a >2-CPU + * system, offline all but one CPU. + * + * 1. cpuctl offline 0 + * sleep 20 + * cpuctl online 0 + * + * 2. cpuctl offline 1 + * sleep 20 + * cpuctl online 1 + * + * 3. cpuctl offline 0 + * sysctl -w kern.heartbeat.max_period=5 + * sleep 10 + * sysctl -w kern.heartbeat.max_period=0 + * sleep 10 + * sysctl -w kern.heartbeat.max_period=5 + * sleep 10 + * cpuctl online 0 + * + * 4. sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK + * # verify system panics after 15sec + * + * 5. sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED + * # verify system panics after 15sec + * + * 6. cpuctl offline 0 + * sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK + * # verify system panics after 15sec + * + * 7. cpuctl offline 0 + * sysctl -w debug.crashme_enable=1 + * sysctl -w debug.crashme.spl_spinout=5 # IPL_VM + * # verify system panics after 15sec + * + * # Not this -- IPL_SCHED and IPL_HIGH spinout on a single CPU + * # require a hardware watchdog timer. + * #cpuctl offline 0 + * #sysctl -w debug.crashme_enable + * #sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED + * # hope watchdog timer kicks in + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(0, "$NetBSD: kern_heartbeat.c,v 1.1 2023/07/07 12:34:50 riastradh Exp $"); + +#ifdef _KERNEL_OPT +#include "opt_ddb.h" +#include "opt_heartbeat.h" +#endif + +#include "heartbeat.h" + +#include <sys/param.h> +#include <sys/types.h> + +#include <sys/atomic.h> +#include <sys/cpu.h> +#include <sys/errno.h> +#include <sys/heartbeat.h> +#include <sys/ipi.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <sys/systm.h> +#include <sys/xcall.h> + +#ifdef DDB +#include <ddb/ddb.h> +#endif + +/* + * Global state. + * + * heartbeat_lock serializes access to heartbeat_max_period_secs + * and heartbeat_max_period_ticks. Two separate variables so we + * can avoid multiplication or division in the heartbeat routine. + * + * heartbeat_sih is stable after initialization in + * heartbeat_start. + */ +kmutex_t heartbeat_lock __cacheline_aligned; +unsigned heartbeat_max_period_secs __read_mostly; +unsigned heartbeat_max_period_ticks __read_mostly; + +void *heartbeat_sih __read_mostly; + +/* + * heartbeat_suspend() + * + * Suspend heartbeat monitoring of the current CPU. + * + * Called after the current CPU has been marked offline but before + * it has stopped running. Caller must have preemption disabled. + */ +void +heartbeat_suspend(void) +{ + + KASSERT(kpreempt_disabled()); + + /* + * Nothing to do -- we just check the SPCF_OFFLINE flag. + */ +} + +/* + * heartbeat_resume() + * + * Resume heartbeat monitoring of the current CPU. + * + * Called after the current CPU has started running but before it + * has been marked online. Also used internally when starting up + * heartbeat monitoring at boot or when the maximum period is set + * from zero to nonzero. Caller must have preemption disabled. + */ +void +heartbeat_resume(void) +{ + struct cpu_info *ci = curcpu(); + int s; + + KASSERT(kpreempt_disabled()); + + /* + * Block heartbeats while we reset the state so we don't + * spuriously think we had a heart attack in the middle of + * resetting the count and the uptime stamp. + */ + s = splsched(); + ci->ci_heartbeat_count = 0; + ci->ci_heartbeat_uptime_cache = atomic_load_relaxed(&time_uptime); + ci->ci_heartbeat_uptime_stamp = 0; + splx(s); +} + +/* + * heartbeat_reset_xc(a, b) + * + * Cross-call handler to reset heartbeat state just prior to + * enabling heartbeat checks. + */ +static void +heartbeat_reset_xc(void *a, void *b) +{ + + heartbeat_resume(); +} + +/* + * set_max_period(max_period) + * + * Set the maximum period, in seconds, for heartbeat checks. + * + * - If max_period is zero, disable them. + * + * - If the max period was zero and max_period is nonzero, ensure + * all CPUs' heartbeat uptime caches are up-to-date before + * re-enabling them. + * + * max_period must be below UINT_MAX/4/hz to avoid arithmetic + * overflow and give room for slop. + * + * Caller must hold heartbeat_lock. + */ +static void +set_max_period(unsigned max_period) +{ + + KASSERTMSG(max_period <= UINT_MAX/4/hz, + "max_period=%u must not exceed UINT_MAX/4/hz=%u (hz=%u)", + max_period, UINT_MAX/4/hz, hz); + KASSERT(mutex_owned(&heartbeat_lock)); + + /* + * If we're enabling heartbeat checks, make sure we have a + * reasonably up-to-date time_uptime cache on all CPUs so we + * don't think we had an instant heart attack. + */ + if (heartbeat_max_period_secs == 0 && max_period != 0) + xc_wait(xc_broadcast(0, &heartbeat_reset_xc, NULL, NULL)); + + /* + * Once the heartbeat state has been updated on all (online) + * CPUs, set the period. At this point, heartbeat checks can + * begin. + */ + atomic_store_relaxed(&heartbeat_max_period_secs, max_period); + atomic_store_relaxed(&heartbeat_max_period_ticks, max_period*hz); +} + +/* + * heartbeat_max_period_ticks(SYSCTLFN_ARGS) + * + * Sysctl handler for sysctl kern.heartbeat.max_period. Verifies + * it lies within a reasonable interval and sets it. + */ +static int +heartbeat_max_period_sysctl(SYSCTLFN_ARGS) +{ + struct sysctlnode node; + unsigned max_period; + int error; + + mutex_enter(&heartbeat_lock); + + max_period = heartbeat_max_period_secs; + node = *rnode; + node.sysctl_data = &max_period; + error = sysctl_lookup(SYSCTLFN_CALL(&node)); + if (error || newp == NULL) + goto out; + + /* + * Ensure there's plenty of slop between heartbeats. + */ + if (max_period > UINT_MAX/4/hz) { + error = EOVERFLOW; + goto out; + } + + /* + * Success! Set the period. This enables heartbeat checks if + * we went from zero period to nonzero period, or disables them + * if the other way around. + */ + set_max_period(max_period); + error = 0; + +out: mutex_exit(&heartbeat_lock); + return error; +} + +/* + * sysctl_heartbeat_setup() + * + * Set up the kern.heartbeat.* sysctl subtree. + */ +SYSCTL_SETUP(sysctl_heartbeat_setup, "sysctl kern.heartbeat setup") +{ + const struct sysctlnode *rnode; + int error; + + mutex_init(&heartbeat_lock, MUTEX_DEFAULT, IPL_NONE); + + /* kern.heartbeat */ + error = sysctl_createv(NULL, 0, NULL, &rnode, + CTLFLAG_PERMANENT, + CTLTYPE_NODE, "heartbeat", + SYSCTL_DESCR("Kernel heartbeat parameters"), + NULL, 0, NULL, 0, + CTL_KERN, CTL_CREATE, CTL_EOL); + if (error) { + printf("%s: failed to create kern.heartbeat: %d\n", + __func__, error); + return; + } + + /* kern.heartbeat.max_period */ + error = sysctl_createv(NULL, 0, &rnode, NULL, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, + CTLTYPE_INT, "max_period", + SYSCTL_DESCR("Max seconds between heartbeats before panic"), + &heartbeat_max_period_sysctl, 0, NULL, 0, + CTL_CREATE, CTL_EOL); + if (error) { + printf("%s: failed to create kern.heartbeat.max_period: %d\n", + __func__, error); + return; + } +} + +/* + * heartbeat_intr(cookie) + * + * Soft interrupt handler to update the local CPU's view of the + * system uptime. This runs at the same priority level as + * callouts, so if callouts are stuck on this CPU, it won't run, + * and eventually another CPU will notice that this one is stuck. + * + * Don't do spl* here -- keep it to a minimum so if anything goes + * wrong we don't end up with hard interrupts blocked and unable + * to detect a missed heartbeat. + */ +static void +heartbeat_intr(void *cookie) +{ + unsigned count = atomic_load_relaxed(&curcpu()->ci_heartbeat_count); + unsigned uptime = atomic_load_relaxed(&time_uptime); + + atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_stamp, count); + atomic_store_relaxed(&curcpu()->ci_heartbeat_uptime_cache, uptime); +} + +/* + * heartbeat_start() + * + * Start system heartbeat monitoring. + */ +void +heartbeat_start(void) +{ + const unsigned max_period = HEARTBEAT_MAX_PERIOD_DEFAULT; + + /* + * Establish a softint so we can schedule it once ready. This + * should be at the lowest softint priority level so that we + * ensure all softint priorities are making progress. + */ + heartbeat_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, + &heartbeat_intr, NULL); + + /* + * Now that the softint is established, kick off heartbeat + * monitoring with the default period. This will initialize + * the per-CPU state to an up-to-date cache of time_uptime. + */ + mutex_enter(&heartbeat_lock); + set_max_period(max_period); + mutex_exit(&heartbeat_lock); +} + +/* + * defibrillator(cookie) + * + * IPI handler for defibrillation. If the CPU's heart has stopped + * beating normally, but the CPU can still execute things, + * acknowledge the IPI to the doctor and then panic so we at least + * get a stack trace from whatever the current CPU is stuck doing, + * if not a core dump. + * + * (This metaphor is a little stretched, since defibrillation is + * usually administered when the heart is beating errattically but + * hasn't stopped, and causes the heart to stop temporarily, and + * one hopes it is not fatal. But we're (software) engineers, so + * we can stretch metaphors like silly putty in a blender.) + */ +static void +defibrillator(void *cookie) +{ + bool *ack = cookie; + + atomic_store_relaxed(ack, true); + panic("%s[%d %s]: heart stopped beating", cpu_name(curcpu()), + curlwp->l_lid, + curlwp->l_name ? curlwp->l_name : curproc->p_comm); +} + +/* + * defibrillate(ci, unsigned d) + * + * The patient CPU ci's heart has stopped beating after d seconds. + * Force the patient CPU ci to panic, or panic on this CPU if the + * patient CPU doesn't respond within 1sec. + */ +static void __noinline +defibrillate(struct cpu_info *ci, unsigned d) +{ + bool ack = false; + ipi_msg_t msg = { + .func = &defibrillator, + .arg = &ack, + }; + unsigned countdown = 1000; /* 1sec */ + + KASSERT(kpreempt_disabled()); + + /* + * First notify the console that the patient CPU's heart seems + * to have stopped beating. + */ + printf("%s: found %s heart stopped beating after %u seconds\n", + cpu_name(curcpu()), cpu_name(ci), d); + + /* + * Next, give the patient CPU a chance to panic, so we get a + * stack trace on that CPU even if we don't get a crash dump. + */ + ipi_unicast(&msg, ci); + + /* + * Busy-wait up to 1sec for the patient CPU to print a stack + * trace and panic. If the patient CPU acknowledges the IPI, + * or if we're panicking anyway, just give up and stop here -- + * the system is coming down soon and we should avoid getting + * in the way. + */ + while (countdown --> 0) { + if (atomic_load_relaxed(&ack) || + atomic_load_relaxed(&panicstr) != NULL) + return; + DELAY(1000); /* 1ms */ + } + + /* + * The patient CPU failed to acknowledge the panic request. + * Panic now; with any luck, we'll get a crash dump. + */ + panic("%s: found %s heart stopped beating and unresponsive", + cpu_name(curcpu()), cpu_name(ci)); +} + +/* + * select_patient() + * + * Select another CPU to check the heartbeat of. Returns NULL if + * there are no other online CPUs. Never returns curcpu(). + * Caller must have kpreemption disabled. + */ +static struct cpu_info * +select_patient(void) +{ + CPU_INFO_ITERATOR cii; + struct cpu_info *first = NULL, *patient = NULL, *ci; + bool passedcur = false; + + KASSERT(kpreempt_disabled()); + + /* + * In the iteration order of all CPUs, find the next online CPU + * after curcpu(), or the first online one if curcpu() is last + * in the iteration order. + */ + for (CPU_INFO_FOREACH(cii, ci)) { + if (ci->ci_schedstate.spc_flags & SPCF_OFFLINE) + continue; + if (passedcur) { + /* + * (...|curcpu()|ci|...) + * + * Found the patient right after curcpu(). + */ + KASSERT(patient != ci); + patient = ci; + break; + } + if (ci == curcpu()) { + /* + * (...|prev|ci=curcpu()|next|...) + * + * Note that we want next (or first, if there's + * nothing after curcpu()). + */ + passedcur = true; + continue; + } + if (first == NULL) { + /* + * (ci|...|curcpu()|...) + * + * Record ci as first in case there's nothing + * after curcpu(). + */ + first = ci; + continue; + } + } + + /* + * If we hit the end, wrap around to the beginning. + */ + if (patient == NULL) { + KASSERT(passedcur); + patient = first; + } + + return patient; +} + +/* + * heartbeat() + * + * 1. Count a heartbeat on the local CPU. + * + * 2. Panic if the system uptime doesn't seem to have advanced in + * a while. + * + * 3. Panic if the soft interrupt on this CPU hasn't advanced the + * local view of the system uptime. + * + * 4. Schedule the soft interrupt to advance the local view of the + * system uptime. + * + * 5. Select another CPU to check the heartbeat of. + * + * 6. Panic if the other CPU hasn't advanced its view of the + * system uptime in a while. + */ +void +heartbeat(void) +{ + unsigned period_ticks, period_secs; + unsigned count, uptime, cache, stamp, d; + struct cpu_info *patient; + + KASSERT(kpreempt_disabled()); + + period_ticks = atomic_load_relaxed(&heartbeat_max_period_ticks); + period_secs = atomic_load_relaxed(&heartbeat_max_period_secs); + if (__predict_false(period_ticks == 0) || + __predict_false(period_secs == 0) || + __predict_false(curcpu()->ci_schedstate.spc_flags & SPCF_OFFLINE)) + return; + + /* + * Count a heartbeat on this CPU. + */ + count = curcpu()->ci_heartbeat_count++; + + /* + * If the uptime hasn't changed, make sure that we haven't + * counted too many of our own heartbeats since the uptime last + * changed, and stop here -- we only do the cross-CPU work once + * per second. + */ + uptime = atomic_load_relaxed(&time_uptime); + cache = atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_cache); + if (__predict_true(cache == uptime)) { + /* + * Timecounter hasn't advanced by more than a second. + * Make sure the timecounter isn't stuck according to + * our heartbeats. + * + * Our own heartbeat count can't roll back, and + * time_uptime should be updated before it wraps + * around, so d should never go negative; hence no + * check for d < UINT_MAX/2. + */ + stamp = + atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_stamp); + d = count - stamp; + if (__predict_false(d > period_ticks)) { + panic("%s: time has not advanced in %u heartbeats", + cpu_name(curcpu()), d); + } + return; + } + + /* + * If the uptime has changed, make sure that it hasn't changed + * so much that softints must be stuck on this CPU. Since + * time_uptime is monotonic, this can't go negative, hence no + * check for d < UINT_MAX/2. + * + * This uses the hard timer interrupt handler on the current + * CPU to ensure soft interrupts at all priority levels have + * made progress. + */ + d = uptime - cache; + if (__predict_false(d > period_secs)) { + panic("%s: softints stuck for %u seconds", + cpu_name(curcpu()), d); + } + + /* + * Schedule a softint to update our cache of the system uptime + * so the next call to heartbeat, on this or another CPU, can + * detect progress on this one. + */ + softint_schedule(heartbeat_sih); + + /* + * Select a patient to check the heartbeat of. If there's no + * other online CPU, nothing to do. + */ + patient = select_patient(); + if (patient == NULL) + return; + + /* + * Verify that time is advancing on the patient CPU. If the + * delta exceeds UINT_MAX/2, that means it is already ahead by + * a little on the other CPU, and the subtraction went + * negative, which is OK. If the CPU has been + * offlined since we selected it, no worries. + * + * This uses the current CPU to ensure the other CPU has made + * progress, even if the other CPU's hard timer interrupt + * handler is stuck for some reason. + * + * XXX Maybe confirm it hasn't gone negative by more than + * max_period? + */ + d = uptime - atomic_load_relaxed(&patient->ci_heartbeat_uptime_cache); + if (__predict_false(d > period_secs) && + __predict_false(d < UINT_MAX/2) && + ((patient->ci_schedstate.spc_flags & SPCF_OFFLINE) == 0)) + defibrillate(patient, d); +} + +/* + * heartbeat_dump() + * + * Print the heartbeat data of all CPUs. Can be called from ddb. + */ +#ifdef DDB +static unsigned +db_read_unsigned(const unsigned *p) +{ + unsigned x; + + db_read_bytes((db_addr_t)p, sizeof(x), (char *)&x); + + return x; +} + +void +heartbeat_dump(void) +{ + struct cpu_info *ci; + + db_printf("Heartbeats:\n"); + for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) { + db_printf("cpu%u: count %u uptime %u stamp %u\n", + db_read_unsigned(&ci->ci_index), + db_read_unsigned(&ci->ci_heartbeat_count), + db_read_unsigned(&ci->ci_heartbeat_uptime_cache), + db_read_unsigned(&ci->ci_heartbeat_uptime_stamp)); + } +} +#endif |
