/*-
* Copyright (c) 2023 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* heartbeat(9) -- periodic checks to ensure CPUs are making progress
*
* Manual tests to run when changing this file. Magic numbers are for
* evbarm; adjust for other platforms. Tests involving cpuctl
* online/offline assume a 2-CPU system -- for full testing on a >2-CPU
* system, offline all but one CPU.
*
* 1. cpuctl offline 0
* sleep 20
* cpuctl online 0
*
* 2. cpuctl offline 1
* sleep 20
* cpuctl online 1
*
* 3. cpuctl offline 0
* sysctl -w kern.heartbeat.max_period=5
* sleep 10
* sysctl -w kern.heartbeat.max_period=0
* sleep 10
* sysctl -w kern.heartbeat.max_period=5
* sleep 10
* cpuctl online 0
*
* 4. sysctl -w debug.crashme_enable=1
* sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK
* # verify system panics after 15sec, with a stack trace through
* # crashme_spl_spinout
*
* 5. sysctl -w debug.crashme_enable=1
* sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED
* # verify system panics after 15sec, with a stack trace through
* # crashme_spl_spinout
*
* 6. cpuctl offline 0
* sysctl -w debug.crashme_enable=1
* sysctl -w debug.crashme.spl_spinout=1 # IPL_SOFTCLOCK
* # verify system panics after 15sec, with a stack trace through
* # crashme_spl_spinout
*
* 7. cpuctl offline 0
* sysctl -w debug.crashme_enable=1
* sysctl -w debug.crashme.spl_spinout=5 # IPL_VM
* # verify system panics after 15sec, with a stack trace through
* # crashme_spl_spinout
*
* # Not this -- IPL_SCHED and IPL_HIGH spinout on a single CPU
* # require a hardware watchdog timer.
* #cpuctl offline 0
* #sysctl -w debug.crashme_enable
* #sysctl -w debug.crashme.spl_spinout=6 # IPL_SCHED
* # hope watchdog timer kicks in
*/
/*
* Global state.
*
* heartbeat_lock serializes access to heartbeat_max_period_secs
* and heartbeat_max_period_ticks. Two separate variables so we
* can avoid multiplication or division in the heartbeat routine.
*
* heartbeat_sih is stable after initialization in
* heartbeat_start.
*/
kmutex_t heartbeat_lock __cacheline_aligned;
unsigned heartbeat_max_period_secs __read_mostly;
unsigned heartbeat_max_period_ticks __read_mostly;
void *heartbeat_sih __read_mostly;
/*
* heartbeat_suspend()
*
* Suspend heartbeat monitoring of the current CPU.
*
* Called after the current CPU has been marked offline but before
* it has stopped running, or after IPL has been raised for
* polling-mode console input. Nestable (but only 2^32 times, so
* don't do this in a loop). Reversed by heartbeat_resume.
*
* Caller must be bound to the CPU, i.e., curcpu_stable() must be
* true. This function does not assert curcpu_stable() since it
* is used in the ddb entry path, where any assertions risk
* infinite regress into undebuggable chaos, so callers must be
* careful.
*/
void
heartbeat_suspend(void)
{
unsigned *p;
p = &curcpu()->ci_heartbeat_suspend;
atomic_store_relaxed(p, *p + 1);
}
/*
* heartbeat_resume_cpu(ci)
*
* Resume heartbeat monitoring of ci.
*
* Called at startup while cold, and whenever heartbeat monitoring
* is re-enabled after being disabled or the period is changed.
* When not cold, ci must be the current CPU.
*
* Must be run at splsched.
*/
static void
heartbeat_resume_cpu(struct cpu_info *ci)
{
/*
* heartbeat_resume()
*
* Resume heartbeat monitoring of the current CPU.
*
* Called after the current CPU has started running but before it
* has been marked online, or when ending polling-mode input
* before IPL is restored. Reverses heartbeat_suspend.
*
* Caller must be bound to the CPU, i.e., curcpu_stable() must be
* true.
*/
void
heartbeat_resume(void)
{
struct cpu_info *ci = curcpu();
unsigned *p;
int s;
KASSERT(curcpu_stable());
/*
* Reset the state so nobody spuriously thinks we had a heart
* attack as soon as the heartbeat checks resume.
*/
s = splsched();
heartbeat_resume_cpu(ci);
splx(s);
p = &ci->ci_heartbeat_suspend;
atomic_store_relaxed(p, *p - 1);
}
/*
* heartbeat_timecounter_suspended()
*
* True if timecounter heartbeat checks are suspended because the
* timecounter may not be advancing, false if heartbeat checks
* should check for timecounter progress.
*/
static bool
heartbeat_timecounter_suspended(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
/*
* The timecounter ticks only on the primary CPU. Check
* whether it's suspended.
*
* XXX Would be nice if we could find the primary CPU without
* iterating over all CPUs.
*/
for (CPU_INFO_FOREACH(cii, ci)) {
if (CPU_IS_PRIMARY(ci))
return atomic_load_relaxed(&ci->ci_heartbeat_suspend);
}
/*
* This should be unreachable -- there had better be a primary
* CPU in the system! If not, the timecounter will be busted
* anyway.
*/
panic("no primary CPU");
}
/*
* heartbeat_reset_xc(a, b)
*
* Cross-call handler to reset heartbeat state just prior to
* enabling heartbeat checks.
*/
static void
heartbeat_reset_xc(void *a, void *b)
{
int s;
s = splsched();
heartbeat_resume_cpu(curcpu());
splx(s);
}
/*
* set_max_period(max_period)
*
* Set the maximum period, in seconds, for heartbeat checks.
*
* - If max_period is zero, disable them.
*
* - If the max period was zero and max_period is nonzero, ensure
* all CPUs' heartbeat uptime caches are up-to-date before
* re-enabling them.
*
* max_period must be below UINT_MAX/4/hz to avoid arithmetic
* overflow and give room for slop.
*
* Caller must hold heartbeat_lock.
*/
static void
set_max_period(unsigned max_period)
{
KASSERTMSG(max_period <= UINT_MAX/4/hz,
"max_period=%u must not exceed UINT_MAX/4/hz=%u (hz=%u)",
max_period, UINT_MAX/4/hz, hz);
KASSERT(mutex_owned(&heartbeat_lock));
/*
* If we're enabling heartbeat checks, make sure we have a
* reasonably up-to-date time_uptime32 cache on all CPUs so we
* don't think we had an instant heart attack.
*/
if (heartbeat_max_period_secs == 0 && max_period != 0) {
if (cold) {
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
/*
* Once the heartbeat state has been updated on all (online)
* CPUs, set the period. At this point, heartbeat checks can
* begin.
*/
atomic_store_relaxed(&heartbeat_max_period_secs, max_period);
atomic_store_relaxed(&heartbeat_max_period_ticks, max_period*hz);
}
/*
* heartbeat_max_period_ticks(SYSCTLFN_ARGS)
*
* Sysctl handler for sysctl kern.heartbeat.max_period. Verifies
* it lies within a reasonable interval and sets it.
*/
static int
heartbeat_max_period_sysctl(SYSCTLFN_ARGS)
{
struct sysctlnode node;
unsigned max_period;
int error;
/*
* Ensure there's plenty of slop between heartbeats.
*/
if (max_period > UINT_MAX/4/hz) {
error = EOVERFLOW;
goto out;
}
/*
* Success! Set the period. This enables heartbeat checks if
* we went from zero period to nonzero period, or disables them
* if the other way around.
*/
set_max_period(max_period);
error = 0;
out: mutex_exit(&heartbeat_lock);
return error;
}
/*
* sysctl_heartbeat_setup()
*
* Set up the kern.heartbeat.* sysctl subtree.
*/
SYSCTL_SETUP(sysctl_heartbeat_setup, "sysctl kern.heartbeat setup")
{
const struct sysctlnode *rnode;
int error;
/* kern.heartbeat.max_period */
error = sysctl_createv(NULL, 0, &rnode, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
CTLTYPE_INT, "max_period",
SYSCTL_DESCR("Max seconds between heartbeats before panic"),
&heartbeat_max_period_sysctl, 0, NULL, 0,
CTL_CREATE, CTL_EOL);
if (error) {
printf("%s: failed to create kern.heartbeat.max_period: %d\n",
__func__, error);
return;
}
}
/*
* heartbeat_intr(cookie)
*
* Soft interrupt handler to update the local CPU's view of the
* system uptime. This runs at the same priority level as
* callouts, so if callouts are stuck on this CPU, it won't run,
* and eventually another CPU will notice that this one is stuck.
*
* Don't do spl* here -- keep it to a minimum so if anything goes
* wrong we don't end up with hard interrupts blocked and unable
* to detect a missed heartbeat.
*/
static void
heartbeat_intr(void *cookie)
{
unsigned count = atomic_load_relaxed(&curcpu()->ci_heartbeat_count);
unsigned uptime = time_uptime32;
/*
* Ensure the maximum period is small enough that we never have
* to worry about 32-bit wraparound even if there's a lot of
* slop. (In fact this is required to be less than
* UINT_MAX/4/hz, but that's not a compile-time constant.)
*/
__CTASSERT(max_period < UINT_MAX/4);
/*
* Establish a softint so we can schedule it once ready. This
* should be at the lowest softint priority level so that we
* ensure all softint priorities are making progress.
*/
heartbeat_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE,
&heartbeat_intr, NULL);
/*
* Now that the softint is established, kick off heartbeat
* monitoring with the default period. This will initialize
* the per-CPU state to an up-to-date cache of time_uptime32.
*/
mutex_enter(&heartbeat_lock);
set_max_period(max_period);
mutex_exit(&heartbeat_lock);
}
/*
* defibrillator(cookie)
*
* IPI handler for defibrillation. If the CPU's heart has stopped
* beating normally, but the CPU can still execute things,
* acknowledge the IPI to the doctor and then panic so we at least
* get a stack trace from whatever the current CPU is stuck doing,
* if not a core dump.
*
* (This metaphor is a little stretched, since defibrillation is
* usually administered when the heart is beating errattically but
* hasn't stopped, and causes the heart to stop temporarily, and
* one hopes it is not fatal. But we're (software) engineers, so
* we can stretch metaphors like silly putty in a blender.)
*/
static void
defibrillator(void *cookie)
{
bool *ack = cookie;
/*
* Acknowledge the interrupt so the doctor CPU won't trigger a
* new panic for defibrillation timeout.
*/
atomic_store_relaxed(ack, true);
/*
* If a panic is already in progress, we may have interrupted
* the logic that prints a stack trace on this CPU -- so let's
* not make it worse by giving the misapprehension of a
* recursive panic.
*/
if (atomic_load_relaxed(&panicstr) != NULL)
return;
/*
* defibrillate(ci, unsigned d)
*
* The patient CPU ci's heart has stopped beating after d seconds.
* Force the patient CPU ci to panic, or panic on this CPU if the
* patient CPU doesn't respond within 1sec.
*/
static void __noinline
defibrillate(struct cpu_info *ci, unsigned d)
{
bool ack = false;
ipi_msg_t msg = {
.func = &defibrillator,
.arg = &ack,
};
unsigned countdown = 1000; /* 1sec */
KASSERT(curcpu_stable());
/*
* First notify the console that the patient CPU's heart seems
* to have stopped beating.
*/
printf("%s: found %s heart stopped beating after %u seconds\n",
cpu_name(curcpu()), cpu_name(ci), d);
/*
* Next, give the patient CPU a chance to panic, so we get a
* stack trace on that CPU even if we don't get a crash dump.
*/
ipi_unicast(&msg, ci);
/*
* Busy-wait up to 1sec for the patient CPU to print a stack
* trace and panic. If the patient CPU acknowledges the IPI,
* just give up and stop here -- the system is coming down soon
* and we should avoid getting in the way.
*/
while (countdown --> 0) {
if (atomic_load_relaxed(&ack))
return;
DELAY(1000); /* 1ms */
}
/*
* The patient CPU failed to acknowledge the panic request.
* Panic now; with any luck, we'll get a crash dump.
*/
panic("%s: found %s heart stopped beating and unresponsive",
cpu_name(curcpu()), cpu_name(ci));
}
/*
* select_patient()
*
* Select another CPU to check the heartbeat of. Returns NULL if
* there are no other online CPUs. Never returns curcpu().
* Caller must have kpreemption disabled.
*/
static struct cpu_info *
select_patient(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *first = NULL, *patient = NULL, *ci;
bool passedcur = false;
KASSERT(curcpu_stable());
/*
* In the iteration order of all CPUs, find the next online CPU
* after curcpu(), or the first online one if curcpu() is last
* in the iteration order.
*/
for (CPU_INFO_FOREACH(cii, ci)) {
if (atomic_load_relaxed(&ci->ci_heartbeat_suspend))
continue;
if (passedcur) {
/*
* (...|curcpu()|ci|...)
*
* Found the patient right after curcpu().
*/
KASSERT(patient != ci);
patient = ci;
break;
}
if (ci == curcpu()) {
/*
* (...|prev|ci=curcpu()|next|...)
*
* Note that we want next (or first, if there's
* nothing after curcpu()).
*/
passedcur = true;
continue;
}
if (first == NULL) {
/*
* (ci|...|curcpu()|...)
*
* Record ci as first in case there's nothing
* after curcpu().
*/
first = ci;
continue;
}
}
/*
* If we hit the end, wrap around to the beginning.
*/
if (patient == NULL) {
KASSERT(passedcur);
patient = first;
}
return patient;
}
/*
* heartbeat()
*
* 1. Count a heartbeat on the local CPU.
*
* 2. Panic if the system uptime doesn't seem to have advanced in
* a while.
*
* 3. Panic if the soft interrupt on this CPU hasn't advanced the
* local view of the system uptime.
*
* 4. Schedule the soft interrupt to advance the local view of the
* system uptime.
*
* 5. Select another CPU to check the heartbeat of.
*
* 6. Panic if the other CPU hasn't advanced its view of the
* system uptime in a while.
*/
void
heartbeat(void)
{
unsigned period_ticks, period_secs;
unsigned count, uptime, cache, stamp, d;
struct cpu_info *patient;
KASSERT(curcpu_stable());
/*
* If heartbeat checks are disabled globally, or if they are
* suspended locally, or if we're already panicking so it's not
* helpful to trigger more panics for more reasons, do nothing.
*/
period_ticks = atomic_load_relaxed(&heartbeat_max_period_ticks);
period_secs = atomic_load_relaxed(&heartbeat_max_period_secs);
if (__predict_false(period_ticks == 0) ||
__predict_false(period_secs == 0) ||
__predict_false(curcpu()->ci_heartbeat_suspend) ||
__predict_false(panicstr != NULL))
return;
/*
* Count a heartbeat on this CPU.
*/
count = curcpu()->ci_heartbeat_count++;
/*
* If the uptime hasn't changed, make sure that we haven't
* counted too many of our own heartbeats since the uptime last
* changed, and stop here -- we only do the cross-CPU work once
* per second.
*/
uptime = time_uptime32;
cache = atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_cache);
if (__predict_true(cache == uptime)) {
/*
* Timecounter hasn't advanced by more than a second.
* Make sure the timecounter isn't stuck according to
* our heartbeats -- unless timecounter heartbeats are
* suspended too.
*
* Our own heartbeat count can't roll back, and
* time_uptime32 should be updated before it wraps
* around, so d should never go negative; hence no
* check for d < UINT_MAX/2.
*/
stamp =
atomic_load_relaxed(&curcpu()->ci_heartbeat_uptime_stamp);
d = count - stamp;
if (__predict_false(d > period_ticks) &&
!heartbeat_timecounter_suspended()) {
panic("%s: time has not advanced in %u heartbeats",
cpu_name(curcpu()), d);
}
return;
}
/*
* If the uptime has changed, make sure that it hasn't changed
* so much that softints must be stuck on this CPU. Since
* time_uptime32 is monotonic and our cache of it is updated at
* most every UINT_MAX/4/hz sec (hence no concern about
* wraparound even after 68 or 136 years), this can't go
* negative, hence no check for d < UINT_MAX/2.
*
* This uses the hard timer interrupt handler on the current
* CPU to ensure soft interrupts at all priority levels have
* made progress.
*/
d = uptime - cache;
if (__predict_false(d > period_secs)) {
panic("%s: softints stuck for %u seconds",
cpu_name(curcpu()), d);
}
/*
* Schedule a softint to update our cache of the system uptime
* so the next call to heartbeat, on this or another CPU, can
* detect progress on this one.
*/
softint_schedule(heartbeat_sih);
/*
* Select a patient to check the heartbeat of. If there's no
* other online CPU, nothing to do.
*/
patient = select_patient();
if (patient == NULL)
return;
/*
* Verify that time is advancing on the patient CPU. If the
* delta exceeds UINT_MAX/2, that means it is already ahead by
* a little on the other CPU, and the subtraction went
* negative, which is OK. If the CPU's heartbeats have been
* suspended since we selected it, no worries.
*
* This uses the current CPU to ensure the other CPU has made
* progress, even if the other CPU's hard timer interrupt
* handler is stuck for some reason.
*
* XXX Maybe confirm it hasn't gone negative by more than
* max_period?
*/
d = uptime - atomic_load_relaxed(&patient->ci_heartbeat_uptime_cache);
if (__predict_false(d > period_secs) &&
__predict_false(d < UINT_MAX/2) &&
atomic_load_relaxed(&patient->ci_heartbeat_suspend) == 0)
defibrillate(patient, d);
}
/*
* heartbeat_dump()
*
* Print the heartbeat data of all CPUs. Can be called from ddb.
*/
#ifdef DDB
static unsigned
db_read_unsigned(const volatile unsigned *p)
{
unsigned x;
