#!/bin/bash # btx-node-guardian.sh — keep a BTX node from getting quietly stuck. # # For any operator running a trusted-mirror BTX node. Free to copy, change and # redistribute (MIT). Written by the easyBTX team from a week of live incidents on # api.btxscan.io; every rule below is something the network taught us the hard way. # # WHAT IT DOES # Every run: reads the node, decides whether a still tip is a problem, and if it # is, dials the archive peers that can fix it. Writes one JSON file you can watch. # # THE DISTINCTION IT EXISTS FOR # A tip that has not moved is NOT automatically a stall. Two states look identical # from outside and want opposite responses: # * blocks_behind == 0 -> you are AT the signed frontier. The attestor has signed # nothing newer, so there is nothing to fetch. btxd still logs # "matmul trusted mirror stall" once a minute; that is noise. DO NOTHING. # * blocks_behind > 0 -> signed work exists that you have not collected. That is # your stall, and dialling archives is the fix. # Measured 2026-08-19: the network attestor was offline ~100 minutes while GPU # consensus nodes ran 43 blocks ahead on unattested work. Every watchdog without # this distinction cried stall the whole time. # # WHAT IT WILL NEVER DO, AND WHY # * restart or stop your node. A restart discards the peer set that is usually your # only attestation source, and an unclean stop has bricked a snapshot datadir. # Crash recovery belongs to systemd Restart=, which is a different thing from # reacting to a stall. # * touch your datadir, conf, or signer keys. Changing matmultrustedpubkey over an # existing chain makes btxd refuse to start, and the -reindex-chainstate it then # suggests DELETES the snapshot chainstate before failing the same check. One-way # door: a pruned snapshot node cannot full-reindex out of it. # * hammer peers. A signer bans you for 24h after 32 ignored GETMMATTEST, and it # only serves the last 16 blocks anyway — historical scans belong on archives. # # USAGE # BTX_CLI=/path/to/btx-cli BTX_DATADIR=$HOME/.btx ./btx-node-guardian.sh # Run it from cron or a systemd timer every 5 minutes. See README.md. # # CONFIG (environment variables, all optional) # BTX_CLI path to btx-cli (default: btx-cli on PATH) # BTX_DATADIR node datadir (default: ~/.btx) # GUARDIAN_STATE where it keeps its counters (default: ~/.btx-guardian) # GUARDIAN_BEACON the JSON it writes (default: ~/.btx-guardian/health.json) # BTX_ARCHIVES space-separated archive peers (default: the published set) # # Exit code is always 0: a guardian that fails must never look like a service failure. set -u CLI="${BTX_CLI:-btx-cli} -datadir=${BTX_DATADIR:-$HOME/.btx}" STATE_DIR="${GUARDIAN_STATE:-$HOME/.btx-guardian}" LOG="${GUARDIAN_LOG:-$HOME/.btx-guardian/guardian.log}" BEACON="${GUARDIAN_BEACON:-$HOME/.btx-guardian/health.json}" DEBUG_LOG="${BTX_DEBUG_LOG:-${BTX_DATADIR:-$HOME/.btx}/debug.log}" REDIAL_OK_SECS=600 # after a redial that reached the node REDIAL_FAIL_SECS=120 # after one that failed outright (budget not consumed) BEHIND_ALERT_SECS=600 # we lag the frontier this long -> our stall FRONTIER_QUIET_SECS=3600 # the frontier itself has not moved this long -> network-level # A frontier that has not moved this long is already far outside normal and a # user has already noticed, but it is not yet the hour-long silence that # FRONTIER_QUIET_SECS is for. Measured 2026-08-20: the attestor went quiet for # 47m46s between blocks 195,697 and 195,698, the explorer read "last block 48 # minutes ago", a community dev diagnosed it from the outside before any of our # tooling said anything, and the beacon reported `ok` for the whole event # because the pause fell 12 minutes short of the hour. Block cadence is ~55-75s, # so ten minutes of silence is already a ~10x outlier worth naming. FRONTIER_SLOW_SECS=600 # the frontier is quiet but not yet hour-silent ELECTRS_LAG_OK=3 DISK_WARN_PCT=85 DISK_CRIT_PCT=93 ARCHIVES="${BTX_ARCHIVES:-207.56.229.99:19335 185.204.25.227:19335 195.137.245.82:20982 node.btx.dev:19335 node.btxchain.org:19335 node.btx.tools:19335}" export NUDGE_CLI="$CLI" mkdir -p "$STATE_DIR" log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$LOG"; } rd() { cat "$STATE_DIR/$1" 2>/dev/null; } wr() { printf '%s' "$2" > "$STATE_DIR/$1"; } is_uint() { case "${1:-}" in ''|*[!0-9]*) return 1;; *) return 0;; esac; } is_int() { case "${1:-}" in ''|-) return 1;; -*) is_uint "${1#-}";; *) is_uint "$1";; esac; } now=$(date +%s) [ -f "$LOG" ] && [ "$(stat -c%s "$LOG" 2>/dev/null || echo 0)" -gt 5242880 ] && { mv -f "$LOG" "$LOG.1"; : > "$LOG"; } # ── Sample ────────────────────────────────────────────────────────────────────────── blocks=$($CLI getblockcount 2>/dev/null) headers=$($CLI getblockchaininfo 2>/dev/null | grep -o '"headers": *[0-9]*' | grep -o '[0-9]*$') # getmatmulattestedtip is NESTED (signed_frontier.height / .blocks_behind); grep on the # flattened text silently picks the wrong "height" and makes the lag read as 0 forever. # Parse it properly, and take blocks_behind from the node itself rather than recomputing. read -r frontier quorum behind onchain </dev/null | python3 -c ' import json,sys try: d=json.load(sys.stdin) except Exception: print("null null null null"); raise SystemExit sf=d.get("signed_frontier") or {} print(sf.get("height", d.get("active_tip_height","null")), str(d.get("active_tip_has_quorum","null")).lower(), sf.get("blocks_behind","null"), str(sf.get("on_active_chain", d.get("on_active_chain","null"))).lower()) ' 2>/dev/null || echo "null null null null") EOF electrs=$(curl -s --max-time 10 http://127.0.0.1:3000/blocks/tip/height 2>/dev/null) svc_btxd=$(systemctl is-active btxd 2>/dev/null) svc_electrs=$(systemctl is-active electrs 2>/dev/null) disk_pct=$(df --output=pcent "${BTX_DATADIR:-$HOME/.btx}" 2>/dev/null | tail -1 | tr -dc '0-9') read -r n_peers n_arch n_auth n_feed n_served n_inflight </dev/null | python3 -c ' import json,sys try: ps=json.load(sys.stdin) except Exception: print("0 0 0 0 0 0"); raise SystemExit def arch(p): try: return (int(p.get("services","0"),16)>>31)&1 except Exception: return 0 a=[p for p in ps if arch(p)] auth=[p for p in a if p.get("connection_type")=="manual" or "noban" in p.get("permissions",[])] feed=[p for p in a if p.get("bytesrecv_per_msg",{}).get("mmattest",0)>0] served=[p for p in ps if p.get("bytessent_per_msg",{}).get("mmattest",0)>0] # Blocks this node is CURRENTLY ASKING FOR, across every peer. Zero while we # trail the frontier is the scheduler-gated signature (btxchain/btx#112): the # node can see the blocks and is requesting none of them, which no amount of # redialling changes. Counted over all peers, not just archives. infl=sum(len(p.get("inflight",[]) or []) for p in ps) print(len(ps),len(a),len(auth),len(feed),len(served),infl) ' 2>/dev/null || echo "0 0 0 0 0 0") EOF # ── Track two clocks: our tip, and the frontier ───────────────────────────────────── if is_uint "$blocks"; then [ "$(rd blocks)" != "$blocks" ] && { wr blocks "$blocks"; wr blocks_at "$now"; } [ -z "$(rd blocks_at)" ] && wr blocks_at "$now" fi if is_uint "${frontier:-}"; then [ "$(rd frontier)" != "$frontier" ] && { wr frontier "$frontier"; wr frontier_at "$now"; } [ -z "$(rd frontier_at)" ] && wr frontier_at "$now" fi tip_frozen=$(( now - $(rd blocks_at 2>/dev/null || echo $now) )) frontier_quiet=$(( now - $(rd frontier_at 2>/dev/null || echo $now) )) verdict=ok; action=none # lag EMPTY means "not measured", which is different from 0 ("nothing to fetch"). # Defaulting it to 0 on an engine without getmatmulattestedtip made the whole # stall classifier unreachable, since it is gated on lag > 0. Unmeasured must # fall back to the height-only rule instead of going silent. lag="" if is_int "${behind:-}"; then lag=$behind elif is_uint "${frontier:-}" && is_uint "${blocks:-}"; then lag=$(( frontier - blocks )); fi # Is there anything to go and get? Measured: a positive lag. Unmeasured: fall # back to headers above blocks, the pre-frontier behaviour. gap=0 if [ -n "$lag" ]; then [ "$lag" -gt 0 ] && gap=1 elif is_uint "${headers:-}" && is_uint "${blocks:-}" && [ "$headers" -gt "$blocks" ]; then gap=1 fi # ── Classify ──────────────────────────────────────────────────────────────────────── if [ "$svc_btxd" != "active" ]; then verdict=btxd_down # systemd Restart= owns recovery; we report elif ! is_uint "$blocks"; then verdict=rpc_unreachable elif [ "$gap" = 1 ] && [ "$tip_frozen" -ge "$BEHIND_ALERT_SECS" ]; then # OUR stall: the signers are ahead of us and we are not catching up. if [ "${n_auth:-0}" = "0" ]; then verdict=no_qualifying_peer elif tail -c 65536 "$DEBUG_LOG" 2>/dev/null | grep -q 'retryable MatMul failure'; then verdict=attestation_missing # Qualifying peers exist and we are asking NONE of them: the scheduler's # gate, not the peer set. Named separately because the remedies are # disjoint -- redial is a guaranteed no-op here and the nudge below is the # remedy. An unmeasured census (empty) must NOT claim this. elif [ -n "${n_inflight:-}" ] && [ "${n_inflight:-1}" = "0" ]; then verdict=block_fetch_gated else verdict=body_missing; fi # `action` is initialised to "none" for the healthy path. This branch # composes onto it, so clear it or every stall logs "none; redialled ...". action="" # ── The nudge ─────────────────────────────────────────────────────────── # RUNS EVERY STALL CYCLE, deliberately OUTSIDE the redial rate limiter. # It used to sit inside it, which meant that on a rate-limited cycle the # node got neither remedy. On 2026-08-20 those cycles alternated, so the one # action that could actually fix the wedge fired every 10 minutes instead of # every 5, on a stall that ran 75 minutes. # # The limiter exists to protect PEERS from repeated dialling: 32 ignored # GETMMATTEST in a row earns a 24-hour ban. A block request is not that. It # is an ordinary GETDATA for four blocks from four peers, once per cycle, # and only ever asks for data. Different cost, so a different budget. # # Redialling fixes a node with nobody to ask. It does NOTHING for a node # whose own block-fetch scheduler has wedged: btxd logs # Block fetch stall detected: ... in_flight=0 peers_downloading=0 # every minute, forever, while requesting nothing from dozens of healthy # peers. Measured on the production explorer 2026-08-20: an hour frozen, # 48 blocks behind the frontier, redial had no effect at all. # # getblockfrompeer asks ONE named peer for ONE named block and bypasses # the scheduler. In production it moved the tip within 20 seconds, and # walking the branch that way recovered all 51 blocks, after which the # scheduler resumed by itself and the stall lines stopped. # # Bounded on purpose: a few blocks per run, not a catch-up loop. Enough # to break the wedge and let the node take over. It only ever REQUESTS # data, so being wrong about the diagnosis costs nothing. NUDGE=$($CLI getchaintips 2>/dev/null | python3 -c 'import json,sys,subprocess,os cli=os.environ.get("NUDGE_CLI","").split() try: ts=json.load(sys.stdin) except Exception: raise SystemExit c=[t for t in ts if t.get("branchlen",0)>0] if not c or not cli: raise SystemExit tipn=subprocess.run(cli+["getblockcount"],capture_output=True,text=True).stdout.strip() if not tipn.isdigit(): raise SystemExit tip=int(tipn); h=max(c,key=lambda x:x["height"])["hash"]; out=[] for _ in range(400): r=subprocess.run(cli+["getblockheader",h],capture_output=True,text=True).stdout.strip() try: hd=json.loads(r) except Exception: break if hd["height"]<=tip: break out.append((hd["height"],h)) h=hd.get("previousblockhash") if not h: break out.sort() print(" ".join(x[1] for x in out[:4]))' 2>/dev/null) if [ -n "${NUDGE:-}" ]; then PIDS=$($CLI getpeerinfo 2>/dev/null | python3 -c 'import json,sys try: ps=json.load(sys.stdin) except Exception: raise SystemExit def arch(p): try: return (int(p.get("services","0"),16)>>31)&1 except Exception: return 0 sel=[p for p in ps if arch(p) or p.get("connection_type")=="manual"][:4] print(" ".join(str(p["id"]) for p in sel))' 2>/dev/null) n=0 for bh in $NUDGE; do for pid in ${PIDS:-}; do $CLI getblockfrompeer "$bh" "$pid" >/dev/null 2>&1 done n=$((n+1)) done [ "$n" -gt 0 ] && { log "nudged $n block(s) via getblockfrompeer (scheduler was requesting nothing)"; action="nudged $n blocks"; } fi # ── The redial, still rate limited ────────────────────────────────────── last=$(rd last_redial); gate=$REDIAL_OK_SECS [ "$(rd last_redial_ok)" = "0" ] && gate=$REDIAL_FAIL_SECS if [ -z "$last" ] || [ $(( now - last )) -ge "$gate" ]; then ok=0 for h in $ARCHIVES; do $CLI addnode "$h" add >/dev/null 2>&1 $CLI addnode "$h" onetry >/dev/null 2>&1 && ok=$((ok+1)) done wr last_redial "$now"; wr last_redial_ok "$([ "$ok" -gt 0 ] && echo 1 || echo 0)" action="${action:+$action; }redialled ${ok} archives" else action="${action:+$action; }redial rate-limited" fi # Logged on EVERY stall cycle, including rate-limited ones, so the record # shows the stall continuing rather than going quiet between redials. log "STALL $verdict lag=$lag frozen=${tip_frozen}s auth=${n_auth:-?} -> $action" elif [ "$gap" = 1 ]; then # BEHIND, BUT THE TIP IS MOVING. The stall branch above requires a frozen # tip, so a node draining a backlog fell through to `ok` while materially # behind: the 2026-08-20 beacon read verdict=ok next to blocks_behind=52, # because catching up resets tip_frozen every block. Recovery is not a # fault and must not alarm, but the runbook tells an operator that `ok` # means nothing to do, and 52 behind is not nothing. verdict=recovering; action=none elif [ "$frontier_quiet" -ge "$FRONTIER_SLOW_SECS" ] && [ "$frontier_quiet" -lt "$FRONTIER_QUIET_SECS" ]; then # The frontier is quiet for longer than any normal block gap, but not yet # the hour that FRONTIER_QUIET_SECS treats as a network-level event. Not # ours to fix and not worth waking anyone, but it must not read `ok`: this # is the exact window in which a user sees a frozen explorer and concludes # their wallet is broken. verdict=frontier_slow; action=none elif [ "$frontier_quiet" -ge "$FRONTIER_QUIET_SECS" ]; then # The frontier itself is not advancing. Not ours to fix — the signer is quiet or the # network is slow — but a human should know, and redialling would be pure noise. verdict=network_frontier_quiet elif is_uint "${electrs:-}" && [ $(( blocks - electrs )) -gt "$ELECTRS_LAG_OK" ]; then [ "$(rd electrs)" = "$electrs" ] && verdict=electrs_lagging || verdict=electrs_catching_up fi is_uint "${electrs:-}" && wr electrs "$electrs" # ── Disk guard ────────────────────────────────────────────────────────────────────── disk_state=ok if is_uint "${disk_pct:-}"; then [ "$disk_pct" -ge "$DISK_WARN_PCT" ] && disk_state=warn if [ "$disk_pct" -ge "$DISK_CRIT_PCT" ]; then disk_state=critical sz=$(stat -c%s "$DEBUG_LOG" 2>/dev/null || echo 0) if [ "$sz" -gt 536870912 ]; then tail -c 67108864 "$DEBUG_LOG" > "${DEBUG_LOG}.tmp" 2>/dev/null && mv -f "${DEBUG_LOG}.tmp" "$DEBUG_LOG" log "DISK critical ${disk_pct}% — capped debug.log (was ${sz} bytes)" action="${action}; capped debug.log" fi fi fi # ── Beacon: public, read-only truth, no SSH required to read it ───────────────────── tmp="${BEACON}.tmp" cat > "$tmp" </dev/null; chmod 644 "$BEACON" 2>/dev/null # ── Public beacon: the same truth, minus anything operational ─────────────────────── # Nothing on this box could tell a human anything. Over a thousand RED health events # fired into a log file, on the same disk the check was warning about. An external # monitor is the fix, and an external monitor needs a URL. # # DELIBERATELY REDUCED. The private beacon carries the peer census, disk usage and # unit states: that maps our topology and our operational posture and is nobody # else's business. What a monitor needs in order to decide "is this node serving # good data" is the verdict and the heights, so that is all this publishes. # Serving the full beacon would have been less work and more exposure. # # Written only when PUBLIC_BEACON is set, so every other node running this script # publishes nothing at all unless its operator opts in. if [ -n "${PUBLIC_BEACON:-}" ]; then ptmp="${PUBLIC_BEACON}.tmp" cat > "$ptmp" </dev/null; chmod 644 "$PUBLIC_BEACON" 2>/dev/null fi [ "$verdict" != "ok" ] && log "verdict=$verdict lag=$lag tip_frozen=${tip_frozen}s frontier_quiet=${frontier_quiet}s auth=${n_auth:-?} disk=${disk_pct:-?}% action=$action" exit 0