#!/bin/bash
# BTX Keeper watchdog — the fail-quiet guarantee.
#
# A Keeper that wedges must never become a hot, loud laptop at 2 a.m. This
# watches for the known no-backoff spin (btxd burning ~a full core while the
# tip is frozen — PR btxchain/btx#105) and for quiet stalls:
#   spin  >= 30 min  -> stop btxd CLEANLY, write a note, notify
#   quiet >= 60 min  -> notify only, node keeps running
# It NEVER kill -9s and NEVER restarts — a human presses start again, armed
# with the note it wrote. Disk guard: warns past 20 GB (pruned target ~10).
set -u

KEEPER_HOME="$HOME/.btx-keeper"
CLI="$KEEPER_HOME/bin/btx-cli"
LOG="$KEEPER_HOME/logs/watchdog.log"
PAUSE_MARKER="$KEEPER_HOME/keeper-paused"
INTERVAL=60
SPIN_CPU_PCT=85
SPIN_WINDOW_MIN=30
QUIET_STALL_MIN=60
DISK_ALERT_GB=20

log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOG"; }

notify() { # $1 title, $2 body
    osascript -e "display notification \"$2\" with title \"$1\"" 2>>"$KEEPER_HOME/logs/watchdog.err"
    printf '# %s\n\n%s\n\n%s\n' "$1" "$2" "$(date)" > "$HOME/Desktop/BTX-KEEPER-NOTE-$(date '+%Y%m%d-%H%M').md"
    log "NOTIFY: $1 — $2"
}

cputime_cs() {
    ps -p "$1" -o cputime= 2>/dev/null | awk '{
        gsub(/^[ \t]+/, "", $0); n = split($0, a, ":"); d = 0
        if (n == 3) { h = a[1]; if (h ~ /-/) { split(h, b, "-"); d = b[1]; h = b[2] }
                      secs = d*86400 + h*3600 + a[2]*60 + a[3] }
        else if (n == 2) { secs = a[1]*60 + a[2] } else { secs = 0 }
        printf "%d", secs * 100 }'
}

btxd_pid() { pgrep -f "^$KEEPER_HOME/bin/btxd" | head -1; }

heights() { "$CLI" -datadir="$KEEPER_HOME" getblockchaininfo 2>/dev/null \
    | python3 -c "import json,sys
d=json.load(sys.stdin); print(d['blocks'], d['headers'])" 2>/dev/null; }

# How far we trail the SIGNED frontier. This is what separates "the network is
# quiet" from "we are behind and not catching up" — two states that look
# identical from a frozen height and want opposite responses. Empty when the
# node does not answer; the caller then falls back to height-only behaviour.
frontier_lag() { "$CLI" -datadir="$KEEPER_HOME" getmatmulattestedtip 2>/dev/null \
    | python3 -c "import json,sys
try:
    d=json.load(sys.stdin); sf=d.get('signed_frontier') or {}
    b=sf.get('blocks_behind')
    print('' if b is None else b)
except Exception: print('')" 2>/dev/null; }

# The archive peers a Keeper follows the chain through. Kept in step with the
# installer's conf and btx-core's BTX_ARCHIVE_PEERS.
ARCHIVES="207.56.229.99:19335 185.204.25.227:19335 195.137.245.82:20982 node.btx.dev:19335 node.btxchain.org:19335 node.btx.tools:19335"

# The ONE safe remediation: an RPC-added peer is MANUAL, so it passes the
# trusted-mirror authority gate with no restart. Rate-limited hard — 32 ignored
# GETMMATTEST in a row earns a 24-hour ban from a signer.
redial_archives() {
    local ok=0
    for h in $ARCHIVES; do
        "$CLI" -datadir="$KEEPER_HOME" addnode "$h" add    >/dev/null 2>&1
        "$CLI" -datadir="$KEEPER_HOME" addnode "$h" onetry >/dev/null 2>&1 && ok=$((ok+1))
    done
    log "redialled $ok archive peers"
}

# Blocks this Keeper is CURRENTLY ASKING FOR, across every peer. Empty when the
# node does not answer.
#
# THE FACT THAT SEPARATES "NOBODY WILL SERVE ME" FROM "I AM ASKING NOBODY".
# Both look identical from a frozen height with headers above it, and they want
# opposite responses. Zero, while signed work is outstanding, means the block
# scheduler's gate has stopped requesting (upstream btxchain/btx#112) and no
# amount of redialling will change it, because peer availability was never the
# input. Measured on the production explorer 2026-08-20: 75 minutes wedged with
# 7 peers through the authority gate, 11 archives, and in_flight=0.
blocks_in_flight() { "$CLI" -datadir="$KEEPER_HOME" getpeerinfo 2>/dev/null \
    | python3 -c "import json,sys
try:
    ps=json.load(sys.stdin)
    print(sum(len(p.get('inflight',[]) or []) for p in ps))
except Exception: print('')" 2>/dev/null; }

# The remedy for a gated scheduler: ask NAMED peers for NAMED blocks, which
# bypasses the scheduler entirely. In production this moved the tip within 20
# seconds and walking the branch recovered all 51 blocks, after which the
# scheduler resumed by itself.
#
# Bounded on purpose: a few blocks per run, not a catch-up loop. Enough to break
# the wedge and let the node take over. It only ever REQUESTS data, so being
# wrong about the diagnosis costs nothing beyond a few messages.
nudge_next_blocks() {
    local hashes pids n=0
    hashes=$(NUDGE_CLI="$CLI -datadir=$KEEPER_HOME" "$CLI" -datadir="$KEEPER_HOME" getchaintips 2>/dev/null \
        | python3 -c "import json,sys,subprocess,os
cli=os.environ.get('NUDGE_CLI','').split()
try: ts=json.load(sys.stdin)
except Exception: raise SystemExit
c=[t for t in ts if t.get('branchlen',0)>0]
if not c or not cli: raise SystemExit
tipn=subprocess.run(cli+['getblockcount'],capture_output=True,text=True).stdout.strip()
if not tipn.isdigit(): raise SystemExit
tip=int(tipn); h=max(c,key=lambda x:x['height'])['hash']; out=[]
for _ in range(400):
    r=subprocess.run(cli+['getblockheader',h],capture_output=True,text=True).stdout.strip()
    try: hd=json.loads(r)
    except Exception: break
    if hd['height']<=tip: break
    out.append((hd['height'],h))
    h=hd.get('previousblockhash')
    if not h: break
out.sort()
print(' '.join(x[1] for x in out[:4]))" 2>/dev/null)
    [ -z "${hashes:-}" ] && return 0
    pids=$("$CLI" -datadir="$KEEPER_HOME" getpeerinfo 2>/dev/null \
        | python3 -c "import json,sys
try: ps=json.load(sys.stdin)
except Exception: raise SystemExit
def arch(p):
    try: return (int(p.get('services','0'),16)>>31)&1
    except Exception: return 0
sel=[p for p in ps if arch(p) or p.get('connection_type')=='manual'][:4]
print(' '.join(str(p['id']) for p in sel))" 2>/dev/null)
    for bh in $hashes; do
        for pid in ${pids:-}; do
            "$CLI" -datadir="$KEEPER_HOME" getblockfrompeer "$bh" "$pid" >/dev/null 2>&1
        done
        n=$((n+1))
    done
    [ "$n" -gt 0 ] && log "nudged $n block(s) via getblockfrompeer (scheduler was requesting nothing)"
}

log "keeper watchdog start"
prev_cpu=0; prev_pid=""; hot_min=0; frozen_min=0; last_b=""; last_h=""; tick=0; quiet_sent=0
while true; do
    sleep "$INTERVAL"; tick=$((tick+1))
    pid=$(btxd_pid)
    if [ -z "$pid" ]; then prev_pid=""; hot_min=0; frozen_min=0; quiet_sent=0; continue; fi
    if [ "$pid" != "$prev_pid" ]; then
        prev_pid="$pid"; prev_cpu=$(cputime_cs "$pid"); hot_min=0; frozen_min=0
        last_b=""; last_h=""; quiet_sent=0; log "tracking btxd pid=$pid"; continue
    fi
    cur=$(cputime_cs "$pid"); [ -z "$cur" ] && cur=$prev_cpu
    pct=$(( (cur - prev_cpu) / INTERVAL )); prev_cpu=$cur
    read -r b h <<< "$(heights)" || true
    if [ -z "${b:-}" ]; then
        :                              # RPC didn't answer (busy/warming): HOLD the
                                       # counters — a spinning node starves RPC, and
                                       # zeroing here meant the spin window could
                                       # never fill during the exact failure it
                                       # exists to catch. No verdicts either way.
    elif [ "$b" = "0" ]; then
        frozen_min=0; hot_min=0        # startup / presync / snapshot load: no verdicts
    elif [ "$b" = "$last_b" ] && [ "$h" = "$last_h" ]; then
        frozen_min=$((frozen_min+1))
        if [ "$pct" -ge "$SPIN_CPU_PCT" ]; then hot_min=$((hot_min+1)); else hot_min=0; fi
    else
        frozen_min=0; hot_min=0; quiet_sent=0
    fi
    [ -n "${b:-}" ] && { last_b="$b"; last_h="$h"; }

    if [ "$hot_min" -ge "$SPIN_WINDOW_MIN" ]; then
        # Marker FIRST, then stop: the run wrapper honors the marker, so it
        # cannot restart the node in the window between our stop and its next
        # tick (it used to — the wrapper's start_node knew nothing about this
        # pause and undid it within 2 minutes).
        printf 'paused by the watchdog: sustained spin (>=%s%% CPU with a frozen tip for %s min) at %s\nTo let the Keeper run again: rm ~/.btx-keeper/keeper-paused\n' \
            "$SPIN_CPU_PCT" "$SPIN_WINDOW_MIN" "$(date)" > "$PAUSE_MARKER"
        "$CLI" -datadir="$KEEPER_HOME" stop >> "$LOG" 2>&1
        notify "BTX Keeper paused itself" \
"The node hit a known bug (working flat-out on one block for ${SPIN_WINDOW_MIN}+ minutes) and stopped itself cleanly. Nothing is lost, and it will NOT restart on its own. To let it run again: delete ~/.btx-keeper/keeper-paused — or just leave it; the next software update fixes this."
        hot_min=0
    elif [ "$frozen_min" -ge "$QUIET_STALL_MIN" ] && [ "$quiet_sent" = 0 ]; then
        # Before saying anything, ask WHY the height is still. If we sit exactly
        # at the signed frontier there is nothing to fetch and nothing to fix —
        # the whole network is waiting on the attestor, and telling the owner
        # their Keeper has a problem would be false. Only when signed work is
        # outstanding is this Keeper actually behind, and only then is redialling
        # the archives useful. (Measured 2026-08-19: a ~100-minute quiet frontier
        # would otherwise have notified every Keeper on the network.)
        lag=$(frontier_lag)
        # Numeric, not string: "-3" (ahead of the frontier) has even less to
        # fetch than 0 and must take the quiet branch too. A string compare
        # missed it and sent the Keeper down the redial+notify path.
        # Unmeasured (empty, old engine or RPC down) deliberately does NOT
        # qualify: it falls through to the redial, which is the safe default.
        if [ -n "${lag:-}" ] && [ "$lag" -le 0 ] 2>/dev/null; then
            log "quiet ${frozen_min}min but AT the signed frontier — network is waiting, no action"
            quiet_sent=1
        else
            # WHICH kind of behind? Redialling answers "nobody will serve me".
            # It does nothing for "I am asking nobody", which is the same
            # picture from the heights alone and needs the opposite action.
            # Unmeasured (empty) keeps the old behaviour: redial is the safe
            # default and must not be skipped on a number we could not read.
            infl=$(blocks_in_flight)
            if [ -n "${infl:-}" ] && [ "$infl" = "0" ]; then
                log "behind the signed frontier by ${lag:-?} with NOTHING in flight — scheduler gated, nudging (redial cannot fix this)"
                nudge_next_blocks
            else
                [ -n "${lag:-}" ] && log "behind the signed frontier by $lag — redialling archives"
                redial_archives
            fi
            notify "BTX Keeper: catching up" \
"The chain height has not moved for an hour and there is signed history your Keeper has not collected yet. It just asked its known sources for the missing blocks and will catch up on its own — nothing for you to do."
            quiet_sent=1
        fi
    fi

    if [ $((tick % 30)) = 0 ]; then
        gb=$(( $(du -sk "$KEEPER_HOME" 2>/dev/null | cut -f1) / 1048576 ))
        log "heartbeat blocks=${b:-?} headers=${h:-?} cpu=${pct}% disk=${gb}GB"
        [ "$gb" -ge "$DISK_ALERT_GB" ] && notify "BTX Keeper: disk use ${gb} GB" "The Keeper folder passed ${DISK_ALERT_GB} GB (pruned target is ~10). Worth a look."
    fi
done