ci(telemetry): report why a node stopped instead of waiting on a corpse

Three consecutive validation runs timed out at Step 3 with nodes stuck at
"unreachable", and the reason was not recoverable from the logs. The node
logs showed the failing nodes stopping at an identical point, immediately
after JobQueue initialisation and before the debug log is opened, with no
error text at all. The harness knew each node's pid and never used it, so a
crashed node was indistinguishable from a slow one.

The readiness loop now checks whether each node process is still alive and
fails as soon as one is not, instead of waiting out the remaining window and
burying the cause under two minutes of progress output. Liveness is not a
bare `kill -0`: an exited-but-unreaped child keeps its pid, so a zombie
answers `kill -0` and reads as alive for the whole window, which is exactly
how a crashed node came to look like a slow one.

On failure each stopped node reports its wait status and the tail of its
stdout. The status is the discriminator that was missing: 137 for a SIGKILL,
139 for a segfault, 134 for an abort, anything below 128 for a deliberate
exit. stdout is printed inline rather than left to the artifact upload,
because a node that dies before its debug log opens writes nothing else and
a cancelled run uploads nothing at all.

This is instrumentation, not a fix. The failure is not attributable to the
recent changes on this branch: the first red run touched only the two Python
files used at Steps 4 and 5, both of which run after this gate, and the same
harness passed 5/5 twice before that.

docker/telemetry/workload/run-full-validation.sh

@@ -354,6 +354,50 @@ done
 # ---------------------------------------------------------------------------
 # Step 3: Wait for consensus
 # ---------------------------------------------------------------------------
+# Report whether a node process is still alive.
+#
+# A child that has exited but not yet been waited on still answers `kill -0`,
+# because the zombie keeps its pid until someone collects it. Checking only
+# `kill -0` therefore reads a dead node as alive for the whole readiness
+# window, which is how a crashed node came to look like a slow one.
+node_running() {
+    local pid="$1" state
+    kill -0 "$pid" 2>/dev/null || return 1
+    if [ -r "/proc/$pid/stat" ]; then
+        state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null || echo "?")
+        [ "$state" != "Z" ] || return 1
+    fi
+    return 0
+}
+
+# Print why each stopped node stopped: its wait status, then its last output.
+#
+# The status is the discriminator this harness was missing -- 137 is SIGKILL
+# (the kernel reclaiming memory), 139 a segfault, 134 an abort, anything under
+# 128 a deliberate exit. The nodes are direct children of this script, so their
+# status is still retrievable until something waits on them.
+#
+# stdout is printed inline rather than left to the artifact upload because a
+# node that dies before its debug log opens writes nothing else, and a
+# cancelled run uploads nothing at all.
+report_stopped_nodes() {
+    local i pid status
+    for i in $(seq 1 "$NUM_NODES"); do
+        pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo "")
+        [ -n "$pid" ] || continue
+        node_running "$pid" && continue
+        status=0
+        wait "$pid" 2>/dev/null || status=$?
+        warn "node$i (pid $pid) is not running — wait status $status"
+        if [ -s "$WORKDIR/node$i/stdout.log" ]; then
+            warn "node$i last output:"
+            tail -n 15 "$WORKDIR/node$i/stdout.log" | sed 's/^/      /' >&2
+        else
+            warn "node$i wrote no stdout at all"
+        fi
+    done
+}
+
 log "Step 3: Waiting for consensus..."
 for attempt in $(seq 1 120); do
     ready=0
@@ -378,6 +422,21 @@ for attempt in $(seq 1 120); do
         ok "All $NUM_NODES nodes proposing (attempt $attempt)"
         break
     fi
+    # A stopped process will never reach proposing. Waiting out the rest of the
+    # window only delays the same failure and buries its cause under two
+    # minutes of progress output.
+    stopped=0
+    for n in $(seq 1 "$NUM_NODES"); do
+        p=$(cat "$WORKDIR/node$n/xrpld.pid" 2>/dev/null || echo "")
+        if [ -n "$p" ] && ! node_running "$p"; then
+            stopped=$((stopped + 1))
+        fi
+    done
+    if [ "$stopped" -gt 0 ]; then
+        echo ""
+        report_stopped_nodes
+        die "$stopped of $NUM_NODES node(s) stopped during startup; only $ready reached proposing. Not proposing:${laggards}. Per-node status is above, then '$0 --cleanup'."
+    fi
     if [ "$attempt" -eq 120 ]; then
         # Fatal, not a warning. A partial cluster still answers queries, so the
         # run would complete and report unrelated span/metric failures: series
@@ -385,6 +444,10 @@ for attempt in $(seq 1 120); do
         # quorum are simply never emitted. One infrastructure error here is
         # worth more than a pile of misleading assertion failures later.
         echo ""
+        # Every node is still running but not proposing, so this is a genuine
+        # convergence problem rather than a crash. Run the reporter anyway: it
+        # is a no-op when nothing stopped, and it costs nothing to be sure.
+        report_stopped_nodes
         die "Consensus timeout — only $ready/$NUM_NODES nodes proposing after ${attempt}s. Not proposing:${laggards}. Check $WORKDIR/node*/debug.log and $WORKDIR/node*/stdout.log (a node that died before its log sink opened writes only the latter), then '$0 --cleanup'."
     fi
     printf "\r  %d/%d nodes proposing..." "$ready" "$NUM_NODES"