diff --git a/.testnet/scenarios/export-suite.yml b/.testnet/scenarios/export-suite.yml index 87ffc0803..b6833b0b2 100644 --- a/.testnet/scenarios/export-suite.yml +++ b/.testnet/scenarios/export-suite.yml @@ -18,8 +18,6 @@ defaults: LedgerConsensus: debug ConsensusExtensions: debug NetworkOPs: info - env: - XAHAU_RESOURCE_PER_PORT: "1" rc: - rng_poll_ms=333 diff --git a/.testnet/scenarios/latency-suite.yml b/.testnet/scenarios/latency-suite.yml index 332235707..fa7fa3ab0 100644 --- a/.testnet/scenarios/latency-suite.yml +++ b/.testnet/scenarios/latency-suite.yml @@ -18,8 +18,6 @@ defaults: LedgerConsensus: debug ConsensusExtensions: debug NetworkOPs: info - env: - XAHAU_RESOURCE_PER_PORT: "1" rc: - rng_poll_ms=250 tests: diff --git a/.testnet/scenarios/rollout/mixed_binary_boundary.py b/.testnet/scenarios/rollout/mixed_binary_boundary.py new file mode 100644 index 000000000..35e882a8f --- /dev/null +++ b/.testnet/scenarios/rollout/mixed_binary_boundary.py @@ -0,0 +1,82 @@ +"""Mixed-binary rollout boundary for ConsensusEntropy. + +Topology (launch with per-node binaries): + n0-n2 @export-rng feature-export-rng build, supports Export + ConsensusEntropy, UNL validators + n3-n5 @release 2026.6.21 mainnet release build, NO CE support, non-UNL trackers + +What this proves: + Phase 1 CE inactive -> heterogeneous net is healthy. New validators emit + legacy 32-byte proposal positions, so old nodes parse them and track + validated ledgers. + Phase 2 CE activates -> the old @release nodes hit the upgrade boundary. Per + this branch's setAmendmentBlocked() (Change.cpp / LedgerMaster.cpp -> + NetworkOPs) they become amendment-blocked and DROP to CONNECTED, but + KEEP RUNNING (RPC stays up) -- they do NOT crash. This is the + "upgrade validators before activating" rollout invariant, observed. + +Launch (fast dev check, CE enabled at genesis): + x-testnet run -n 6 \ + --node-binary n0:@export-rng --node-binary n1:@export-rng --node-binary n2:@export-rng \ + --node-binary n3:@release --node-binary n4:@release --node-binary n5:@release \ + --feature @ConsensusEntropy \ + --scenario-script .testnet/scenarios/rollout/mixed_binary_boundary.py --teardown + +Launch (real transition, activates at the next flag ledger ~256; seed pre-satisfies the 1-min hold): + x-testnet run -n 6 \ + --seed-majority @ConsensusEntropy \ + --scenario-script .testnet/scenarios/rollout/mixed_binary_boundary.py --teardown +""" + +from helpers import CONSENSUS_ENTROPY_FEATURE + +VALIDATORS = [0, 1, 2] +OLD_NODES = [3, 4, 5] + + +def _ce_enabled(ctx, node_id=0): + s = ctx.feature_check(CONSENSUS_ENTROPY_FEATURE, node_id=node_id) + return bool(s and s.get("enabled")) + + +def _blocked(ctx, nid): + info = ctx.rpc.server_info(nid) or {} + return bool(info.get("info", {}).get("amendment_blocked")) + + +async def scenario(ctx, log): + await ctx.wait_for_ledger_close(timeout=90) + + if not _ce_enabled(ctx): + # Phase 1: with CE inactive the old @release trackers must stay in sync. + target = (await ctx.wait_for_ledgers(3, timeout=180)).result + for nid in OLD_NODES: + await ctx.wait_for_ledger(target, node_id=nid, timeout=120) + log(f"phase1 OK: old @release nodes tracked to ledger {target} (CE inactive)") + + # Trigger phase 2: vote CE up on the new validators only. + ctx.feature(CONSENSUS_ENTROPY_FEATURE, vetoed=False, nodes=VALIDATORS) + log("voted ConsensusEntropy accept on n0-n2; awaiting activation...") + else: + log("CE enabled at genesis; skipping phase 1, checking the boundary directly") + + await ctx.wait_for_feature( + CONSENSUS_ENTROPY_FEATURE, + check=lambda s: s.get("enabled"), + nodes=VALIDATORS, + timeout=1200, + ) + log("ConsensusEntropy ENABLED on validators n0-n2") + + # The upgrade boundary: old non-UNL nodes must become amendment-blocked... + await ctx.wait_for_nodes( + lambda nid: _blocked(ctx, nid), nodes=OLD_NODES, timeout=180 + ) + # ...report it in the log... + ctx.assert_log("server blocked", nodes=OLD_NODES) + # ...and still be alive (RPC responsive) -> blocked, not crashed. + for nid in OLD_NODES: + assert ctx.rpc.server_info(nid), f"n{nid} RPC unreachable (crashed?)" + + log( + "PASS: n3-n5 (@release) amendment-blocked and STILL RUNNING -- upgrade boundary confirmed" + ) diff --git a/.testnet/scenarios/rollout/rolling_upgrade.py b/.testnet/scenarios/rollout/rolling_upgrade.py new file mode 100644 index 000000000..da187b3a2 --- /dev/null +++ b/.testnet/scenarios/rollout/rolling_upgrade.py @@ -0,0 +1,118 @@ +"""Realistic rolling binary upgrade, then ConsensusEntropy activation. + +Topology (ALL start on @release, a build with NO CE support): + n0-n4 5 UNL validators (quorum 4) + n5 tracker (non-UNL) -> WILL be upgraded; keeps working after activation + n6 tracker (non-UNL) -> NOT upgraded (the straggler); becomes + amendment-blocked after activation but does NOT crash + +TOPOLOGY-AWARE RESTARTS: the fixed-peer mesh takes time to form, and a node whose +peers haven't fully connected acts as a junction -- restarting it partitions the +rest (observed: restarting n0 dropped n1 to 0 peers -> isolated -> stalled). So we +require a well-connected mesh (>= MESH_MIN peers on every node) BEFORE rolling and +after EACH restart, before touching the next node. If the mesh never forms this +fails loud (a peering problem), rather than silently stalling consensus. + +Why --seed-majority is used (NOT a magic vote): the validators still VOTE the +amendment up (ctx.feature below). Seeding only pre-writes the sfMajorities record +with CloseTime=0 so the hold is already satisfied, collapsing activation to ONE +flag ledger. If the validators don't vote yes it is cleared (tfLostMajority). See +prepare_genesis_file() in testnet/config.py. + +Arc: all-old healthy + mesh formed -> rolling-upgrade 5 validators + 1 tracker +(mesh re-forms between each) -> vote CE -> activation at the flag ledger -> +n5 (upgraded) keeps tracking, n6 (straggler) amendment-blocked but not crashed. + +Run: x-testnet --rippled-path @release suite \\ + .testnet/scenarios/rollout/rollout-suite.yml --stop-on-fail +""" + +from helpers import CONSENSUS_ENTROPY_FEATURE + +VALIDATORS = [0, 1, 2, 3, 4] +UPGRADED_TRACKER = 5 +STRAGGLER_TRACKER = 6 +ALL_NODES = VALIDATORS + [UPGRADED_TRACKER, STRAGGLER_TRACKER] +# Full mesh is 6 peers each (7 nodes). Require >= 4 so no node is a junction: +# removing any one still leaves the rest connected. +MESH_MIN = 4 + + +def _info(ctx, nid): + return (ctx.rpc.server_info(nid) or {}).get("info", {}) + + +def _blocked(ctx, nid): + return bool(_info(ctx, nid).get("amendment_blocked")) + + +def _peers(ctx, nid): + return int(_info(ctx, nid).get("peers") or 0) + + +async def _wait_mesh(ctx, log, *, timeout=180): + """Wait until every node is well-connected (>= MESH_MIN peers), so a restart + can't partition the network. Fails loud (TimeoutError) if the mesh never + forms -- that means a peering problem, not a transient.""" + await ctx.wait_for_nodes( + lambda x: _peers(ctx, x) >= MESH_MIN, nodes=ALL_NODES, timeout=timeout + ) + peers = {nid: _peers(ctx, nid) for nid in ALL_NODES} + log(f"peer mesh healthy (>= {MESH_MIN} peers each): {peers}") + + +async def scenario(ctx, log): + # 1. all-old net healthy AND the full peer mesh has formed before we touch it + await ctx.wait_for_ledger_close(timeout=90) + base = (await ctx.wait_for_ledgers(2, timeout=120)).result + for nid in ALL_NODES: + await ctx.wait_for_ledger(base, node_id=nid, timeout=120) + await _wait_mesh(ctx, log, timeout=180) + log(f"all-old net healthy at ledger {base} (CE inactive)") + + # 2. rolling-upgrade validators (then one tracker), one at a time. After each + # restart, give the mesh time to re-form (the restarted node re-dials its + # fixed peers) BEFORE rolling the next, so we never take out a junction. + for nid in [*VALIDATORS, UPGRADED_TRACKER]: + ref = next(v for v in VALIDATORS if v != nid) + role = "validator" if nid in VALIDATORS else "tracker" + log(f"rolling upgrade: n{nid} ({role}) -> @export-rng") + await ctx.restart_node_with_binary(nid, "@export-rng", delay=3) + await _wait_mesh(ctx, log, timeout=180) # mesh re-formed before next roll + target = (await ctx.wait_for_ledgers(2, node_id=ref, timeout=180)).result + await ctx.wait_for_ledger(target, node_id=nid, timeout=180) + log(f"n{nid} rejoined; mesh re-formed; quorum advanced to {target}") + log( + f"validators + n{UPGRADED_TRACKER} on @export-rng; " + f"n{STRAGGLER_TRACKER} left on @release; CE still inactive" + ) + + # 3. vote CE up on the validators (real vote; the seed only pre-satisfied the hold) + ctx.feature(CONSENSUS_ENTROPY_FEATURE, vetoed=False, nodes=VALIDATORS) + log("voted ConsensusEntropy accept on n0-n4; crossing the flag ledger...") + + # 4. activation at the flag ledger + await ctx.wait_for_feature( + CONSENSUS_ENTROPY_FEATURE, + check=lambda s: s.get("enabled"), + nodes=VALIDATORS, + timeout=900, + ) + log("ConsensusEntropy ENABLED on the upgraded quorum") + + # 5. upgraded tracker keeps working; straggler is amendment-blocked (not crashed) + await ctx.wait_for_nodes( + lambda x: _blocked(ctx, x), nodes=[STRAGGLER_TRACKER], timeout=180 + ) + ctx.assert_log("server blocked", nodes=[STRAGGLER_TRACKER]) + assert not _blocked(ctx, UPGRADED_TRACKER), ( + f"n{UPGRADED_TRACKER} (upgraded tracker) should NOT be amendment-blocked" + ) + await ctx.wait_for_ledgers(2, node_id=UPGRADED_TRACKER, timeout=120) + for nid in (UPGRADED_TRACKER, STRAGGLER_TRACKER): + assert ctx.rpc.server_info(nid), f"n{nid} RPC down (crashed?)" + log( + f"PASS: rolling upgrade preserved quorum 4 (mesh-gated); CE activated; " + f"n{UPGRADED_TRACKER} (upgraded tracker) still tracking; " + f"n{STRAGGLER_TRACKER} (@release straggler) amendment-blocked and still running" + ) diff --git a/.testnet/scenarios/rollout/rollout-suite.yml b/.testnet/scenarios/rollout/rollout-suite.yml new file mode 100644 index 000000000..2ee5ff421 --- /dev/null +++ b/.testnet/scenarios/rollout/rollout-suite.yml @@ -0,0 +1,42 @@ +# Rolling binary upgrade -> ConsensusEntropy activation. +# +# All nodes start on @release (set via `--rippled-path @release` on the CLI); the +# scenario rolling-restarts them to @export-rng, then casts a REAL vote. The +# --seed-majority (majority_features) only pre-satisfies the amendment hold time +# so activation lands at ONE flag ledger instead of two -- it is NOT a vote, and +# is cleared (tfLostMajority) if the validators don't vote yes before the flag +# ledger. start_ledger gives runway for the 6 restarts + vote to finish before +# the first flag ledger (256), so the seed survives to a real vote. +# +# Run: +# x-testnet --rippled-path @release suite \ +# .testnet/scenarios/rollout/rollout-suite.yml --stop-on-fail +# +# Suite runs auto-snapshot to .testnet/output/runs/ on failure (survives +# teardown), so `x-testnet logs-search --run latest/rolling_upgrade_ce ...` works. + +defaults: + network: + node_count: 7 # n0-n4 validators, n5/n6 non-UNL trackers + validators: 5 + quorum: 4 # 80%: a restart leaves exactly quorum on the peers up + find_ports: true + start_ledger: 210 # measured: the mesh-gated upgrade+vote is ~23 ledgers, so + # the vote lands ~233 with a ~23-ledger margin before flag + # ledger 256 (activation). Miss -> vote after 256 -> next + # flag is 512 (~13 min) or tfLostMajority clears the seed. + majority_features: + - ConsensusEntropy # pre-satisfy the hold ONLY (still needs a real vote) + track_features: + - ConsensusEntropy + log_levels: + LedgerConsensus: debug + NetworkOPs: info + # Dense localhost mesh (needed for rolling restarts): the generator gives each + # peer a distinct 127.0.0. in [ips_fixed] (rippled dedups fixed peers by + # IP, ignoring port). On macOS bring those aliases up first (see below); Linux + # routes all of 127/8 already. Works with stock binaries -- no rebuild. + +tests: + - name: rolling_upgrade_ce + script: .testnet/scenarios/rollout/rolling_upgrade.py diff --git a/.testnet/scenarios/suite.yml b/.testnet/scenarios/suite.yml index 8094c4b41..b0cf7221a 100644 --- a/.testnet/scenarios/suite.yml +++ b/.testnet/scenarios/suite.yml @@ -16,8 +16,6 @@ defaults: LedgerConsensus: debug ConsensusExtensions: debug NetworkOPs: info - env: - XAHAU_RESOURCE_PER_PORT: "1" rc: - rng_poll_ms=333