mirror of
https://github.com/Xahau/xahaud.git
synced 2026-07-24 07:30:10 +00:00
test(consensus): rolling-upgrade + CE-activation rollout scenario
.testnet/scenarios/rollout/: a 7-node localhost net (5 UNL validators quorum 4 + 2 non-UNL trackers) starts on @release, rolling-restarts the validators + 1 tracker to @export-rng (mesh-gated, quorum preserved), votes ConsensusEntropy, activates at flag ledger 256, and asserts the upgraded tracker keeps tracking while the @release straggler is amendment-blocked but not crashed. Requires distinct loopback aliases (x-testnet setup-aliases). Also drops the dead XAHAU_RESOURCE_PER_PORT env from the entropy/export/latency suites (never read by the binary; the localhost mesh now comes from distinct-IP fixed peers).
This commit is contained in:
@@ -18,8 +18,6 @@ defaults:
|
||||
LedgerConsensus: debug
|
||||
ConsensusExtensions: debug
|
||||
NetworkOPs: info
|
||||
env:
|
||||
XAHAU_RESOURCE_PER_PORT: "1"
|
||||
rc:
|
||||
- rng_poll_ms=333
|
||||
|
||||
|
||||
@@ -18,8 +18,6 @@ defaults:
|
||||
LedgerConsensus: debug
|
||||
ConsensusExtensions: debug
|
||||
NetworkOPs: info
|
||||
env:
|
||||
XAHAU_RESOURCE_PER_PORT: "1"
|
||||
rc:
|
||||
- rng_poll_ms=250
|
||||
tests:
|
||||
|
||||
82
.testnet/scenarios/rollout/mixed_binary_boundary.py
Normal file
82
.testnet/scenarios/rollout/mixed_binary_boundary.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Mixed-binary rollout boundary for ConsensusEntropy.
|
||||
|
||||
Topology (launch with per-node binaries):
|
||||
n0-n2 @export-rng feature-export-rng build, supports Export + ConsensusEntropy, UNL validators
|
||||
n3-n5 @release 2026.6.21 mainnet release build, NO CE support, non-UNL trackers
|
||||
|
||||
What this proves:
|
||||
Phase 1 CE inactive -> heterogeneous net is healthy. New validators emit
|
||||
legacy 32-byte proposal positions, so old nodes parse them and track
|
||||
validated ledgers.
|
||||
Phase 2 CE activates -> the old @release nodes hit the upgrade boundary. Per
|
||||
this branch's setAmendmentBlocked() (Change.cpp / LedgerMaster.cpp ->
|
||||
NetworkOPs) they become amendment-blocked and DROP to CONNECTED, but
|
||||
KEEP RUNNING (RPC stays up) -- they do NOT crash. This is the
|
||||
"upgrade validators before activating" rollout invariant, observed.
|
||||
|
||||
Launch (fast dev check, CE enabled at genesis):
|
||||
x-testnet run -n 6 \
|
||||
--node-binary n0:@export-rng --node-binary n1:@export-rng --node-binary n2:@export-rng \
|
||||
--node-binary n3:@release --node-binary n4:@release --node-binary n5:@release \
|
||||
--feature @ConsensusEntropy \
|
||||
--scenario-script .testnet/scenarios/rollout/mixed_binary_boundary.py --teardown
|
||||
|
||||
Launch (real transition, activates at the next flag ledger ~256; seed pre-satisfies the 1-min hold):
|
||||
x-testnet run -n 6 <same --node-binary set> \
|
||||
--seed-majority @ConsensusEntropy \
|
||||
--scenario-script .testnet/scenarios/rollout/mixed_binary_boundary.py --teardown
|
||||
"""
|
||||
|
||||
from helpers import CONSENSUS_ENTROPY_FEATURE
|
||||
|
||||
VALIDATORS = [0, 1, 2]
|
||||
OLD_NODES = [3, 4, 5]
|
||||
|
||||
|
||||
def _ce_enabled(ctx, node_id=0):
|
||||
s = ctx.feature_check(CONSENSUS_ENTROPY_FEATURE, node_id=node_id)
|
||||
return bool(s and s.get("enabled"))
|
||||
|
||||
|
||||
def _blocked(ctx, nid):
|
||||
info = ctx.rpc.server_info(nid) or {}
|
||||
return bool(info.get("info", {}).get("amendment_blocked"))
|
||||
|
||||
|
||||
async def scenario(ctx, log):
|
||||
await ctx.wait_for_ledger_close(timeout=90)
|
||||
|
||||
if not _ce_enabled(ctx):
|
||||
# Phase 1: with CE inactive the old @release trackers must stay in sync.
|
||||
target = (await ctx.wait_for_ledgers(3, timeout=180)).result
|
||||
for nid in OLD_NODES:
|
||||
await ctx.wait_for_ledger(target, node_id=nid, timeout=120)
|
||||
log(f"phase1 OK: old @release nodes tracked to ledger {target} (CE inactive)")
|
||||
|
||||
# Trigger phase 2: vote CE up on the new validators only.
|
||||
ctx.feature(CONSENSUS_ENTROPY_FEATURE, vetoed=False, nodes=VALIDATORS)
|
||||
log("voted ConsensusEntropy accept on n0-n2; awaiting activation...")
|
||||
else:
|
||||
log("CE enabled at genesis; skipping phase 1, checking the boundary directly")
|
||||
|
||||
await ctx.wait_for_feature(
|
||||
CONSENSUS_ENTROPY_FEATURE,
|
||||
check=lambda s: s.get("enabled"),
|
||||
nodes=VALIDATORS,
|
||||
timeout=1200,
|
||||
)
|
||||
log("ConsensusEntropy ENABLED on validators n0-n2")
|
||||
|
||||
# The upgrade boundary: old non-UNL nodes must become amendment-blocked...
|
||||
await ctx.wait_for_nodes(
|
||||
lambda nid: _blocked(ctx, nid), nodes=OLD_NODES, timeout=180
|
||||
)
|
||||
# ...report it in the log...
|
||||
ctx.assert_log("server blocked", nodes=OLD_NODES)
|
||||
# ...and still be alive (RPC responsive) -> blocked, not crashed.
|
||||
for nid in OLD_NODES:
|
||||
assert ctx.rpc.server_info(nid), f"n{nid} RPC unreachable (crashed?)"
|
||||
|
||||
log(
|
||||
"PASS: n3-n5 (@release) amendment-blocked and STILL RUNNING -- upgrade boundary confirmed"
|
||||
)
|
||||
118
.testnet/scenarios/rollout/rolling_upgrade.py
Normal file
118
.testnet/scenarios/rollout/rolling_upgrade.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""Realistic rolling binary upgrade, then ConsensusEntropy activation.
|
||||
|
||||
Topology (ALL start on @release, a build with NO CE support):
|
||||
n0-n4 5 UNL validators (quorum 4)
|
||||
n5 tracker (non-UNL) -> WILL be upgraded; keeps working after activation
|
||||
n6 tracker (non-UNL) -> NOT upgraded (the straggler); becomes
|
||||
amendment-blocked after activation but does NOT crash
|
||||
|
||||
TOPOLOGY-AWARE RESTARTS: the fixed-peer mesh takes time to form, and a node whose
|
||||
peers haven't fully connected acts as a junction -- restarting it partitions the
|
||||
rest (observed: restarting n0 dropped n1 to 0 peers -> isolated -> stalled). So we
|
||||
require a well-connected mesh (>= MESH_MIN peers on every node) BEFORE rolling and
|
||||
after EACH restart, before touching the next node. If the mesh never forms this
|
||||
fails loud (a peering problem), rather than silently stalling consensus.
|
||||
|
||||
Why --seed-majority is used (NOT a magic vote): the validators still VOTE the
|
||||
amendment up (ctx.feature below). Seeding only pre-writes the sfMajorities record
|
||||
with CloseTime=0 so the hold is already satisfied, collapsing activation to ONE
|
||||
flag ledger. If the validators don't vote yes it is cleared (tfLostMajority). See
|
||||
prepare_genesis_file() in testnet/config.py.
|
||||
|
||||
Arc: all-old healthy + mesh formed -> rolling-upgrade 5 validators + 1 tracker
|
||||
(mesh re-forms between each) -> vote CE -> activation at the flag ledger ->
|
||||
n5 (upgraded) keeps tracking, n6 (straggler) amendment-blocked but not crashed.
|
||||
|
||||
Run: x-testnet --rippled-path @release suite \\
|
||||
.testnet/scenarios/rollout/rollout-suite.yml --stop-on-fail
|
||||
"""
|
||||
|
||||
from helpers import CONSENSUS_ENTROPY_FEATURE
|
||||
|
||||
VALIDATORS = [0, 1, 2, 3, 4]
|
||||
UPGRADED_TRACKER = 5
|
||||
STRAGGLER_TRACKER = 6
|
||||
ALL_NODES = VALIDATORS + [UPGRADED_TRACKER, STRAGGLER_TRACKER]
|
||||
# Full mesh is 6 peers each (7 nodes). Require >= 4 so no node is a junction:
|
||||
# removing any one still leaves the rest connected.
|
||||
MESH_MIN = 4
|
||||
|
||||
|
||||
def _info(ctx, nid):
|
||||
return (ctx.rpc.server_info(nid) or {}).get("info", {})
|
||||
|
||||
|
||||
def _blocked(ctx, nid):
|
||||
return bool(_info(ctx, nid).get("amendment_blocked"))
|
||||
|
||||
|
||||
def _peers(ctx, nid):
|
||||
return int(_info(ctx, nid).get("peers") or 0)
|
||||
|
||||
|
||||
async def _wait_mesh(ctx, log, *, timeout=180):
|
||||
"""Wait until every node is well-connected (>= MESH_MIN peers), so a restart
|
||||
can't partition the network. Fails loud (TimeoutError) if the mesh never
|
||||
forms -- that means a peering problem, not a transient."""
|
||||
await ctx.wait_for_nodes(
|
||||
lambda x: _peers(ctx, x) >= MESH_MIN, nodes=ALL_NODES, timeout=timeout
|
||||
)
|
||||
peers = {nid: _peers(ctx, nid) for nid in ALL_NODES}
|
||||
log(f"peer mesh healthy (>= {MESH_MIN} peers each): {peers}")
|
||||
|
||||
|
||||
async def scenario(ctx, log):
|
||||
# 1. all-old net healthy AND the full peer mesh has formed before we touch it
|
||||
await ctx.wait_for_ledger_close(timeout=90)
|
||||
base = (await ctx.wait_for_ledgers(2, timeout=120)).result
|
||||
for nid in ALL_NODES:
|
||||
await ctx.wait_for_ledger(base, node_id=nid, timeout=120)
|
||||
await _wait_mesh(ctx, log, timeout=180)
|
||||
log(f"all-old net healthy at ledger {base} (CE inactive)")
|
||||
|
||||
# 2. rolling-upgrade validators (then one tracker), one at a time. After each
|
||||
# restart, give the mesh time to re-form (the restarted node re-dials its
|
||||
# fixed peers) BEFORE rolling the next, so we never take out a junction.
|
||||
for nid in [*VALIDATORS, UPGRADED_TRACKER]:
|
||||
ref = next(v for v in VALIDATORS if v != nid)
|
||||
role = "validator" if nid in VALIDATORS else "tracker"
|
||||
log(f"rolling upgrade: n{nid} ({role}) -> @export-rng")
|
||||
await ctx.restart_node_with_binary(nid, "@export-rng", delay=3)
|
||||
await _wait_mesh(ctx, log, timeout=180) # mesh re-formed before next roll
|
||||
target = (await ctx.wait_for_ledgers(2, node_id=ref, timeout=180)).result
|
||||
await ctx.wait_for_ledger(target, node_id=nid, timeout=180)
|
||||
log(f"n{nid} rejoined; mesh re-formed; quorum advanced to {target}")
|
||||
log(
|
||||
f"validators + n{UPGRADED_TRACKER} on @export-rng; "
|
||||
f"n{STRAGGLER_TRACKER} left on @release; CE still inactive"
|
||||
)
|
||||
|
||||
# 3. vote CE up on the validators (real vote; the seed only pre-satisfied the hold)
|
||||
ctx.feature(CONSENSUS_ENTROPY_FEATURE, vetoed=False, nodes=VALIDATORS)
|
||||
log("voted ConsensusEntropy accept on n0-n4; crossing the flag ledger...")
|
||||
|
||||
# 4. activation at the flag ledger
|
||||
await ctx.wait_for_feature(
|
||||
CONSENSUS_ENTROPY_FEATURE,
|
||||
check=lambda s: s.get("enabled"),
|
||||
nodes=VALIDATORS,
|
||||
timeout=900,
|
||||
)
|
||||
log("ConsensusEntropy ENABLED on the upgraded quorum")
|
||||
|
||||
# 5. upgraded tracker keeps working; straggler is amendment-blocked (not crashed)
|
||||
await ctx.wait_for_nodes(
|
||||
lambda x: _blocked(ctx, x), nodes=[STRAGGLER_TRACKER], timeout=180
|
||||
)
|
||||
ctx.assert_log("server blocked", nodes=[STRAGGLER_TRACKER])
|
||||
assert not _blocked(ctx, UPGRADED_TRACKER), (
|
||||
f"n{UPGRADED_TRACKER} (upgraded tracker) should NOT be amendment-blocked"
|
||||
)
|
||||
await ctx.wait_for_ledgers(2, node_id=UPGRADED_TRACKER, timeout=120)
|
||||
for nid in (UPGRADED_TRACKER, STRAGGLER_TRACKER):
|
||||
assert ctx.rpc.server_info(nid), f"n{nid} RPC down (crashed?)"
|
||||
log(
|
||||
f"PASS: rolling upgrade preserved quorum 4 (mesh-gated); CE activated; "
|
||||
f"n{UPGRADED_TRACKER} (upgraded tracker) still tracking; "
|
||||
f"n{STRAGGLER_TRACKER} (@release straggler) amendment-blocked and still running"
|
||||
)
|
||||
42
.testnet/scenarios/rollout/rollout-suite.yml
Normal file
42
.testnet/scenarios/rollout/rollout-suite.yml
Normal file
@@ -0,0 +1,42 @@
|
||||
# Rolling binary upgrade -> ConsensusEntropy activation.
|
||||
#
|
||||
# All nodes start on @release (set via `--rippled-path @release` on the CLI); the
|
||||
# scenario rolling-restarts them to @export-rng, then casts a REAL vote. The
|
||||
# --seed-majority (majority_features) only pre-satisfies the amendment hold time
|
||||
# so activation lands at ONE flag ledger instead of two -- it is NOT a vote, and
|
||||
# is cleared (tfLostMajority) if the validators don't vote yes before the flag
|
||||
# ledger. start_ledger gives runway for the 6 restarts + vote to finish before
|
||||
# the first flag ledger (256), so the seed survives to a real vote.
|
||||
#
|
||||
# Run:
|
||||
# x-testnet --rippled-path @release suite \
|
||||
# .testnet/scenarios/rollout/rollout-suite.yml --stop-on-fail
|
||||
#
|
||||
# Suite runs auto-snapshot to .testnet/output/runs/ on failure (survives
|
||||
# teardown), so `x-testnet logs-search --run latest/rolling_upgrade_ce ...` works.
|
||||
|
||||
defaults:
|
||||
network:
|
||||
node_count: 7 # n0-n4 validators, n5/n6 non-UNL trackers
|
||||
validators: 5
|
||||
quorum: 4 # 80%: a restart leaves exactly quorum on the peers up
|
||||
find_ports: true
|
||||
start_ledger: 210 # measured: the mesh-gated upgrade+vote is ~23 ledgers, so
|
||||
# the vote lands ~233 with a ~23-ledger margin before flag
|
||||
# ledger 256 (activation). Miss -> vote after 256 -> next
|
||||
# flag is 512 (~13 min) or tfLostMajority clears the seed.
|
||||
majority_features:
|
||||
- ConsensusEntropy # pre-satisfy the hold ONLY (still needs a real vote)
|
||||
track_features:
|
||||
- ConsensusEntropy
|
||||
log_levels:
|
||||
LedgerConsensus: debug
|
||||
NetworkOPs: info
|
||||
# Dense localhost mesh (needed for rolling restarts): the generator gives each
|
||||
# peer a distinct 127.0.0.<id+1> in [ips_fixed] (rippled dedups fixed peers by
|
||||
# IP, ignoring port). On macOS bring those aliases up first (see below); Linux
|
||||
# routes all of 127/8 already. Works with stock binaries -- no rebuild.
|
||||
|
||||
tests:
|
||||
- name: rolling_upgrade_ce
|
||||
script: .testnet/scenarios/rollout/rolling_upgrade.py
|
||||
@@ -16,8 +16,6 @@ defaults:
|
||||
LedgerConsensus: debug
|
||||
ConsensusExtensions: debug
|
||||
NetworkOPs: info
|
||||
env:
|
||||
XAHAU_RESOURCE_PER_PORT: "1"
|
||||
rc:
|
||||
- rng_poll_ms=333
|
||||
|
||||
|
||||
Reference in New Issue
Block a user