diff --git a/docker/telemetry/.env.devbox.example b/docker/telemetry/.env.devbox.example new file mode 100644 index 0000000000..782808d2e8 --- /dev/null +++ b/docker/telemetry/.env.devbox.example @@ -0,0 +1,16 @@ +# Host-specific values for the two-node deployment. Copy to .env.devbox on the +# host and fill in; .env.* is gitignored, so the real file is never committed. +# +# Keeping these out of the templates is deliberate: this repository is public, +# and an earlier commit already removed a personal home directory from shipped +# config. Nothing host-identifying should return to git. + +# Account xrpld runs as. +RUN_USER= + +# Absolute path of the repository checkout on this host. +REPO_DIR= + +# Mount holding the node data directories. The units require it, so a missing +# disk fails the unit instead of silently filling the root filesystem. +DATA_MOUNT= diff --git a/docker/telemetry/.gitignore b/docker/telemetry/.gitignore index 41139c450a..dce518044f 100644 --- a/docker/telemetry/.gitignore +++ b/docker/telemetry/.gitignore @@ -8,6 +8,7 @@ data/ !.env.alerting.example !.env.grafanacloud.example !.env.grafanacloud-alloy.example +!.env.devbox.example # Do not commit grafana cloud versions with default filters grafanacloud/*.json diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 2dbbcccd49..5f5f88d4e1 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -41,6 +41,9 @@ receivers: filelog: include: - /var/log/xrpld/*/debug.log + # Stamp each record with its source path so per-node identity can be derived + # from it below. + include_file_path: true operators: - type: regex_parser regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' @@ -48,6 +51,21 @@ receivers: parse_from: attributes.timestamp layout: "%Y-%b-%d %H:%M:%S.%f" location: UTC + # Derive per-node identity from the log path. By convention the directory + # basename is the node's service_instance_id + # (/var/log/xrpld/xrpld-mainnet/debug.log), so promote it to the + # service.instance.id resource attribute. Without this the logs pipeline + # carries no service_instance_id label while the metrics pipeline does, so + # the dashboards' $node filter matches nothing for logs while still working + # for metrics -- a partial failure that reads as "no logs" rather than as a + # misconfiguration. Both node configs name their log directory after their + # service_instance_id for exactly this reason. + - type: regex_parser + parse_from: attributes["log.file.path"] + regex: '/xrpld/(?P[^/]+)/debug\.log$' + - type: move + from: attributes.node_instance + to: resource["service.instance.id"] processors: batch: diff --git a/docker/telemetry/systemd/README.md b/docker/telemetry/systemd/README.md new file mode 100644 index 0000000000..0dd833c770 --- /dev/null +++ b/docker/telemetry/systemd/README.md @@ -0,0 +1,103 @@ +# Running two xrpld instances on one host + +Templates and an installer for running **two nodes side by side**, one per +storage backend, so NuDB and RocksDB can be compared with everything else equal. + +These are committed deliberately. They previously existed only on the host and +were lost when that machine was rebuilt, which meant reconstructing the RocksDB +node's settings from notes. + +## What differs between the two + +Everything else is identical on purpose — any other divergence would confound the +backend comparison. + +| | instance 1 | instance 2 | +| --- | --- | --- | +| Unit | `xrpld-mainnet` | `xrpld-mainnet2` | +| Config | `xrpld-telemetry-mainnet.cfg` | `xrpld-telemetry-mainnet2.cfg` | +| `service_instance_id` | `xrpld-mainnet` | `xrpld-mainnet2` | +| Backend | NuDB | RocksDB | +| rpc / ws-admin / ws-public / peer | 5015 / 6016 / 6015 / 51245 | 5025 / 6026 / 6025 / 51255 | +| Data | `data/mainnet` | `data2/mainnet` | +| Logs | `data/logs/xrpld-mainnet/` | `data2/logs/xrpld-mainnet2/` | + +Ports continue the offset-by-ten scheme already in use — devnet on 5005, Mainnet +on 5015 — so all three configs can bind on one host. + +## No host-specific values in git + +This repository is public, and an earlier commit already removed a personal home +directory from shipped config. The unit templates therefore carry placeholders, +and the real values live in an untracked `.env.devbox`: + +```sh +cp docker/telemetry/.env.devbox.example docker/telemetry/.env.devbox +chmod 600 docker/telemetry/.env.devbox +$EDITOR docker/telemetry/.env.devbox # RUN_USER, REPO_DIR, DATA_MOUNT +``` + +`.env.*` is gitignored, so the real file cannot be committed. The installer +refuses to run if the file is not mode `600`, and refuses to install a unit that +still contains an unsubstituted placeholder. + +## Install + +```sh +sh docker/telemetry/systemd/install-units.sh +sudo systemctl start xrpld-mainnet # wait for tracking/full +sudo systemctl start xrpld-mainnet2 +``` + +Staggering the starts is a nicety rather than a requirement when the data +directories are on fast local storage, but two bootstrapping nodes still contend +for the job pool. + +## Two things that are easy to get wrong + +**The log directory basename must equal the `service_instance_id`.** The +collector's filelog receiver derives per-node identity from the log path +(`include_file_path` plus a regex on `/xrpld//debug.log`). Name the directory +anything else and that node's *logs* lose their `service_instance_id` label while +its *metrics* keep theirs — so the dashboards' `$node` filter matches nothing for +logs and reads as "no logs" rather than as a misconfiguration. The collector +expects the logs under `/var/log/xrpld//`, so symlink or bind-mount each +node's log directory there. + +**Put the data directories on fast local storage.** The configs use +repo-relative paths so they stay portable; point them at the fast disk with +symlinks: + +```sh +mkdir -p "$DATA_MOUNT"/xrpld/data "$DATA_MOUNT"/xrpld/data2 +ln -sfn "$DATA_MOUNT"/xrpld/data docker/telemetry/data +ln -sfn "$DATA_MOUNT"/xrpld/data2 docker/telemetry/data2 +``` + +Measured on an i4i instance: moving the nodestore off EBS onto the instance-store +NVMe took time-to-`full` from 2234 s to 681 s (3.3x), because NuDB's roughly +tenfold key-file write amplification saturated EBS while the local NVMe sat near +idle. + +If that storage is an instance store, note it is **volatile** — contents survive +a reboot but are lost on a stop/start. Both units carry `RequiresMountsFor`, so a +missing mount fails the unit loudly instead of silently filling the root +filesystem. + +## Telemetry + +Both instances export OTLP to the collector on `localhost:4318`. Bring the +collector up with **both** compose files: + +```sh +docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml up -d --force-recreate +``` + +The base file alone yields a collector with only local exporters, so telemetry +silently never leaves the host and the Cloud dashboards read empty with no error +anywhere. `--force-recreate` is needed after a config change, or Compose reuses +the running container with its old config. + +Grafana Cloud credentials go in `.env.grafanacloud`, also gitignored, also mode +`600`. diff --git a/docker/telemetry/systemd/install-units.sh b/docker/telemetry/systemd/install-units.sh new file mode 100755 index 0000000000..96a1501fad --- /dev/null +++ b/docker/telemetry/systemd/install-units.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# Install the two-node systemd units, filling host-specific values from an +# untracked .env.devbox alongside the telemetry configs. +# +# The templates carry placeholders rather than real values because this +# repository is public: an earlier commit already removed a personal home +# directory from shipped config, and nothing host-identifying should return to +# git. .env.devbox holds those values on the host only and is covered by the +# .env.* ignore rule. +# +# Usage, from the repository root: +# sh docker/telemetry/systemd/install-units.sh +# +# Written in POSIX sh so it runs under bash, dash and ash alike. +set -eu + +here=$(cd "$(dirname "$0")" && pwd) +tel=$(dirname "$here") +env_file="$tel/.env.devbox" + +if [ ! -f "$env_file" ]; then + echo "ERROR: $env_file not found." >&2 + echo " Copy $tel/.env.devbox.example to it and fill in the values." >&2 + exit 1 +fi + +# Refuse a world-readable env file: it names the account xrpld runs as, and this +# script is the only thing that should be reading it. +mode=$(stat -c %a "$env_file" 2>/dev/null || echo "") +case "$mode" in + 600|400) : ;; + "") echo "WARN: could not read permissions of $env_file" >&2 ;; + *) echo "ERROR: $env_file is mode $mode; expected 600. Run: chmod 600 $env_file" >&2; exit 1 ;; +esac + +# shellcheck disable=SC1090 +. "$env_file" + +for var in RUN_USER REPO_DIR DATA_MOUNT; do + eval "val=\${$var:-}" + if [ -z "$val" ]; then + echo "ERROR: $var is empty in $env_file" >&2 + exit 1 + fi +done + +# Fail early on values that would produce a unit systemd silently never starts. +id "$RUN_USER" >/dev/null 2>&1 || { echo "ERROR: user '$RUN_USER' does not exist" >&2; exit 1; } +[ -d "$REPO_DIR" ] || { echo "ERROR: REPO_DIR '$REPO_DIR' is not a directory" >&2; exit 1; } +[ -x "$REPO_DIR/.build/xrpld" ] || echo "WARN: $REPO_DIR/.build/xrpld not built yet; the unit will fail to start until it is" >&2 +[ -d "$DATA_MOUNT" ] || echo "WARN: DATA_MOUNT '$DATA_MOUNT' does not exist yet; the unit will refuse to start until it is mounted" >&2 + +for unit in xrpld-mainnet xrpld-mainnet2; do + tpl="$here/$unit.service.template" + [ -f "$tpl" ] || { echo "ERROR: missing template $tpl" >&2; exit 1; } + out=$(mktemp) + sed -e "s|__RUN_USER__|$RUN_USER|g" \ + -e "s|__REPO_DIR__|$REPO_DIR|g" \ + -e "s|__DATA_MOUNT__|$DATA_MOUNT|g" \ + "$tpl" > "$out" + + if grep -q '__[A-Z_]*__' "$out"; then + echo "ERROR: unsubstituted placeholder left in $unit:" >&2 + grep -o '__[A-Z_]*__' "$out" | sort -u | sed 's/^/ /' >&2 + rm -f "$out" + exit 1 + fi + + sudo install -m 0644 "$out" "/etc/systemd/system/$unit.service" + rm -f "$out" + echo "installed /etc/systemd/system/$unit.service" +done + +sudo systemctl daemon-reload +echo "done. Start them staggered:" +echo " sudo systemctl start xrpld-mainnet # wait for tracking/full" +echo " sudo systemctl start xrpld-mainnet2" diff --git a/docker/telemetry/systemd/xrpld-mainnet.service.template b/docker/telemetry/systemd/xrpld-mainnet.service.template new file mode 100644 index 0000000000..405a96b2e9 --- /dev/null +++ b/docker/telemetry/systemd/xrpld-mainnet.service.template @@ -0,0 +1,38 @@ +# xrpld-mainnet -- xrpld-mainnet, NuDB backend. +# +# TEMPLATE. The placeholders below are filled in by install-units.sh from an +# untracked .env.devbox on the host, so no host-specific user or path is ever +# committed. Do not replace them by hand in this file. +# +# __RUN_USER__ the account xrpld runs as +# __REPO_DIR__ absolute path of the repository checkout +# __DATA_MOUNT__ mount the node's data directory lives on + +[Unit] +Description=xrpld (xrpld-mainnet, NuDB) with OpenTelemetry +After=network-online.target +Wants=network-online.target +# The nodestore normally lives on a separate mount whose fstab entry uses +# 'nofail', so a missing disk does not block boot -- which also means it fails +# silently. Without this the node would start anyway and write the nodestore to +# the root filesystem until it filled. Requiring the mount fails the unit loudly +# instead. +RequiresMountsFor=__DATA_MOUNT__ +StartLimitIntervalSec=5min +StartLimitBurst=5 + +[Service] +Type=simple +User=__RUN_USER__ +Group=__RUN_USER__ +WorkingDirectory=__REPO_DIR__ +ExecStart=__REPO_DIR__/.build/xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg +Restart=on-failure +RestartSec=5s +# xrpld flushes its OpenTelemetry buffers on shutdown; killing it early loses the +# final metrics export. +TimeoutStopSec=5min +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/docker/telemetry/systemd/xrpld-mainnet2.service.template b/docker/telemetry/systemd/xrpld-mainnet2.service.template new file mode 100644 index 0000000000..b1cf042f18 --- /dev/null +++ b/docker/telemetry/systemd/xrpld-mainnet2.service.template @@ -0,0 +1,38 @@ +# xrpld-mainnet2 -- xrpld-mainnet2, RocksDB backend. +# +# TEMPLATE. The placeholders below are filled in by install-units.sh from an +# untracked .env.devbox on the host, so no host-specific user or path is ever +# committed. Do not replace them by hand in this file. +# +# __RUN_USER__ the account xrpld runs as +# __REPO_DIR__ absolute path of the repository checkout +# __DATA_MOUNT__ mount the node's data directory lives on + +[Unit] +Description=xrpld (xrpld-mainnet2, RocksDB) with OpenTelemetry +After=network-online.target +Wants=network-online.target +# The nodestore normally lives on a separate mount whose fstab entry uses +# 'nofail', so a missing disk does not block boot -- which also means it fails +# silently. Without this the node would start anyway and write the nodestore to +# the root filesystem until it filled. Requiring the mount fails the unit loudly +# instead. +RequiresMountsFor=__DATA_MOUNT__ +StartLimitIntervalSec=5min +StartLimitBurst=5 + +[Service] +Type=simple +User=__RUN_USER__ +Group=__RUN_USER__ +WorkingDirectory=__REPO_DIR__ +ExecStart=__REPO_DIR__/.build/xrpld --conf docker/telemetry/xrpld-telemetry-mainnet2.cfg +Restart=on-failure +RestartSec=5s +# xrpld flushes its OpenTelemetry buffers on shutdown; killing it early loses the +# final metrics export. +TimeoutStopSec=5min +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/docker/telemetry/xrpld-telemetry-mainnet.cfg b/docker/telemetry/xrpld-telemetry-mainnet.cfg index 720c9a1b6b..c7b43f8594 100644 --- a/docker/telemetry/xrpld-telemetry-mainnet.cfg +++ b/docker/telemetry/xrpld-telemetry-mainnet.cfg @@ -113,10 +113,10 @@ docker/telemetry/data/mainnet # --- Logging ---------------------------------------------------------------- # Path is resolved relative to this config file's directory (docker/telemetry), -# so this writes to docker/telemetry/data/logs/mainnet/debug.log — the same +# so this writes to docker/telemetry/data/logs/xrpld-mainnet/debug.log — the same # dir the compose stack bind-mounts into the collector as /var/log/xrpld. [debug_logfile] -data/logs/mainnet/debug.log +data/logs/xrpld-mainnet/debug.log [rpc_startup] { "command": "log_level", "severity": "warning" } diff --git a/docker/telemetry/xrpld-telemetry-mainnet2.cfg b/docker/telemetry/xrpld-telemetry-mainnet2.cfg new file mode 100644 index 0000000000..07d500be50 --- /dev/null +++ b/docker/telemetry/xrpld-telemetry-mainnet2.cfg @@ -0,0 +1,189 @@ +# Second Mainnet instance, for running two nodes on one host. +# +# The point of the pair is to compare storage backends with everything else +# equal, so this differs from xrpld-telemetry-mainnet.cfg only where two nodes +# on one host must differ: ports, data and log paths, service_instance_id, and +# [node_db] type. Any other divergence would confound the comparison. +# +# Ports continue the offset-by-ten scheme the other configs already use -- +# devnet on 5005, Mainnet on 5015, so this one on 5025 -- so all three can bind +# on the same host without colliding. +# +# The log directory basename equals service_instance_id on purpose: the +# collector's filelog receiver derives per-node identity from the log path, so a +# directory named anything else leaves this node's logs without a +# service_instance_id label while its metrics keep one, and the dashboards' +# $node filter then matches nothing for logs while still working for metrics. +# +# Host-specific values (run user, repo location) are NOT here. They live in an +# untracked .env.devbox on the host -- see systemd/README.md. +# xrpld configuration for Mainnet with full OpenTelemetry tracing. +# +# Connects to XRP Ledger Mainnet as an observer-only tracking node +# (no validator credentials) and exercises ALL instrumented workflows: +# RPC, transactions, consensus, peer overlay, ledger ops, and +# pathfinding. +# +# Usage: +# 1. Start the observability stack: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# 2. Run xrpld: +# ./xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg +# 3. Wait for sync (server_state=full), then exercise workflows: +# curl -s http://localhost:5015 -d '{"method":"server_info"}' +# 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 + +# --- Server ports ----------------------------------------------------------- + +# Ports are offset by +10 from the devnet config (xrpld-telemetry.cfg) so both +# nodes can run at the same time. They are host processes sharing one network +# namespace, so identical ports would leave the second node unable to bind. + +[server] +port_rpc_admin_local +port_ws_admin_local +port_ws_public +port_peer + +[port_rpc_admin_local] +port = 5025 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_ws_admin_local] +port = 6026 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = ws + +# Bound to loopback: this port has no `admin` key, so every caller resolves to +# Role::GUEST. Every workflow documented for this config is driven over the two +# admin ports above, which also serve WebSocket, and this node follows Mainnet, so +# there is no reason to accept off-host clients. +[port_ws_public] +port = 6025 +ip = 127.0.0.1 +protocol = ws + +# Stays on all interfaces: this is the peer-protocol listener, and binding it to +# loopback would stop inbound overlay connections. +[port_peer] +port = 51255 +ip = 0.0.0.0 +protocol = peer + +# --- Network ---------------------------------------------------------------- + +[network_id] +main + +# [ips] omitted on purpose. xrpld uses its built-in mainnet hub list +# (r.ripple.com, sahyadri.isrdc.in, hubs.xrpkuwait.com, +# hub.xrpl-commons.org) for peer discovery. + +[validators_file] +validators-mainnet.txt + +[peer_private] +0 + +[peers_max] +21 + +# --- Pathfinding (exercises ripple_path_find / path_find workflows) --------- + +[path_search] +7 + +[path_search_fast] +2 + +[path_search_max] +10 + +# --- Signing ---------------------------------------------------------------- + +# [signing_support] is deliberately omitted (it defaults to false). It is only +# consulted for non-admin callers, and every signing path in this repo already +# runs as admin over the loopback admin ports above, so enabling it would add +# nothing except exposing sign/sign_for/channel_authorize to guests on a node +# that follows Mainnet. Upstream also deprecates these commands. + +# --- Database --------------------------------------------------------------- + +# Paths carry the network name so this node never shares a store with the devnet +# config. Both are relative to the working directory (the repo root, per the +# usage note above), so an unqualified `data/` would have the two nodes opening +# the same NuDB and the same SQLite ledger databases — including the case where +# they run one after the other rather than concurrently. +[node_db] +type=RocksDB +path=docker/telemetry/data2/mainnet/rocksdb +open_files=2000 +filter_bits=12 +cache_mb=256 +file_size_mb=8 +file_size_mult=2 +online_delete=2000 +advisory_delete=0 + +[database_path] +docker/telemetry/data2/mainnet + +[ledger_history] +1000 + +# --- Logging ---------------------------------------------------------------- + +# Path is resolved relative to this config file's directory (docker/telemetry), +# so this writes to docker/telemetry/data2/logs/xrpld-mainnet2/debug.log — the same +# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +[debug_logfile] +data2/logs/xrpld-mainnet2/debug.log + +[rpc_startup] +{ "command": "log_level", "severity": "warning" } + +# --- SSL -------------------------------------------------------------------- + +# Keep the secure default (1). The Mainnet validator lists in +# validators-mainnet.txt are fetched from public HTTPS publishers that present +# valid CA certificates, so there is no self-signed cert to work around here. +# Setting this to 0 would skip both the certificate chain check and the +# hostname match on those fetches. The list blob's own signature is verified +# against the pinned publisher keys either way; this adds TLS peer +# authentication on top of that. +[ssl_verify] +1 + +# --- Insight (native OTel metrics via beast::insight) ----------------------- + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +# Sets the OTel service.instance.id resource attribute, which Prometheus +# exposes as the `service_instance_id` label. Dashboards filter on it via the +# $node template variable, so without this every insight-backed panel is +# empty. Matches [telemetry] service_instance_id for a single node identity. +service_instance_id=xrpld-mainnet2 + +# --- OpenTelemetry tracing -------------------------------------------------- + +[telemetry] +enabled=1 +service_instance_id=xrpld-mainnet2 +endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics +# Mainnet has high span throughput across peer/ledger/consensus. Head +# sampling is fixed at 1.0 (sample everything) and not configurable; +# reduce Tempo/collector load with collector-side tail sampling. +batch_size=512 +batch_delay_ms=5000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1