diff --git a/docker/telemetry/.gitignore b/docker/telemetry/.gitignore index 366d329bf4..85b4980ab6 100644 --- a/docker/telemetry/.gitignore +++ b/docker/telemetry/.gitignore @@ -1,5 +1,9 @@ -# Runtime data generated by xrpld and telemetry stack -data/ +# Runtime data generated by xrpld and telemetry stack. No trailing slash, so the +# patterns also match the symlinks a host points at its fast disk -- a +# trailing-slash pattern matches only real directories, which left both paths +# showing as untracked on exactly the hosts that follow the runbook. +data +data2 # Env. file carrying environmental setup data for local or cloud runs. .env.* diff --git a/docker/telemetry/RUNBOOK-two-node-host.md b/docker/telemetry/RUNBOOK-two-node-host.md index 8f40505c89..d5e7fef6fd 100644 --- a/docker/telemetry/RUNBOOK-two-node-host.md +++ b/docker/telemetry/RUNBOOK-two-node-host.md @@ -53,21 +53,22 @@ key for every metric, trace and log already stored, so a new name starts a fresh series and no dashboard will show the old and new data together. See [systemd/README.md](systemd/README.md) for why they are not committed. -**Log directories where the collector looks.** The collector reads -`/var/log/xrpld//debug.log` and derives each node's identity from -that directory name, so the basename must match the instance id exactly. Using -the ids from the settings file keeps the two in step: +**Log directories where the collector looks.** The collector bind-mounts one log +root — `data/logs` — and derives each node's identity from the subdirectory name +inside it, so the basename must match the instance id exactly. **Both** nodes log +there, whatever their nodestore uses; a log directory outside that root is simply +never read. Using the ids from the settings file keeps the names in step: ```sh . docker/telemetry/.env.devbox # NODE1_INSTANCE_ID, NODE2_INSTANCE_ID mkdir -p "$FAST_MOUNT"/xrpld/data/logs/"$NODE1_INSTANCE_ID" \ - "$FAST_MOUNT"/xrpld/data2/logs/"$NODE2_INSTANCE_ID" -sudo mkdir -p /var/log/xrpld -sudo ln -sfn "$FAST_MOUNT"/xrpld/data/logs/"$NODE1_INSTANCE_ID" /var/log/xrpld/"$NODE1_INSTANCE_ID" -sudo ln -sfn "$FAST_MOUNT"/xrpld/data2/logs/"$NODE2_INSTANCE_ID" /var/log/xrpld/"$NODE2_INSTANCE_ID" + "$FAST_MOUNT"/xrpld/data/logs/"$NODE2_INSTANCE_ID" ``` +Nothing needs to be placed in the host's `/var/log/xrpld`: that path exists only +_inside_ the collector container, which is where the log root is mounted. + **Rootless Docker.** If the container runtime is rootless, its systemd user units need a session bus to install, and the variable is absent over a plain non-interactive SSH connection — the install appears to run and leaves nothing diff --git a/docker/telemetry/systemd/README.md b/docker/telemetry/systemd/README.md index fe38197838..f7d4c6af1b 100644 --- a/docker/telemetry/systemd/README.md +++ b/docker/telemetry/systemd/README.md @@ -21,7 +21,7 @@ backend comparison. | Backend | NuDB | RocksDB | | rpc / ws-admin / ws-public / peer | 5015 / 6016 / 6015 / 51245 | 5025 / 6026 / 6025 / 51255 | | Data | `data/mainnet` | `data2/mainnet` | -| Logs | `data/logs/$NODE1_INSTANCE_ID/` | `data2/logs/$NODE2_INSTANCE_ID/` | +| Logs | `data/logs/$NODE1_INSTANCE_ID/` | `data/logs/$NODE2_INSTANCE_ID/` | Ports continue the offset-by-ten scheme already in use — devnet on 5005, Mainnet on 5015 — so all three configs can bind on one host. @@ -83,16 +83,23 @@ for the job pool. ## Two things that are easy to get wrong -**The log directory basename must equal the `service_instance_id`.** The -installer substitutes both from one value so they cannot drift, but the -directory itself still has to exist under that name. The -collector's filelog receiver derives per-node identity from the log path -(`include_file_path` plus a regex on `/xrpld//debug.log`). Name the directory -anything else and that node's _logs_ lose their `service_instance_id` label while -its _metrics_ keep theirs — so the dashboards' `$node` filter matches nothing for -logs and reads as "no logs" rather than as a misconfiguration. The collector -expects the logs under `/var/log/xrpld//`, so symlink or bind-mount each -node's log directory there. +**The log directory basename must equal the `service_instance_id`, and sit in the +one log root.** The installer substitutes the id into both the setting and the +path from one value so they cannot drift, but the directory itself still has to +exist under that name. The collector's filelog receiver derives per-node identity +from the log path (`include_file_path` plus a regex on `/xrpld//debug.log`). +Name the directory anything else and that node's _logs_ lose their +`service_instance_id` label while its _metrics_ keep theirs — so the dashboards' +`$node` filter matches nothing for logs and reads as "no logs" rather than as a +misconfiguration. + +The root is `data/logs`, which compose mounts into the collector as +`/var/log/xrpld`. That is a path inside the container, not on the host, so +nothing needs creating in the host's `/var/log`. **Both** nodes log under +`data/logs`, including the one whose nodestore is under `data2/`: a log directory +outside the mounted root is never read at all, and since both data directories +sit on the same disk, splitting the logs would buy no I/O separation to pay for +the lost collection. **Put the data directories on fast local storage.** The configs use repo-relative paths so they stay portable; point them at the fast disk with diff --git a/docker/telemetry/systemd/install-units.sh b/docker/telemetry/systemd/install-units.sh index 8a002e8e11..493c594d95 100755 --- a/docker/telemetry/systemd/install-units.sh +++ b/docker/telemetry/systemd/install-units.sh @@ -147,23 +147,25 @@ render_cfg() { render_cfg "$tel/xrpld-telemetry-mainnet.cfg" "$NODE1_INSTANCE_ID" "$tel/xrpld-telemetry-mainnet.host.cfg" render_cfg "$tel/xrpld-telemetry-mainnet2.cfg" "$NODE2_INSTANCE_ID" "$tel/xrpld-telemetry-mainnet2.host.cfg" -# The log directories have to exist under both the names the node writes and the -# path the collector reads, or one node's logs go unlabelled while everything -# else looks healthy. +# Each node needs a log directory named after its instance id inside the log root +# the collector mounts (data/logs, exposed to the container as /var/log/xrpld). +# A directory missing here, or placed outside that root, costs that node its logs +# while its metrics keep flowing -- which reads as a quiet node, not as a +# collection gap. check_log_dir() { node_dir=$1 id=$2 [ -d "$tel/$node_dir/logs/$id" ] || cat >&2 <&2 } +# Both nodes log under data/, whatever their nodestore uses: the collector +# mounts that one directory as its log root and identifies each node by the +# subdirectory name, so a log directory outside it is never read. check_log_dir data "$NODE1_INSTANCE_ID" -check_log_dir data2 "$NODE2_INSTANCE_ID" +check_log_dir data "$NODE2_INSTANCE_ID" for unit in xrpld-mainnet xrpld-mainnet2; do tpl="$here/$unit.service.template" diff --git a/docker/telemetry/xrpld-telemetry-mainnet2.cfg b/docker/telemetry/xrpld-telemetry-mainnet2.cfg index 07d500be50..8f2adbd8f5 100644 --- a/docker/telemetry/xrpld-telemetry-mainnet2.cfg +++ b/docker/telemetry/xrpld-telemetry-mainnet2.cfg @@ -137,10 +137,16 @@ docker/telemetry/data2/mainnet # --- Logging ---------------------------------------------------------------- # Path is resolved relative to this config file's directory (docker/telemetry), -# so this writes to docker/telemetry/data2/logs/xrpld-mainnet2/debug.log — the same +# so this writes to docker/telemetry/data/logs/xrpld-mainnet2/debug.log — the # dir the compose stack bind-mounts into the collector as /var/log/xrpld. +# +# The logs stay under data/ even though this instance's nodestore is under +# data2/: the collector mounts one log root and finds each node by the +# subdirectory name, so a log directory outside that root is simply never read. +# Both data/ and data2/ point at the same fast disk, so splitting the logs would +# buy no I/O separation to pay for the lost collection. [debug_logfile] -data2/logs/xrpld-mainnet2/debug.log +data/logs/xrpld-mainnet2/debug.log [rpc_startup] { "command": "log_level", "severity": "warning" }