From 3153f3ef56901b12dcd603c5364ba16565dcd94a Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:18:47 +0100 Subject: [PATCH] docs(telemetry): align runbook and plan docs with the shipped phase-9/10 code The reference docs had drifted from the code in ways that break the reader rather than merely misinform: PromQL examples that return no data, a rollback flag that is a no-op, a sampling knob that does not exist, and two span parents that moved. Code is treated as the truth throughout; where the code is the defective side, the doc now records it as a known issue instead of describing the bug as intent. Renames the docs missed: histogram names gain the exporter's unit suffix (ios_latency_milliseconds_bucket and four siblings), ledger_history_mismatch gains _total, the StatsD-era quantile label gives way to le buckets, rpc.request becomes rpc.http_request, traces_spanmetrics_calls_total becomes span_calls_total, and the nine dotted xrpl.* span attributes are recorded as renamed rather than left as live keys. Re-parenting: consensus.update_positions and consensus.check are children of consensus.establish, not of consensus.round. Units and labels: state_accounting_*_duration is microseconds, not seconds; cache_metrics label values are case-sensitive; object_count carries demangled C++ type names. Nodestore read and write latency stays microseconds -- the nanosecond accumulator change did not move the exported unit. Adds what shipped but was undocumented: the ledger.acquire span, seven consensus.round events, twelve span attributes, node_writes_duration_us, the 7-day validation-agreement window, the TxQ admission and reduce-relay metric families, metrics_endpoint, and the phase-10 validation workflow. Corrects claims that never held: 10% head sampling (it is fixed at 100%), configurable redaction (it is unconditional), -DXRPL_ENABLE_TELEMETRY=OFF (the flag is -Dtelemetry=OFF, default ON), FindOpenTelemetry.cmake and the xrpl_telemetry target (neither exists), Promtail and a StatsD exporter in the pipeline (neither exists), and Loki stream selection on job= (only service_name is a stream label). Phase 9 is marked complete, its provisioned alerting is attributed to the branch that shipped it, and Phase 11 stays at zero except the one prerequisite its code closes. Counts are reconciled repo-wide: 41 emitted span families, 15 dashboards on disk with 14 asserted, 13 alert rules in 5 groups. Hardens the gate that let this drift through: Rule E of the naming check now covers the reference docs, its allow-dotted marker is key-scoped and warns on stale or empty use, a missing checked file is reported instead of silently skipped, the test suite runs in CI, and doc paths trigger the check. C++ and CMake changes are comment-only: three MetricsRegistry instrument names, eight OTelCollector claims of a metric-name prefix that formatName never adds, and the telemetry option's inverted default. --- .../scripts/otel-naming/check_otel_naming.py | 215 ++++- .../otel-naming/test_check_otel_naming.py | 611 ++++++++++++++- .github/workflows/on-pr.yml | 10 + .../workflows/reusable-check-otel-naming.yml | 7 + CMakeLists.txt | 18 +- CONTRIBUTING.md | 23 + OpenTelemetryPlan/02-design-decisions.md | 394 +++++++--- .../03-implementation-strategy.md | 248 ++++-- .../05-configuration-reference.md | 578 ++++++++++++-- OpenTelemetryPlan/06-implementation-phases.md | 739 +++++++++++++----- .../07-observability-backends.md | 427 ++++++---- OpenTelemetryPlan/08-appendix.md | 83 +- .../09-data-collection-reference.md | 641 ++++++++++----- OpenTelemetryPlan/OpenTelemetryPlan.md | 69 +- OpenTelemetryPlan/Phase10_taskList.md | 94 ++- OpenTelemetryPlan/Phase11_taskList.md | 172 +++- OpenTelemetryPlan/Phase2_taskList.md | 10 +- OpenTelemetryPlan/Phase3_taskList.md | 33 +- OpenTelemetryPlan/Phase4_taskList.md | 210 +++-- OpenTelemetryPlan/Phase7_taskList.md | 3 +- OpenTelemetryPlan/Phase9_taskList.md | 565 +++++++++---- cfg/xrpld-example.cfg | 35 +- docker/telemetry/TESTING.md | 175 ++++- .../grafana/dashboards/validate_dashboards.py | 1 + .../grafana/provisioning/alerting/rules.yaml | 13 +- docker/telemetry/xrpld-telemetry-mainnet.cfg | 3 +- docker/telemetry/xrpld-telemetry.cfg | 1 - docs/build/telemetry.md | 28 +- docs/telemetry-glossary.md | 12 +- docs/telemetry-runbook.md | 450 ++++++++--- include/xrpl/beast/insight/OTelCollector.h | 10 +- include/xrpl/proto/xrpl.proto | 13 +- .../xrpl/telemetry/TraceContextPropagator.h | 24 +- src/libxrpl/beast/insight/OTelCollector.cpp | 67 +- src/xrpld/telemetry/MetricsRegistry.h | 6 +- 35 files changed, 4598 insertions(+), 1390 deletions(-) diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py index 8cce091e41..132874d8b6 100644 --- a/.github/scripts/otel-naming/check_otel_naming.py +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -47,7 +47,9 @@ Layers L3 tempo : docker/telemetry/tempo.yaml (span filter tags) L4 dashboards: docker/telemetry/grafana/dashboards/*.json (PromQL labels; Loki/LogQL queries exempt -- see LOG_QUERY_DATASOURCES) - L5 runbook : docs/telemetry-runbook.md (attr tables) + L5 docs : every *.md under docs/ and docker/telemetry/ (attr tables; + the set is DISCOVERED, never enumerated -- see + RULE_E_DOC_ROOTS) L6 metrics : MetricsRegistry.cpp instrument labels (native-metric label keys, a valid dashboard-label source besides L1) @@ -71,11 +73,17 @@ Rules (each FAILS the build, when its inputs are present) captures or an in-query `| regexp` stage, so they have no L1/L6 source to resolve against (see LOG_QUERY_DATASOURCES). The exemption is per QUERY, not per file, so a mixed dashboard still has its PromQL panels checked. - E No dotted `xrpl..` attribute key in the runbook (only the - L1 resource attrs xrpl.network.* and the EXTERNAL_INFRA_LABELS dotted - form -- xrpl.work.item/.branch/.node.role -- may be dotted). Span names, - filenames, - OTel-standard keys, and metric labels are not flagged. + E No dotted `xrpl..` attribute key in any L5 doc. The doc set + is DISCOVERED, not listed: every `*.md` under RULE_E_DOC_ROOTS, so a doc + added or renamed inside a root is covered without editing this script. + Only the L1 resource + attrs xrpl.network.* and the EXTERNAL_INFRA_LABELS dotted form -- + xrpl.work.item/.branch/.node.role -- may be dotted. Span names, filenames, + OTel-standard keys, and metric labels are not flagged. A doc that names the + wrong form as a deliberate counter-example opts that mention out with a + key-scoped marker (RULE_E_MARKER_SYNTAX), which exempts ONLY the keys it + lists on the line it appears on -- any other dotted token on that same line + still fails. Warnings (printed, but do NOT fail the build) ---------------------------------------------- @@ -84,6 +92,13 @@ Warnings (printed, but do NOT fail the build) *SpanNames.h (single source of truth); defining one in-place bypasses the naming rules. A warning (not a failure) because the argument may instead be a legitimately dynamic local (e.g. a computed span-name leaf). + E A RULE_E_REQUIRED_DOCS anchor is not present in the tree (renamed, moved, + or deleted -- so doc discovery can no longer be trusted to have found the + docs that matter), an allow-dotted marker names no key (so it exempts + nothing), or it names a key the line no longer mentions (a stale + exemption). Warnings, not failures, because presence-gating must keep + working on partial branches -- but the skip is now visible instead of + silent. Exit code is non-zero if any present-and-enforced rule finds a violation. Warnings never change the exit code. @@ -482,7 +497,7 @@ def main() -> None: run_rule_b_collector(root, l1_keys, report) run_rule_c_tempo(root, l1_keys, report) run_rule_d_dashboards(root, l1_keys, metric_labels, report) - run_rule_e_runbook(root, l1_keys, report) + run_rule_e_docs(root, l1_keys, report) report.render_and_exit() @@ -869,6 +884,107 @@ EXTERNAL_INFRA_LABELS = { } +# L5 doc layer (Rule E): the doc trees whose markdown publishes span-attribute +# keys a reader is expected to copy into a TraceQL/PromQL query. A dotted +# `xrpl.*` key left in any of them hands out a query that silently matches +# nothing, so the whole tree is checked, not just the operator runbook: +# * docs/ operator runbook, telemetry glossary, and the +# build/telemetry guides (recursively) +# * docker/telemetry/ stack bring-up guide; its TraceQL/PromQL examples +# are meant to be pasted verbatim +# +# The set is DISCOVERED (every `*.md` under these roots), never enumerated. Two +# reasons this beats a hardcoded path list: +# * it cannot go stale -- a telemetry doc added, renamed, or moved within a +# root is picked up with no edit here, which is the same "derive it, do not +# hardcode it" principle the rest of this script follows (design +# principle 1), and coverage therefore grows with the docs; and +# * it names only doc ROOTS that ship on the default branch, so the rule can +# never be wired to a path that does not exist there. +# Each root is presence-gated (design principle 2): a branch that carries only +# one of them still gets that one checked. Both roots are also CI path-triggers +# for this check -- see `.github/workflows/on-pr.yml`. +RULE_E_DOC_ROOTS = ( + Path("docs"), + Path("docker") / "telemetry", +) + +# The anchor doc(s) discovery is REQUIRED to find. Discovery on its own can pass +# vacuously: if a root were renamed away, or emptied of markdown, Rule E would +# report a clean zero/near-zero-file run and nobody would notice the layer had +# stopped being checked. Demanding the one doc that is the whole reason Rule E +# exists turns that silent green into a visible warning (plus a count on the OK +# line). Deliberately tiny: this is a tripwire, not a second doc list. Every +# entry must live under RULE_E_DOC_ROOTS, or discovery could never find it. +RULE_E_REQUIRED_DOCS = (Path("docs") / "telemetry-runbook.md",) + + +def rule_e_docs(root: Path) -> List[Path]: + """Discover the Rule E doc set: every `*.md` under RULE_E_DOC_ROOTS. + + Returns repo-relative paths, sorted for stable reporting and de-duplicated + so overlapping roots (say `docs` and `docs/build`) cannot check one file + twice. A root absent from the tree is skipped rather than treated as an + error, so presence gating applies to the roots as it does to every other + layer.""" + found: Set[Path] = set() + for rel_root in RULE_E_DOC_ROOTS: + base = root / rel_root + if not base.is_dir(): + continue + found.update(p.relative_to(root) for p in base.rglob("*.md") if p.is_file()) + return sorted(found) + + +# Line-scoped AND key-scoped opt-out for Rule E. Put it on a line that names a +# dotted key as a deliberate counter-example ("`tx_hash`, not `xrpl.tx.hash`") — +# a mention, not a published attribute key — and list exactly the keys that line +# is allowed to mention: +# +# ... use `tx_hash`, not `xrpl.tx.hash`. +# +# +# The keys are part of the marker so an exemption cannot widen silently: a dotted +# token added to an already-marked line later, and not named in the marker, still +# fails. A marker with an empty key list therefore exempts NOTHING (and warns) — +# it is not a blanket line opt-out. NEVER use the marker to keep a real attribute +# table dotted; fix the table instead. +RULE_E_MARKER_SYNTAX = "" +# The marker itself; group(1) is the raw key list (empty for the bare form). +# `[^>]` keeps the match inside one comment, so a marker cannot swallow the rest +# of the line, and the whole thing is matched per line (never across lines). +RULE_E_MARKER = re.compile(r"") +# Keys inside the marker are separated by commas and/or whitespace. +RULE_E_MARKER_KEY_SEP = re.compile(r"[,\s]+") +# The only doc form Rule E flags: a backticked dotted `xrpl..`. +RULE_E_DOTTED_TOKEN = re.compile(r"`(xrpl\.[a-z][a-z0-9_.]*)`") + + +def rule_e_allowed_keys(line: str) -> Tuple[Set[str], int]: + """Parse every Rule-E allow-dotted marker on one line. + + Returns `(keys named by the markers, number of markers seen)`. The count is + returned separately so the caller can tell "no marker" (enforce everything, + silently) from "a marker that names no key" (enforce everything, and warn + that the marker does nothing). + + Keys may be written bare or backticked and separated by commas and/or + spaces, so both of these parse to the same two keys:: + + + + """ + keys: Set[str] = set() + count = 0 + for m in RULE_E_MARKER.finditer(line): + count += 1 + for raw in RULE_E_MARKER_KEY_SEP.split(m.group(1)): + token = raw.strip().strip("`") + if token: + keys.add(token) + return keys, count + + # Datasource types whose query language draws its label names from the log # stream at query time rather than from anything this repo's OTel code emits. # @@ -1059,20 +1175,33 @@ def run_rule_d_dashboards( report.ok(note + ")") -def run_rule_e_runbook(root: Path, l1_keys: Set[str], report: Report) -> None: - path = root / "docs" / "telemetry-runbook.md" - if not path.is_file(): - report.skip("E", "runbook not present") +def run_rule_e_docs(root: Path, l1_keys: Set[str], report: Report) -> None: + discovered = rule_e_docs(root) + if not discovered: + roots = ", ".join(str(rel) for rel in RULE_E_DOC_ROOTS) + report.skip("E", f"no doc-layer file present (no *.md under {roots})") return + # Presence-gating (design principle 2) must stay VISIBLE. Discovery cannot go + # stale, but it can go QUIET: a root renamed away, or stripped of its + # telemetry docs, still yields a clean run over whatever markdown is left, so + # the layer would stop being checked while the rule reported green. Warn per + # missing anchor — non-fatal, so a partial branch still passes — and name the + # count on the OK line, so the doc set cannot quietly shrink. + missing_required = [ + rel for rel in RULE_E_REQUIRED_DOCS if not (root / rel).is_file() + ] + for rel in missing_required: + report.warning( + "E", str(rel), "absent", "required Rule E doc not in tree (renamed?)" + ) if not l1_keys: report.skip("E", "no L1 key set to validate against") return - text = read_source(path) found = False # Only the dotted `xrpl..` attribute form is a violation. The # `xrpl.`-with-trailing-dot anchor is the discriminator: it matches the old # dotted attribute convention being migrated away from, while everything - # else legitimately dotted in the runbook does NOT match it — + # else legitimately dotted in these docs does NOT match it — # * span names (`consensus.round`, `tx.process`) no `xrpl.` prefix # * filenames (`xrpld.cfg`, `RCLConsensus.cpp`) `xrpld.`/`.cpp`, not `xrpl.` # * OTel-standard (`service.name`, `http.method`) no `xrpl.` prefix @@ -1085,19 +1214,57 @@ def run_rule_e_runbook(root: Path, l1_keys: Set[str], report: Report) -> None: # identities dotted (xrpl.work.item/.branch/.node.role -- see the alloy # pipeline that owns them), so also skip a token whose dotted-to-underscore # form is in that set. + # A doc that TEACHES the convention has to be able to name the wrong form as + # a counter-example ("`tx_hash`, not `xrpl.tx.hash`"). The allow-dotted + # marker is the opt-out for exactly that: the token is a mention, not a + # published attribute key. It follows the repo's existing inline-marker + # precedent (``), and is scoped BOTH to the line + # and to the keys it names, so it can neither exempt a whole table nor grow + # to cover a violation appended to an already-marked line. external_infra_dotted = {lbl.replace("_", ".") for lbl in EXTERNAL_INFRA_LABELS} - for m in re.finditer(r"`(xrpl\.[a-z][a-z0-9_.]*)`", text): - token = m.group(1) - if token in l1_keys: # legitimate dotted resource attr (xrpl.network.*) - continue - if token in external_infra_dotted: # perf-iac resource-attribute layer - continue - found = True - report.violation( - "E", str(path.relative_to(root)), token, "underscore, not dotted" - ) + for rel in discovered: + for lineno, line in enumerate(read_source(root / rel).splitlines(), start=1): + allowed, markers = rule_e_allowed_keys(line) + if markers and not allowed: + report.warning( + "E", + f"{rel}:{lineno}", + "allow-dotted", + f"marker names no key: exempts nothing. Use {RULE_E_MARKER_SYNTAX}", + ) + tokens = [m.group(1) for m in RULE_E_DOTTED_TOKEN.finditer(line)] + for token in tokens: + if token in l1_keys: # legitimate dotted resource attr (xrpl.network.*) + continue + if token in external_infra_dotted: # perf-iac resource-attr layer + continue + if token in allowed: # named counter-example on this line + continue + found = True + report.violation( + "E", + f"{rel}:{lineno}", + token, + "underscore, not dotted", + ) + # A key the marker names but the line no longer mentions is a stale + # exemption: harmless today, but it is how a marker silently starts + # covering more than the author reviewed. Flagged, never fatal. + for stale in sorted(allowed.difference(tokens)): + report.warning( + "E", f"{rel}:{lineno}", stale, "allow-dotted key not on this line" + ) if not found: - report.ok("E: runbook attribute references consistent with L1") + note = ( + "E: doc attribute references consistent with L1 " + f"({len(discovered)} file(s) checked" + ) + if missing_required: + note += ( + f", {len(missing_required)} of {len(RULE_E_REQUIRED_DOCS)} " + "required doc(s) absent" + ) + report.ok(note + ")") if __name__ == "__main__": diff --git a/.github/scripts/otel-naming/test_check_otel_naming.py b/.github/scripts/otel-naming/test_check_otel_naming.py index 80af35880c..055e43b28e 100644 --- a/.github/scripts/otel-naming/test_check_otel_naming.py +++ b/.github/scripts/otel-naming/test_check_otel_naming.py @@ -9,7 +9,7 @@ Run from anywhere: Each rule is exercised in isolation against a synthetic tree / synthetic L1 key set, covering positive (must flag), negative (must not flag), and boundary -cases. Rule E (runbook dotted-attribute detection) has the densest coverage +cases. Rule E (doc-layer dotted-attribute detection) has the densest coverage because its discriminator — the `xrpl..` prefix vs span names, filenames, OTel-standard keys, and metric labels — is the subtlest. """ @@ -19,6 +19,7 @@ import importlib.util import io import json import shutil +import subprocess import tempfile import unittest from pathlib import Path @@ -47,13 +48,25 @@ L1 = { def _run_rule_e(runbook_text: str): """Run Rule E against a synthetic runbook; return the flagged tokens.""" + return _run_rule_e_docs({"docs/telemetry-runbook.md": runbook_text})[0] + + +def _run_rule_e_docs(docs, l1_keys=None): + """Run Rule E against a synthetic doc set. + + `docs` maps a repo-relative path to that file's text; only the listed files + are created, so per-file presence gating can be exercised. Returns + (sorted flagged tokens, the Report) so location and skip/ok lines can be + asserted as well as the tokens.""" d = Path(tempfile.mkdtemp()) try: - (d / "docs").mkdir() - (d / "docs" / "telemetry-runbook.md").write_text(runbook_text) + for rel, text in docs.items(): + path = d / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) report = chk.Report() - chk.run_rule_e_runbook(d, set(L1), report) - return sorted(v[2] for v in report.violations) + chk.run_rule_e_docs(d, set(L1) if l1_keys is None else set(l1_keys), report) + return sorted(v[2] for v in report.violations), report finally: shutil.rmtree(d) @@ -166,7 +179,7 @@ class RuleERunbook(unittest.TestCase): d = Path(tempfile.mkdtemp()) try: report = chk.Report() - chk.run_rule_e_runbook(d, set(L1), report) + chk.run_rule_e_docs(d, set(L1), report) self.assertEqual(report.violations, []) self.assertTrue(any("SKIP: E" in s for s in report.skips)) finally: @@ -178,13 +191,593 @@ class RuleERunbook(unittest.TestCase): (d / "docs").mkdir() (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") report = chk.Report() - chk.run_rule_e_runbook(d, set(), report) + chk.run_rule_e_docs(d, set(), report) self.assertEqual(report.violations, []) self.assertTrue(any("SKIP: E" in s for s in report.skips)) finally: shutil.rmtree(d) +# The doc paths the Rule E tests build synthetic trees from. All four are real, +# committable files that live under RULE_E_DOC_ROOTS, so a test tree has the same +# shape discovery meets in the repo. GLOSSARY and BUILD_GUIDE additionally prove +# discovery is not runbook-shaped: BUILD_GUIDE sits one directory deeper, so a +# non-recursive glob would miss it. +RUNBOOK_DOC = "docs/telemetry-runbook.md" +GLOSSARY_DOC = "docs/telemetry-glossary.md" +BUILD_GUIDE_DOC = "docs/build/telemetry.md" +TESTING_DOC = "docker/telemetry/TESTING.md" +ALL_DOCS = (RUNBOOK_DOC, GLOSSARY_DOC, BUILD_GUIDE_DOC, TESTING_DOC) + +# Top-level trees that ship to the default branch, so anything the check is +# configured to read must live inside one of them. A configured path outside +# these would name a tree that is not part of the shipped repository: the rule +# would warn forever about a doc that can never appear, and the matching CI +# path-trigger would be dead weight. +COMMITTABLE_ROOTS = (Path("docs"), Path("docker")) + + +def _inside_any(rel: Path, roots) -> bool: + """True if `rel` is one of `roots` or lives underneath one of them.""" + return any(root == rel or root in rel.parents for root in roots) + + +def _git_tracks(root: Path, rel: Path) -> bool: + """True if git tracks `rel` (a file) or anything under it (a directory). + + Evidence that a configured path is genuinely part of the repository rather + than a local scratch or ignored directory.""" + out = subprocess.run( + ["git", "ls-files", "--", str(rel)], + cwd=str(root), + capture_output=True, + text=True, + ) + return bool(out.stdout.strip()) + + +class RuleEDocSet(unittest.TestCase): + """Rule E scans every discovered doc, not just the runbook. The glossary, the + build/telemetry guide and the stack testing guide all publish attribute keys + readers copy queries from, so a dotted key left in any of them is as + operator-breaking as one in the runbook.""" + + RUNBOOK = RUNBOOK_DOC + GLOSSARY = GLOSSARY_DOC + BUILD_GUIDE = BUILD_GUIDE_DOC + TESTING = TESTING_DOC + + def test_flags_dotted_attr_in_testing_guide(self): + # TESTING.md publishes copy-paste TraceQL/PromQL, so a dotted key there + # hands the reader a query that silently matches nothing. + tokens, report = _run_rule_e_docs( + {self.TESTING: '{name="tx.process" && span.`xrpl.tx.hash`="AB"}'} + ) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + self.assertEqual(report.violations[0][1], f"{self.TESTING}:1") + self.assertEqual(report.violations[0][3], "underscore, not dotted") + + def test_testing_guide_checked_when_every_other_doc_absent(self): + # Per-file presence gating: TESTING.md alone must not skip the rule. + tokens, report = _run_rule_e_docs({self.TESTING: "`xrpl.peer.version`"}) + self.assertEqual(tokens, ["xrpl.peer.version"]) + self.assertEqual(report.skips, []) + self.assertEqual(report.violations[0][1], f"{self.TESTING}:1") + # A violating run emits the violation, not an "ok" line. + self.assertEqual(report.checked, []) + + def test_testing_guide_alone_reports_one_clean_file(self): + tokens, report = _run_rule_e_docs({self.TESTING: "`tx_hash`"}) + self.assertEqual(tokens, []) + self.assertTrue(any("E:" in c and "1 file(s)" in c for c in report.checked)) + + def test_clean_testing_guide_passes_with_bare_keys(self): + # The real docker/telemetry/TESTING.md shape: bare/underscore attr keys + # plus dotted SPAN names, which must not be flagged. + tokens, report = _run_rule_e_docs( + {self.TESTING: '{name="tx.process" && span.tx_hash!=""} `ledger_seq`'} + ) + self.assertEqual(tokens, []) + self.assertEqual(report.violations, []) + + def test_exemptions_apply_in_testing_guide_too(self): + tokens, _ = _run_rule_e_docs( + {self.TESTING: "`xrpl.network.type` `xrpl.work.item`"} + ) + self.assertEqual(tokens, []) + + def test_flags_dotted_attr_in_glossary(self): + tokens, report = _run_rule_e_docs({self.GLOSSARY: "| `xrpl.tx.hash` | h |"}) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + self.assertEqual(report.violations[0][1], f"{self.GLOSSARY}:1") + + def test_flags_dotted_attr_in_nested_build_guide(self): + # One directory below the root: proves discovery recurses rather than + # globbing only the root's own children. + tokens, report = _run_rule_e_docs({self.BUILD_GUIDE: "`xrpl.consensus.mode`"}) + self.assertEqual(tokens, ["xrpl.consensus.mode"]) + self.assertEqual(report.violations[0][1], f"{self.BUILD_GUIDE}:1") + + def test_location_carries_the_offending_line_number(self): + tokens, report = _run_rule_e_docs( + {self.GLOSSARY: "clean\nstill clean\n| `xrpl.tx.hash` |"} + ) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + self.assertEqual(report.violations[0][1], f"{self.GLOSSARY}:3") + + def test_flags_across_all_four_docs(self): + tokens, report = _run_rule_e_docs( + { + self.RUNBOOK: "`xrpl.tx.hash`", + self.GLOSSARY: "`xrpl.peer.id`", + self.BUILD_GUIDE: "`xrpl.ledger.seq`", + self.TESTING: "`xrpl.rpc.command`", + } + ) + self.assertEqual( + tokens, + [ + "xrpl.ledger.seq", + "xrpl.peer.id", + "xrpl.rpc.command", + "xrpl.tx.hash", + ], + ) + self.assertEqual( + sorted(v[1] for v in report.violations), + sorted( + [ + f"{self.RUNBOOK}:1", + f"{self.GLOSSARY}:1", + f"{self.BUILD_GUIDE}:1", + f"{self.TESTING}:1", + ] + ), + ) + + def test_other_doc_checked_when_runbook_absent(self): + # Presence gating is per discovered file: a branch without the runbook + # still gets the docs it does carry checked, instead of the rule skipping. + tokens, report = _run_rule_e_docs({self.GLOSSARY: "`xrpl.tx.hash`"}) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + self.assertEqual(report.skips, []) + + def test_absent_doc_is_not_an_error(self): + tokens, report = _run_rule_e_docs({self.RUNBOOK: "`tx_hash`"}) + self.assertEqual(tokens, []) + self.assertTrue(any("1 file(s)" in c for c in report.checked)) + + def test_clean_doc_set_reports_file_count(self): + tokens, report = _run_rule_e_docs( + { + self.RUNBOOK: "`tx_hash` `consensus.round`", + self.GLOSSARY: "`xrpl.network.id` `ledger_seq`", + self.BUILD_GUIDE: "`service.name` `peer_id`", + self.TESTING: "`rpc_command` `tx.process`", + } + ) + self.assertEqual(tokens, []) + self.assertEqual(report.violations, []) + self.assertTrue(any("E:" in c and "4 file(s)" in c for c in report.checked)) + + def test_skips_when_no_doc_present(self): + tokens, report = _run_rule_e_docs({}) + self.assertEqual(tokens, []) + self.assertEqual(report.checked, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + + def test_exemptions_apply_in_every_discovered_doc(self): + # The L1 resource attrs and perf-iac dotted identities must stay exempt + # in every discovered doc, exactly as in the runbook. + tokens, _ = _run_rule_e_docs( + { + self.GLOSSARY: "`xrpl.network.id` `xrpl.network.type`", + self.BUILD_GUIDE: "`xrpl.work.item` `xrpl.branch` `xrpl.node.role`", + } + ) + self.assertEqual(tokens, []) + + +def _marker(*keys: str) -> str: + """Build an allow-dotted marker naming `keys` (no key = the bare form).""" + body = ": " + ", ".join(keys) if keys else "" + return f"" + + +def _marker_warnings(report): + """The marker-related warnings only. + + A synthetic doc set that does not create the runbook leaves that required + anchor absent, which is warned about in its own right (see + RuleERequiredDocPresence). Filtering those out keeps the marker tests + asserting the marker's own behaviour.""" + return [tuple(w) for w in report.warnings if w[2] != "absent"] + + +class RuleEAllowDottedMarkerParsing(unittest.TestCase): + """`rule_e_allowed_keys` — the marker parser. The key list is what bounds the + exemption, so its parsing (and the marker COUNT, which distinguishes "no + marker" from "a marker that names nothing") is asserted directly.""" + + def test_no_marker_yields_no_keys_and_no_marker_count(self): + self.assertEqual( + chk.rule_e_allowed_keys("plain `xrpl.tx.hash` line"), (set(), 0) + ) + + def test_bare_marker_counts_but_names_no_key(self): + self.assertEqual(chk.rule_e_allowed_keys(_marker()), (set(), 1)) + + def test_single_key(self): + self.assertEqual( + chk.rule_e_allowed_keys(_marker("xrpl.tx.hash")), ({"xrpl.tx.hash"}, 1) + ) + + def test_comma_separated_keys(self): + self.assertEqual( + chk.rule_e_allowed_keys(_marker("xrpl.tx.hash", "xrpl.node.server_state")), + ({"xrpl.tx.hash", "xrpl.node.server_state"}, 1), + ) + + def test_space_separated_and_backticked_keys(self): + self.assertEqual( + chk.rule_e_allowed_keys( + "" + ), + ({"xrpl.tx.hash", "xrpl.peer.id"}, 1), + ) + + def test_no_surrounding_whitespace(self): + self.assertEqual( + chk.rule_e_allowed_keys(""), + ({"xrpl.tx.hash"}, 1), + ) + + def test_two_markers_on_one_line_union(self): + line = _marker("xrpl.tx.hash") + " text " + _marker("xrpl.peer.id") + self.assertEqual( + chk.rule_e_allowed_keys(line), ({"xrpl.tx.hash", "xrpl.peer.id"}, 2) + ) + + def test_prose_around_marker_is_not_absorbed_as_a_key(self): + line = "use `tx_hash`, not `xrpl.tx.hash`. " + _marker("xrpl.tx.hash") + " ok" + self.assertEqual(chk.rule_e_allowed_keys(line), ({"xrpl.tx.hash"}, 1)) + + def test_trailing_comma_does_not_yield_empty_key(self): + self.assertEqual( + chk.rule_e_allowed_keys(""), + ({"xrpl.tx.hash"}, 1), + ) + + +class RuleEAllowDottedMarkerEnforcement(unittest.TestCase): + """The marker exempts ONLY the keys it names, only on its own line. A + blanket line opt-out would let a genuine violation appended to any marked + line ride in silently, which is exactly what these tests forbid.""" + + REFERENCE = GLOSSARY_DOC + + def test_named_key_is_exempt(self): + tokens, report = _run_rule_e_docs( + { + self.REFERENCE: "use `tx_hash`, not `xrpl.tx.hash`. " + + _marker("xrpl.tx.hash") + } + ) + self.assertEqual(tokens, []) + self.assertEqual(report.violations, []) + self.assertEqual(_marker_warnings(report), []) + + def test_unlisted_key_on_marked_line_still_fails(self): + # The hole this closes: one marker must not cover a second dotted key + # someone appends to the line later. + tokens, report = _run_rule_e_docs( + { + self.REFERENCE: "not `xrpl.tx.hash` and also `xrpl.peer.id` " + + _marker("xrpl.tx.hash") + } + ) + self.assertEqual(tokens, ["xrpl.peer.id"]) + self.assertEqual(report.violations[0][0], "E") + self.assertEqual(report.violations[0][1], f"{self.REFERENCE}:1") + self.assertEqual(report.violations[0][3], "underscore, not dotted") + + def test_bare_marker_exempts_nothing_and_warns(self): + tokens, report = _run_rule_e_docs( + {self.REFERENCE: "not `xrpl.tx.hash`. " + _marker()} + ) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + self.assertEqual(report.violations[0][1], f"{self.REFERENCE}:1") + warnings = _marker_warnings(report) + self.assertEqual( + [w[:3] for w in warnings], [("E", f"{self.REFERENCE}:1", "allow-dotted")] + ) + self.assertIn(chk.RULE_E_MARKER_SYNTAX, warnings[0][3]) + + def test_marker_does_not_leak_to_adjacent_lines(self): + tokens, report = _run_rule_e_docs( + { + self.REFERENCE: "| `xrpl.tx.hash` | before |\n" + + "not `xrpl.tx.hash`. " + + _marker("xrpl.tx.hash") + + "\n| `xrpl.tx.hash` | after |" + } + ) + self.assertEqual(tokens, ["xrpl.tx.hash", "xrpl.tx.hash"]) + self.assertEqual( + sorted(v[1] for v in report.violations), + [f"{self.REFERENCE}:1", f"{self.REFERENCE}:3"], + ) + + def test_all_named_keys_exempt_on_a_table_row(self): + # The real shape in the plan docs: a "was -> is now" row naming two old + # dotted keys, with both listed in the marker. + tokens, _ = _run_rule_e_docs( + { + self.REFERENCE: "| `xrpl.validation.full`, `xrpl.peer.validation.full`" + " | one bare `full_validation` | " + + _marker("xrpl.validation.full", "xrpl.peer.validation.full") + } + ) + self.assertEqual(tokens, []) + + def test_partially_listed_table_row_fails_on_the_unlisted_key(self): + tokens, report = _run_rule_e_docs( + { + self.REFERENCE: "| `xrpl.validation.full`, `xrpl.peer.validation.full`" + " | one bare `full_validation` | " + _marker("xrpl.validation.full") + } + ) + self.assertEqual(tokens, ["xrpl.peer.validation.full"]) + self.assertEqual(report.violations[0][1], f"{self.REFERENCE}:1") + + def test_key_prefix_does_not_exempt_a_longer_key(self): + # Exemption is an exact token match, not a prefix match. + tokens, _ = _run_rule_e_docs( + {self.REFERENCE: "`xrpl.tx.hash` " + _marker("xrpl.tx")} + ) + self.assertEqual(tokens, ["xrpl.tx.hash"]) + + def test_dotted_prefix_token_can_be_listed_verbatim(self): + # The docs grep for prefixes such as `xrpl.node.` (trailing dot); the + # marker must accept that exact token. + tokens, report = _run_rule_e_docs( + { + self.REFERENCE: "a grep for `xrpl.node.` and `xrpl.peer.` returns " + "nothing " + _marker("xrpl.node.", "xrpl.peer.") + } + ) + self.assertEqual(tokens, []) + self.assertEqual(_marker_warnings(report), []) + + def test_stale_key_warns_but_does_not_fail(self): + tokens, report = _run_rule_e_docs( + {self.REFERENCE: "all clean now. " + _marker("xrpl.tx.hash")} + ) + self.assertEqual(tokens, []) + self.assertEqual(report.violations, []) + self.assertEqual( + _marker_warnings(report), + [ + ( + "E", + f"{self.REFERENCE}:1", + "xrpl.tx.hash", + "allow-dotted key not on this line", + ) + ], + ) + + def test_marker_naming_an_l1_key_is_not_reported_stale(self): + # `xrpl.network.id` is exempt via L1, so a marker naming it is redundant + # but not stale — the key IS on the line. + tokens, report = _run_rule_e_docs( + {self.REFERENCE: "`xrpl.network.id` " + _marker("xrpl.network.id")} + ) + self.assertEqual(tokens, []) + self.assertEqual(_marker_warnings(report), []) + + def test_marker_works_in_every_doc_of_the_set(self): + docs = { + rel: "not `xrpl.tx.hash`. " + _marker("xrpl.tx.hash") for rel in ALL_DOCS + } + tokens, report = _run_rule_e_docs(docs) + self.assertEqual(tokens, []) + self.assertEqual(report.violations, []) + + +class RuleEDocDiscovery(unittest.TestCase): + """`rule_e_docs` — the discovery that replaced a hardcoded doc list. What has + to hold: it finds the real docs in this repo (a rule scanning an empty set + passes without checking anything), it recurses into subdirectories, it never + checks a file twice, and every path it is configured with lies inside a tree + that actually ships.""" + + def test_discovery_finds_more_than_one_doc_in_this_repo(self): + # The vacuous-pass guard, measured against the real tree: one file (or + # none) would mean discovery had collapsed and Rule E was reporting green + # over almost nothing. + found = chk.rule_e_docs(chk.repo_root()) + self.assertGreater(len(found), 1, found) + + def test_discovery_returns_no_duplicates(self): + # Roots may overlap (a root and a subdirectory of it); a file must still + # be checked once, so line numbers are not reported twice. + found = chk.rule_e_docs(chk.repo_root()) + self.assertEqual(len(found), len(set(found))) + + def test_discovery_finds_the_known_telemetry_docs(self): + # Named here as an expectation of DISCOVERY, not as the checker's config: + # if any of these is renamed, discovery still covers it under its new + # name and only this assertion needs updating. + found = set(chk.rule_e_docs(chk.repo_root())) + for rel in (RUNBOOK_DOC, GLOSSARY_DOC, TESTING_DOC): + self.assertIn(Path(rel), found) + + def test_discovery_recurses_into_subdirectories(self): + d = Path(tempfile.mkdtemp()) + try: + nested = d / "docs" / "build" / "deep" + nested.mkdir(parents=True) + (nested / "telemetry.md").write_text("`tx_hash`") + self.assertEqual( + chk.rule_e_docs(d), [Path("docs") / "build" / "deep" / "telemetry.md"] + ) + finally: + shutil.rmtree(d) + + def test_discovery_ignores_non_markdown(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "notes.txt").write_text("`xrpl.tx.hash`") + (d / "docs" / "runbook.md").write_text("`tx_hash`") + self.assertEqual(chk.rule_e_docs(d), [Path("docs") / "runbook.md"]) + finally: + shutil.rmtree(d) + + def test_discovery_tolerates_an_absent_root(self): + # Presence gating applies to the roots too: a tree carrying only one of + # them must not raise, and must still discover that one. + d = Path(tempfile.mkdtemp()) + try: + (d / "docker" / "telemetry").mkdir(parents=True) + (d / "docker" / "telemetry" / "TESTING.md").write_text("`tx_hash`") + self.assertEqual( + chk.rule_e_docs(d), [Path("docker") / "telemetry" / "TESTING.md"] + ) + finally: + shutil.rmtree(d) + + def test_no_configured_path_lies_outside_the_committable_roots(self): + # The check may only be wired to trees that ship. A configured path + # outside them would make Rule E warn about a doc that can never appear + # and leave the matching CI path-trigger dead. + for rel in chk.RULE_E_DOC_ROOTS + chk.RULE_E_REQUIRED_DOCS: + self.assertTrue(_inside_any(rel, COMMITTABLE_ROOTS), rel) + + def test_configured_paths_exist_and_are_version_controlled(self): + # "Committable" is not just a naming claim: git must actually track the + # contents of every configured path in this repo. + root = chk.repo_root() + for rel in chk.RULE_E_DOC_ROOTS: + self.assertTrue((root / rel).is_dir(), rel) + self.assertTrue(_git_tracks(root, rel), rel) + for rel in chk.RULE_E_REQUIRED_DOCS: + self.assertTrue((root / rel).is_file(), rel) + self.assertTrue(_git_tracks(root, rel), rel) + + def test_every_required_doc_is_discoverable(self): + # An anchor outside every root could never be found, so the tripwire + # would fire on every run and stop meaning anything. + for rel in chk.RULE_E_REQUIRED_DOCS: + self.assertTrue(_inside_any(rel, chk.RULE_E_DOC_ROOTS), rel) + self.assertIn(rel, chk.rule_e_docs(chk.repo_root())) + + +class RuleERequiredDocPresence(unittest.TestCase): + """A RULE_E_REQUIRED_DOCS anchor that is not in the tree must be VISIBLE. + Discovery cannot go stale, but it can go quiet — a root emptied of telemetry + docs still yields a clean run — so the anchor's absence is warned about and + counted, while staying non-fatal so a partial branch still passes.""" + + RUNBOOK = RUNBOOK_DOC + + def test_every_required_doc_resolves_in_this_repo(self): + # If the runbook is renamed and RULE_E_REQUIRED_DOCS is not updated, the + # anchor warning would fire on every run; catch it here instead. + root = chk.repo_root() + self.assertEqual( + [ + str(rel) + for rel in chk.RULE_E_REQUIRED_DOCS + if not (root / rel).is_file() + ], + [], + ) + + def test_missing_anchor_is_warned_once(self): + # Discovery finds the glossary, so the rule runs — but the anchor it is + # required to find is gone, which must be said out loud. + tokens, report = _run_rule_e_docs({GLOSSARY_DOC: "`tx_hash`"}) + self.assertEqual(tokens, []) + self.assertEqual( + [tuple(w) for w in report.warnings], + [ + ( + "E", + self.RUNBOOK, + "absent", + "required Rule E doc not in tree (renamed?)", + ) + ], + ) + + def test_ok_line_names_the_absent_anchor_count(self): + tokens, report = _run_rule_e_docs({GLOSSARY_DOC: "`tx_hash`"}) + self.assertEqual(tokens, []) + self.assertTrue( + any( + "1 file(s) checked" in c + and f"1 of {len(chk.RULE_E_REQUIRED_DOCS)} required doc(s) absent" in c + for c in report.checked + ), + report.checked, + ) + + def test_full_doc_set_warns_nothing_and_reports_the_full_count(self): + docs = {rel: "`tx_hash`" for rel in ALL_DOCS} + tokens, report = _run_rule_e_docs(docs) + self.assertEqual(tokens, []) + self.assertEqual(report.warnings, []) + self.assertTrue( + any( + f"{len(ALL_DOCS)} file(s) checked" in c and "absent" not in c + for c in report.checked + ), + report.checked, + ) + + def test_anchor_present_warns_nothing_even_with_other_docs_absent(self): + # Only the ANCHOR is required. A tree carrying just the runbook is a + # legitimate partial branch, not a shrinking doc set. + tokens, report = _run_rule_e_docs({self.RUNBOOK: "`tx_hash`"}) + self.assertEqual(tokens, []) + self.assertEqual(report.warnings, []) + self.assertTrue( + any("1 file(s) checked" in c and "absent" not in c for c in report.checked), + report.checked, + ) + + def test_absent_anchor_is_warned_even_when_l1_is_empty(self): + # The anchor can be verified without an L1 key set, so the visibility + # signal must not depend on the (separately gated) key comparison. + tokens, report = _run_rule_e_docs({GLOSSARY_DOC: "`xrpl.tx.hash`"}, l1_keys=[]) + self.assertEqual(tokens, []) + self.assertEqual(len(report.warnings), len(chk.RULE_E_REQUIRED_DOCS)) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + + def test_no_anchor_warning_when_discovery_finds_nothing(self): + # Nothing to check at all is already reported by the SKIP line, which + # names the roots it searched, so an anchor warning would be noise. + tokens, report = _run_rule_e_docs({}) + self.assertEqual(tokens, []) + self.assertEqual(report.warnings, []) + roots = ", ".join(str(rel) for rel in chk.RULE_E_DOC_ROOTS) + self.assertEqual( + report.skips, + [f"SKIP: E — no doc-layer file present (no *.md under {roots})"], + ) + + def test_doc_outside_every_root_is_not_discovered(self): + # The counterpart of the roots being committable: markdown parked outside + # them is not part of the L5 layer, so a dotted key there cannot fail the + # build — and equally cannot make the rule look like it checked something. + tokens, report = _run_rule_e_docs({"elsewhere/notes.md": "`xrpl.tx.hash`"}) + self.assertEqual(tokens, []) + self.assertEqual(report.checked, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + + class DslParser(unittest.TestCase): """The makeStr/join/seg:: constexpr DSL resolver — the foundation of the L1 key set. Covers flat, nested, cross-file, alias, and multi-line forms.""" @@ -1064,7 +1657,7 @@ class RuleEReportTuple(unittest.TestCase): (d / "docs").mkdir() (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") report = chk.Report() - chk.run_rule_e_runbook(d, {"xrpl.network.id"}, report) + chk.run_rule_e_docs(d, {"xrpl.network.id"}, report) self.assertEqual(len(report.violations), 1) rule, _loc, token, expected = report.violations[0] self.assertEqual(rule, "E") @@ -1081,7 +1674,7 @@ class RuleEReportTuple(unittest.TestCase): "`tx_hash` `consensus.round`" ) report = chk.Report() - chk.run_rule_e_runbook(d, {"tx_hash"}, report) + chk.run_rule_e_docs(d, {"tx_hash"}, report) self.assertEqual(report.violations, []) self.assertTrue(any("E:" in c for c in report.checked)) finally: diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index d48d1fe30b..d119467c0f 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -77,6 +77,16 @@ jobs: .github/workflows/reusable-check-rename.yml .github/workflows/on-pr.yml + # The non-code layers the OTel naming check validates: the docs that + # publish attribute tables (Rule E) and the telemetry stack config — + # collector, Tempo, dashboards (Rules B, C, D). Without these paths a + # docs-only or dashboard-only pull request sets `go=false`, so the + # very layers those rules exist to police would never be checked. + # As with `README.md` below, matching one of these also switches on + # the rest of the workflow; there is a single `go` gate. + docs/** + docker/telemetry/** + # Keep the paths below in sync with those in `on-trigger.yml`. .github/actions/build-deps/** .github/actions/generate-version/** diff --git a/.github/workflows/reusable-check-otel-naming.yml b/.github/workflows/reusable-check-otel-naming.yml index a7af2da8cd..54cab30640 100644 --- a/.github/workflows/reusable-check-otel-naming.yml +++ b/.github/workflows/reusable-check-otel-naming.yml @@ -21,6 +21,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Test the OTel naming checker + # The checker's own unit tests, run before the check itself so a broken + # rule is reported as a broken rule rather than as a naming violation + # (or, worse, as a rule that silently stops flagging anything). + # stdlib `unittest` only: the repo installs no third-party test runner + # for CI, and the checker itself is deliberately dependency-free. + run: python -m unittest discover -s .github/scripts/otel-naming -p 'test_*.py' --verbose - name: Check OTel naming # The script is stdlib-only and reads only files already in the tree; # it enforces each rule only when the layer it needs is present, so it diff --git a/CMakeLists.txt b/CMakeLists.txt index fd3784b66c..86fa97dbde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,10 +141,20 @@ if(rocksdb) endif() # OpenTelemetry distributed tracing (optional). -# When ON, links against opentelemetry-cpp and defines XRPL_ENABLE_TELEMETRY -# so that SpanGuard factory methods produce real OTel spans. -# When OFF (default), all tracing code compiles to no-ops with zero overhead. -# Enable via: conan install -o telemetry=True, or cmake -Dtelemetry=ON. +# When ON, links against opentelemetry-cpp and defines XRPL_ENABLE_TELEMETRY so +# that SpanGuard factory methods produce real OTel spans. +# When OFF, all tracing code compiles to no-ops with zero overhead and +# opentelemetry-cpp is not needed at all. +# +# The value below is temporarily ON so that CI compiles the telemetry code +# paths while this feature is in review. OFF is the intended shipped default; +# flipping it back is tracked as a separate change. Do not rely on the current +# value - select it explicitly with cmake -Dtelemetry=ON|OFF or +# conan install -o telemetry=True|False. +# +# -DXRPL_ENABLE_TELEMETRY=OFF does not turn anything off: that name is only a +# compile definition added below, not a CMake option, so CMake just lists it as +# an unused variable at the end of configuration. option(telemetry "Enable OpenTelemetry tracing" ON) if(telemetry) find_package(opentelemetry-cpp CONFIG REQUIRED) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2642b8a377..c0587ac337 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -413,6 +413,29 @@ land in one pull request or several. Run it locally with: python .github/scripts/otel-naming/check_otel_naming.py ``` +### Naming a wrong form in prose (`otel-naming:allow-dotted`) + +The doc rule (E) flags any dotted `` `xrpl..` `` key in the +telemetry docs, because a reader copies those keys straight into a TraceQL or +PromQL query. A doc that _teaches_ the convention, or records a rename, has to be +able to name the wrong form as a counter-example. That mention is opted out with +a marker naming exactly the keys the line is allowed to mention: + +```markdown +Use `tx_hash`, not `xrpl.tx.hash`. + +``` + +- The marker applies to **its own line only**, and exempts **only the keys it + lists** (comma- and/or space-separated, backticks optional). A dotted key on a + marked line that the marker does not name still fails, so an exemption cannot + quietly widen when someone edits the line later. +- A marker with no key list exempts nothing and reports a warning; so does a + marker naming a key the line no longer mentions (a stale exemption). +- Never use it to keep a real attribute table dotted. If the doc publishes a key + an operator is meant to query, fix the key — the marker is for mentions, not + for published attributes. + See [.github/scripts/otel-naming/README.md](.github/scripts/otel-naming/README.md) for the full rule list. diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index ff71e44a80..cbe8151e69 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -125,54 +125,79 @@ path in Phase 1b through Phase 5. > **Status column.** This catalog is the design inventory; it is not a > statement of what currently emits. `Live` means the span is present in the -> implemented inventory ([09-data-collection-reference.md §1.1](./09-data-collection-reference.md#11-complete-span-inventory-37-spans)), +> implemented inventory ([09-data-collection-reference.md §1.1](./09-data-collection-reference.md#11-complete-span-inventory-41-spans)), > which is the authoritative list. `Renamed`/`Split` means the concept shipped > under a different name than planned here. **Not built** means no span is > emitted for it today. > +> **"Not built" is not one thing.** All 14 such entries fall into three cases, and the +> fourth column says which — filing them all as oversights would be wrong: +> +> - **Superseded by metrics or logs (7)** — a deliberate trade-off: the signal is already +> carried by a metric or by a log-derived panel, and a span would add per-event volume +> without adding information. `tx.relay`, `fee.escalate`, `validator.list.fetch`, +> `validator.manifest`, `shamap.sync`, `job.enqueue`, `job.execute`. +> - **Gap (6)** — nothing was decided; they were simply never instrumented. The four +> `peer.*` entries, plus `ledger.replay` and `ledger.delta` — and those last two are the +> sharpest, because they have **no metric substitute at all**. +> - **Deferred (1)** — scheduled work: `amendment.vote` (Phase 11). +> > The four `peer.*` entries are the peer-span coverage gap: only > `peer.proposal.receive` and `peer.validation.receive` exist, so protocol > message send/receive and connection lifecycle are untraced. See > [09 §6.4](./09-data-collection-reference.md#64-peer-span-coverage-gap-not-implemented). +> +> `tx.validate` did ship, but renamed and split three ways: the apply pipeline +> traces `tx.preflight` (stateless checks), `tx.preclaim` (ledger-state checks) +> and `tx.transactor` (application), each stamped with a `stage` attribute. +> Names come from `TxApplySpanNames.h:90,94,99`. The spans are created in two +> different files, not one: `tx.preflight` and `tx.preclaim` come from +> `applySteps.cpp` (`invokePreflight()` at `:211-212`, `invokePreclaim()` at +> `:258-261`, both via the shared `makeStageSpan()` helper at `:89-126`), while +> `tx.transactor` is created in `Transactor::operator()()` +> (`Transactor.cpp:1601-1605`). Query them with +> `name=~"tx\.(preflight|preclaim|transactor)"` — a **single** backslash; RE2 +> reads `\\.` as a literal backslash followed by any character, which matches +> nothing here — never `name="tx.validate"`. -| Span name | Description | Status | -| ------------------------------ | --------------------------------------- | ------------------------------------------------ | -| `tx.receive` | Transaction received from network | Live | -| `tx.validate` | Transaction signature/format validation | **Not built** | -| `tx.process` | Full transaction processing | Live | -| `tx.relay` | Transaction relay to peers | **Not built** | -| `tx.apply` | Apply transaction to ledger | Live | -| `consensus.round` | Complete consensus round | Live | -| `consensus.phase.open` | Open phase - collecting transactions | Live | -| `consensus.phase.establish` | Establish phase - reaching agreement | Renamed `consensus.establish` | -| `consensus.phase.accept` | Accept phase - applying consensus | Renamed `consensus.accept` | -| `consensus.proposal.receive` | Receive peer proposal | Live | -| `consensus.proposal.send` | Send our proposal | Live | -| `consensus.validation.receive` | Receive peer validation | Live | -| `consensus.validation.send` | Send our validation | Live | -| `rpc.request` | HTTP/WebSocket request handling | Split into `rpc.http_request` / `rpc.ws_message` | -| `rpc.command.*` | Specific RPC command (dynamic) | Live | -| `peer.connect` | Peer connection establishment | **Not built** | -| `peer.disconnect` | Peer disconnection | **Not built** | -| `peer.message.send` | Send protocol message | **Not built** | -| `peer.message.receive` | Receive protocol message | **Not built** | -| `ledger.acquire` | Ledger acquisition from network | Live | -| `ledger.build` | Build new ledger | Live | -| `ledger.validate` | Ledger validation | Live | -| `ledger.close` | Close ledger | Renamed `consensus.ledger_close` | -| `ledger.replay` | Ledger replay executed | **Not built** | -| `ledger.delta` | Delta-based ledger acquired | **Not built** | -| `pathfind.request` | Path request initiated | Live | -| `pathfind.compute` | Path computation executed | Live | -| `txq.enqueue` | Transaction queued | Live | -| `txq.apply` | Queued transaction applied | Renamed `txq.apply_direct` / `txq.accept_tx` | -| `fee.escalate` | Fee escalation triggered | **Not built** | -| `validator.list.fetch` | UNL list fetched | **Not built** | -| `validator.manifest` | Manifest update processed | **Not built** | -| `amendment.vote` | Amendment voting executed | **Not built** | -| `shamap.sync` | State tree synchronization | **Not built** | -| `job.enqueue` | Job added to queue | **Not built** | -| `job.execute` | Job execution | **Not built** | +| Span name | Description | Status | Why not built / where the signal lives instead | +| ------------------------------ | --------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tx.receive` | Transaction received from network | Live | — | +| `tx.validate` | Transaction signature/format validation | Renamed + split → `tx.preflight`, `tx.preclaim`, `tx.transactor` | — | +| `tx.process` | Full transaction processing | Live | — | +| `tx.relay` | Transaction relay to peers | **Not built** | **Superseded by metrics.** Relay volume is carried by the overlay traffic counters (`total_bytes_in/out`, `total_messages_in/out`, per-`TrafficCount` category). Relay is also per-peer fan-out, so one span per relay multiplies by peer count for data the counters already aggregate. | +| `tx.apply` | Apply transaction to ledger | Live | — | +| `consensus.round` | Complete consensus round | Live | — | +| `consensus.phase.open` | Open phase - collecting transactions | Live | — | +| `consensus.phase.establish` | Establish phase - reaching agreement | Renamed `consensus.establish` | — | +| `consensus.phase.accept` | Accept phase - applying consensus | Renamed `consensus.accept` | — | +| `consensus.proposal.receive` | Receive peer proposal | Live | — | +| `consensus.proposal.send` | Send our proposal | Live | — | +| `consensus.validation.receive` | Receive peer validation | Live | — | +| `consensus.validation.send` | Send our validation | Live | — | +| `rpc.request` | HTTP/WebSocket request handling | Split into `rpc.http_request` / `rpc.ws_message` | — | +| `rpc.command.*` | Specific RPC command (dynamic) | Live | — | +| `peer.connect` | Peer connection establishment | **Not built** | **Gap, scoped as its own change** — see [09 §6.4](./09-data-collection-reference.md#64-peer-span-coverage-gap-not-implemented). Adding these changes the 41-family span count and the 40 catalogued in `expected_spans.json`. | +| `peer.disconnect` | Peer disconnection | **Not built** | **Gap.** Partially observable: the aggregate count via the `Overlay.Peer_Disconnects` insight gauge and resource-charge drops via `server_info{metric="peer_disconnects_resources"}`, but not per-reason. Disconnect reasons are only recoverable from `debug.log` (the `log-derived-insights` dashboard). | +| `peer.message.send` | Send protocol message | **Not built** | **Gap.** Of the 13 protocol message families only `mtGET_OBJECTS` has native instrumentation (`getobject_*`); byte/message volume is aggregated by `TrafficCount` category, not traced per message. | +| `peer.message.receive` | Receive protocol message | **Not built** | **Gap.** Same as `peer.message.send`. | +| `ledger.acquire` | Ledger acquisition from network | Live | — | +| `ledger.build` | Build new ledger | Live | — | +| `ledger.validate` | Ledger validation | Live | — | +| `ledger.close` | Close ledger | Renamed `consensus.ledger_close` | — | +| `ledger.replay` | Ledger replay executed | **Not built** | **Gap, no substitute.** `LedgerReplayer.cpp` and `LedgerReplayTask.cpp` contain zero `SpanGuard` uses and no metric covers the replay path. A real hole, not a trade-off. | +| `ledger.delta` | Delta-based ledger acquired | **Not built** | **Gap, no substitute.** `LedgerDeltaAcquire.cpp` contains zero `SpanGuard` uses. The `acquire_*` stats cover whole-ledger acquisition, not the delta path. | +| `pathfind.request` | Path request initiated | Live | — | +| `pathfind.compute` | Path computation executed | Live | — | +| `txq.enqueue` | Transaction queued | Live | — | +| `txq.apply` | Queued transaction applied | Renamed `txq.apply_direct` / `txq.accept_tx` | — | +| `fee.escalate` | Fee escalation triggered | **Not built** | **Superseded by metrics + existing spans.** Escalation state is `txq_metrics{metric=…}` and `load_factor_metrics{metric=…}`; the queueing path that triggers it is already traced by the six `txq.*` spans. An event span would restate a gauge. | +| `validator.list.fetch` | UNL list fetched | **Not built** | **Superseded by metrics.** `validator_health{metric="unl_expiry_days"}`, `{metric="unl_blocked"}` and `{metric="validation_quorum"}` carry the outcome. A fetch span would fire on a slow timer and tell an operator nothing the gauges do not. | +| `validator.manifest` | Manifest update processed | **Not built** | **Superseded by logs.** Per-master-key manifest dispositions are on the `log-derived-insights` dashboard (`ManifestCache` partition, requires `log_level ManifestCache debug`). | +| `amendment.vote` | Amendment voting executed | **Not built** | **Deferred to Phase 11.** `validator_health{metric="amendment_blocked"}` covers the blocked state in the meantime. | +| `shamap.sync` | State tree synchronization | **Not built** | **Superseded by metrics.** Covered by the nine `acquire_*` stats, `nodestore_state{metric=…}` and the five `getobject_*` families. Per-node-fetch spans would be prohibitive volume. | +| `job.enqueue` | Job added to queue | **Not built** | **Superseded by metrics.** `job_queued_total` and `job_queued_us{job_type}` plus the 105 per-job-type `jobq_*` gauges. A span per enqueue is one span per unit of daemon work, for latency the histogram already records exactly. | +| `job.execute` | Job execution | **Not built** | **Superseded by metrics.** `job_started_total`, `job_finished_total`, `job_running_us{job_type}`. Same volume argument as `job.enqueue`. | ### 2.3.3 Attribute Naming Conventions @@ -227,24 +252,36 @@ Resource attributes identify the process and are set once at startup. They use the standard OpenTelemetry semantic conventions plus custom dotted `xrpl.*` keys (the dotted form is reserved for resource scope per §2.3.3). -| Key | Type / value | Description | -| --------------------- | ------------------------------------------------------- | ------------------------------ | -| `service.name` | `"xrpld"` | Standard `SERVICE_NAME` | -| `service.version` | `build_info::getVersionString()` | Standard `SERVICE_VERSION` | -| `service.instance.id` | node public key (base58) | Standard `SERVICE_INSTANCE_ID` | -| `xrpl.network.id` | network id (e.g. 0 for mainnet) | Network identifier | -| `xrpl.network.type` | `"mainnet"` \| `"testnet"` \| `"devnet"` \| `"unknown"` | Network kind | -| `xrpl.node.type` | `"validator"` \| `"stock"` \| `"reporting"` | Node role | -| `xrpl.node.cluster` | cluster name | Cluster name, if clustered | +Five are set, by `Telemetry.cpp:380-387` (tracer resource) and the matching +block in `initMetrics()` (metrics resource); the custom key constants are +`SpanNames.h:117-118`. + +| Key | Type / value | Description | Status | +| --------------------- | -------------------------------------------------------------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `service.name` | `"xrpld"` | Standard `SERVICE_NAME` | Set | +| `service.version` | `build_info::getVersionString()` | Standard `SERVICE_VERSION` | Set | +| `service.instance.id` | node public key (base58), or `[telemetry] service_instance_id` | Standard `SERVICE_INSTANCE_ID` | Set — but the node-key fallback reaches traces only; see [05 §5.1.1](./05-configuration-reference.md) | +| `xrpl.network.id` | network id (e.g. 0 for mainnet) | Network identifier | Set | +| `xrpl.network.type` | `"mainnet"` \| `"testnet"` \| `"devnet"` \| `"unknown"` | Network kind | Set | +| `xrpl.node.type` | `"validator"` \| `"stock"` \| `"reporting"` | Node role | **Not implemented** — no constant, no set-site. Node role is therefore not queryable from a trace. (Dashboards do offer an `$xrpl_node_role` filter, but it matches a Prometheus label stamped by the external perf-iac deployment — `check_otel_naming.py:872` — not by anything in this repo) | +| `xrpl.node.cluster` | cluster name | Cluster name, if clustered | **Not implemented** — no constant, no set-site | + +The collector adds two more resource attributes of its own (`deployment.environment` +and, when the node did not stamp it, `xrpl.network.type`) via the +`resource/tier` processor, and deletes the SDK-injected `telemetry.sdk.*` trio +via `resource/stripsdk`. See [05 §5.5.1](./05-configuration-reference.md). ### 2.4.2 Span Attributes by Category > Span attribute keys use the underscore form from §2.3.3 (shared/qualified > keys are `_`; per-span unique keys are bare). The dotted form > is reserved for the resource attributes in §2.4.1 above. This catalog lists -> the planned attribute set by category; the exact emitted key for each -> implemented span is defined by the `*SpanNames.h` constants, which are the -> single source of truth where the two differ. +> the planned attribute set by category; the exact emitted key **and its type** +> for each implemented span is defined by the `*SpanNames.h` constants and their +> set-sites, which win where the two differ. The types in the tables below are +> the ones originally planned and are **not** all what shipped — `peer_id` is +> the notable case (planned as a base58 string, shipped as an int64). §2.4.3 +> is the implemented view. #### Transaction Attributes @@ -303,15 +340,15 @@ Establish-phase gap fill and cross-node correlation attributes (Phase 4a): #### Peer & Message Attributes -| Key | Type | Description | -| -------------------- | ------- | -------------------------- | -| `peer_id` | string | Peer public key (base58) | -| `peer_address` | string | IP:port | -| `peer_latency_ms` | float64 | Measured latency | -| `peer_cluster` | string | Cluster name if clustered | -| `message_type` | string | Protocol message type name | -| `message_size_bytes` | int64 | Message size | -| `message_compressed` | bool | Whether compressed | +| Key | Type | Description | +| -------------------- | ------- | ------------------------------------------------------------------------- | +| `peer_id` | string | Peer public key (base58) — **planned only; shipped as int64, see §2.4.3** | +| `peer_address` | string | IP:port | +| `peer_latency_ms` | float64 | Measured latency | +| `peer_cluster` | string | Cluster name if clustered | +| `message_type` | string | Protocol message type name | +| `message_size_bytes` | int64 | Message size | +| `message_compressed` | bool | Whether compressed | #### Ledger & Job Attributes @@ -373,22 +410,72 @@ Establish-phase gap fill and cross-node correlation attributes (Phase 4a): ### 2.4.3 Data Collection Summary -The following table summarizes what data is collected by category: +§2.4.2 above is the _planned_ catalogue; this table is the **implemented** one. +Its left column lists the keys of the `attr` namespaces of the `*SpanNames.h` +headers; every key shown has at least one live `attr::` set-site in +non-test code. The right column lists keys this document once claimed were +collected but which have no constant and no set-site at all. -| Category | Attributes Collected | Purpose | -| --------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------------- | -| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | -| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers`, `round_time_ms` | Analyze consensus timing | -| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | -| **Peer** | `peer_id` (public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | -| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | -| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | -| **PathFinding** | `pathfind_fast`, `pathfind_search_level`, `pathfind_num_paths`, `pathfind_ledger_index`, `pathfind_num_requests` | Payment path analysis | -| **TxQ** | `txq_queue_depth`, `txq_fee_level`, `txq_eviction_reason` | Queue depth and fee tracking | -| **Fee** | `fee_load_factor`, `fee_escalation_level` | Fee escalation monitoring | -| **Validator** | `validator_list_size`, `validator_list_age_sec` | UNL health monitoring | -| **Amendment** | `amendment_name`, `amendment_status` | Protocol upgrade tracking | -| **SHAMap** | `shamap_type`, `shamap_missing_nodes`, `shamap_duration_ms` | State tree sync performance | +**This table is a category-level roll-up, not the authority.** The +authoritative per-span breakdown — which span carries which attribute — is +[09-data-collection-reference.md §1.2](./09-data-collection-reference.md#12-complete-attribute-inventory-bareunderscore-keys), +and the exact key _spelling_ is owned by the `*SpanNames.h` constants. Where +this table disagrees with either, they win. + +> **Known divergence (documented, not resolved here).** 09 §1.2's Consensus +> subsection lists 47 keys; `include/xrpl/consensus/ConsensusSpanNames.h` +> defines 54 in its `attr` namespace (48 own `makeStr` constants plus 6 +> `using` re-exports of the shared keys in `SpanNames.h`), all 54 with +> set-sites. Five of the difference — `open_duration_ms`, +> `peer_positions_at_close`, `position_hash_prefix`, `prev_ledger_prefix`, +> `disputes_resolved_count` — are emitted but absent from 09 §1.2's consensus +> table; the other two, `proposal_trusted` and `validation_trusted`, are +> documented in 09 §1.2's Peer subsection instead (they are shared keys set on +> both the `peer.*` and the `consensus.*` receive spans — `PeerImp.cpp:1953` +> and `:2027` for the proposal pair, `:2591` and `:2635` for the validation +> pair). Fixing 09 is tracked separately; the Consensus row below lists all 54. + +| Category | Attributes emitted (from `*SpanNames.h`) | Named here but NOT emitted | Purpose | +| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| **Transaction** | `tx_hash`, `tx_type`, `ter_result`, `fee`, `sequence`, `current_ledger_seq`, `current_ledger_hash`, `local`, `path`, `suppressed`, `tx_status`, `peer_version`, `peer_id`, `stage`, `applied` | `tx_result` (renamed → `ter_result`), `tx_fee` (→ `fee`), `ledger_index` (→ `current_ledger_seq`), `relay_count`. **`ledger_seq` is not a `tx.*` key**: no `tx.*` span sets it — the receive and apply-stage spans stamp `current_ledger_seq` (`NetworkOPs.cpp:1422`, `PeerImp.cpp:1337`, `Transactor.cpp:1613`, `applySteps.cpp:115`) and, where a view exists, `current_ledger_hash` (`Transactor.cpp:1615`, `applySteps.cpp:121`) | Trace transaction lifecycle | +| **Consensus** | All 54 keys in `ConsensusSpanNames.h`'s `attr` namespace (48 own constants + 6 `using` re-exports), each with a set-site: `consensus_ledger_id`, `consensus_round`, `consensus_round_id`, `consensus_phase`, `consensus_mode`, `consensus_state`, `consensus_result`, `consensus_stalled`, `proposers`, `proposers_finished`, `previous_proposers`, `previous_ledger_seq`, `previous_round_time_ms`, `round_time_ms`, `open_duration_ms`, `quorum`, `proposing`, `is_bow_out`, `trace_strategy`, `converge_percent`, `establish_count`, `tx_count`, `tx_count_open`, `tx_id`, `disputes_count`, `disputes_resolved_count`, `dispute_our_vote`, `dispute_yays`, `dispute_nays`, `agree_count`, `disagree_count`, `threshold_percent`, `avalanche_threshold`, `close_time_threshold`, `have_close_time_consensus`, `close_time_resolution_ms`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `parent_close_time`, `peer_positions_at_close`, `prev_ledger_prefix`, `position_hash_prefix`, `mode_old`, `mode_new`, `validation_sign_time`, `proposal_trusted`, `validation_trusted`; re-exported shared keys `ledger_seq`, `ledger_hash`, `full_validation`, `close_time`, `close_time_correct`, `close_resolution_ms` | — | Analyze consensus timing | +| **RPC** | `command`, `version`, `rpc_role`, `rpc_status`, `request_payload_size`, `is_batch`, `batch_size`, `load_type` | `duration_ms` (span duration is a TraceQL intrinsic — query `duration`), `params` | Monitor RPC performance | +| **Peer** | `peer_id` (**int64**, the process-local `Peer::id_` slot number — not a key of any kind; also set on `tx.receive`), `proposal_trusted`, `validation_trusted`, `ledger_hash`, `full_validation`. (`peer_version` is **not** a peer-span key: the constant lives in `TxSpanNames.h:79` and its only set-site is `PeerImp.cpp:1342` on the `tx.receive` span — see the Transaction row) | `peer_address`, `peer_latency_ms`, `peer_cluster`, `message_type`, `message_size_bytes`, `message_compressed` — the peer-span coverage gap (§2.3.2) | Network topology analysis | +| **Ledger** | `ledger_seq`, `tx_count`, `tx_failed`, `validations`, `acquire_reason`, `timeouts`, `peer_count`, `outcome`, `close_time`, `close_time_correct`, `close_resolution_ms` | `ledger_index` (→ `ledger_seq`), `ledger_tx_count` (→ `tx_count`). `ledger_hash` is a live key, but **no `ledger.*` span sets it** — only `consensus.validation.send` (`RCLConsensus.cpp:977`; that span is the one returned by `createValidationSpan()`, which names `cs::validationSend` at `RCLConsensus.cpp:1365,1373`) and `peer.validation.receive` (`PeerImp.cpp:2573`) do. The `LedgerSpanNames.h:41` `using` alias has zero uses. `consensus.ledger_close` sets **no** hash: its four attributes are `ledger_seq`, `consensus_mode`, `tx_count_open` and `close_time_resolution_ms` (`RCLConsensus.cpp:354-361`) | Ledger progression tracking | +| **gRPC** | `method`, `grpc_role`, `grpc_status` | — | gRPC surface monitoring | +| **Job** | — (no job spans exist) | `job_type`, `job_queue_ms`, `job_worker`. JobQueue is observed via **metrics**, not spans — but by **two disjoint families**, and only one of them has a `job_type` label. See the note below the table | JobQueue performance | +| **PathFinding** | `pathfind_fast`, `pathfind_search_level`, `pathfind_num_paths`, `pathfind_ledger_index`, `pathfind_num_requests`, `pathfind_num_source_assets`, `pathfind_dest_currency`, `pathfind_source_account` (hashed), `pathfind_dest_account` (hashed) | `pathfind_source_currency`, `pathfind_path_count`, `pathfind_cache_hit` | Payment path analysis | +| **TxQ** | `txq_status`, `fee_level_paid`, `required_fee_level`, `queue_size`, `ledger_changed`, `expired_count`, `ter_code`, `retries_remaining`, `num_cleared`, `tx_type`, plus the re-exported shared keys `tx_hash`, `ledger_seq`, `current_ledger_seq`, `current_ledger_hash` | `txq_queue_depth` (→ `queue_size`), `txq_fee_level` (→ `fee_level_paid`), `txq_eviction_reason` | Queue depth and fee tracking | +| **Fee** | — (no `fee.escalate` span, §2.3.2) | `fee_load_factor`, `fee_escalation_level`. Fee escalation is dashboarded from metrics (`fee-market`), not spans | Fee escalation monitoring | +| **Validator** | — (no `validator.*` span, §2.3.2) | `validator_list_size`, `validator_list_age_sec`. UNL health is dashboarded from metrics (`validator-health`) | UNL health monitoring | +| **Amendment** | — (no `amendment.vote` span, §2.3.2) | `amendment_name`, `amendment_status` | Protocol upgrade tracking | +| **SHAMap** | — (no `shamap.sync` span, §2.3.2) | `shamap_type`, `shamap_missing_nodes`, `shamap_duration_ms` | State tree sync performance | + +The right-hand column is the honest gap list: every key in it appears in the +§2.4.2 design catalogue but has **zero set-sites** in the code. Where a rename +happened the live name is given in parentheses; where the concept shipped as a +metric rather than a span that is stated. Do not build a dashboard panel, an +alert rule, or a TraceQL query against anything in that column — the query will +return empty, and (per the PromQL/TraceQL asymmetry) a `=~".*"` matcher on an +absent attribute silently blanks a TraceQL panel while quietly passing in +PromQL. + +> **JobQueue metrics: two families, one label.** The Job row above has no span +> attributes, and the metrics that replace them do **not** all carry a +> `job_type` label. Getting this wrong produces a panel that renders but is +> wrong, so treat the two families as separate query surfaces: +> +> | Family | Where the job type lives | Source | +> | ------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +> | Native `XRPL_METRIC_*`: `job_queued_total`, `job_started_total`, `job_finished_total`, `job_queued_us`, `job_running_us` | In a **`job_type` label** | `MetricsRegistry.cpp:360-362` (counters), `:94-95` (histogram names), `:101` (label key) | +> | `beast::insight` `jobq` group: `jobq__waiting` / `_running` / `_deferred` / `_q` | In the **metric name itself** — there is **no** `job_type` label at all | `JobTypeData.h:29-32` (naming contract), `:35-38` (suffixes), `Application.cpp:392` (group) | +> +> **The trap:** `sum by (job_type)(jobq_…)` collapses every job type into a +> single series with an empty `job_type`, because an absent PromQL label is +> equivalent to `""` — the query returns a plausible-looking number rather than +> an error. Aggregate the `jobq_*` family with a name matcher +> (`{__name__=~"jobq_.*_waiting"}`) and reserve `by (job_type)` for the +> `job_*_total` / `job_*_us` family. ### 2.4.4 Privacy & Sensitive Data Policy @@ -400,26 +487,47 @@ OpenTelemetry instrumentation is designed to collect **operational metadata only The following data is explicitly **excluded** from telemetry collection: -| Excluded Data | Reason | -| ----------------------- | ----------------------------------------- | -| **Private Keys** | Never exposed; not relevant to tracing | -| **Account Balances** | Financial data; privacy sensitive | -| **Transaction Amounts** | Financial data; privacy sensitive | -| **Raw TX Payloads** | May contain sensitive memo/data fields | -| **Personal Data** | No PII collected | -| **IP Addresses** | Configurable; excluded by default in prod | +| Excluded Data | Reason | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Private Keys** | Never exposed; not relevant to tracing | +| **Account Balances** | Financial data; privacy sensitive | +| **Transaction Amounts** | Financial data; privacy sensitive | +| **Raw TX Payloads** | May contain sensitive memo/data fields | +| **Personal Data** | No PII collected | +| **IP Addresses** | **Never in spans** — no span sets an address attribute (`peer_address` has zero set-sites); peer spans identify peers by `peer_id`, an int64 process-local slot number. **But the log pipeline is a different story** — see the note below this table | + +> **Peer IPs DO leave the node — via the log pipeline, not via spans.** The +> "IP Addresses" row above is scoped to spans, and only to spans. This same +> document describes a log pipeline (§2.6.5) that carries peer addresses: +> +> 1. `PeerImp`'s constructor logs the peer's `remoteAddress_` — an `IP:port` — +> at `info` severity (`PeerImp.h:837-842`), and other overlay call sites log +> addresses too. These land in the ordinary `debug.log` stream. +> 2. The collector's `filelog` receiver tails exactly that file +> (`otel-collector-config.yaml:38-47`, `include: [/var/log/xrpld/*/debug.log]`) +> and the `logs` pipeline exports it to Loki (`:236-239`). +> +> So a deployment running the shipped stack **does** ship peer IPs off-box, as +> log bodies. There is no attribute to drop and no span-level switch to flip, +> because the IPs are inside free-text log messages rather than in structured +> fields — a `delete` action on an attribute key would not touch them. +> +> **The control points are therefore log-side, not trace-side:** Loki +> retention and access control on the log store; the `filelog` receiver's +> `include` list (dropping it disables log↔trace correlation entirely); or a +> collector-side transform on the log body. Do not describe the telemetry +> pipeline as IP-free without qualifying it to traces. #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Unconditional Redaction** | Account redaction is **not** configurable and cannot be turned off: `redactAccount()` (`Redaction.cpp:14-29`) hashes every **non-empty** address handed to it, with no flag and no bypass (an empty input returns empty — `Redaction.cpp:18-19` — so there is no raw value to leak either way). That is a stronger guarantee than a config switch: there is no insecure-by-default state to misconfigure | +| **Collector Tail Sampling** | **Optional, and OFF in the base stack.** xrpld head sampling is fixed at 1.0 (`Telemetry.h:234` `static constexpr double samplingRatio = 1.0;`), so 100% of traces leave the node. `docker/telemetry/otel-collector-config.yaml` has **no** `tail_sampling` processor either, so the local stack stores 100%. The only shipped policy is in the Grafana Cloud overlay (`otel-collector-config.grafanacloud.yaml:60-67`, wired at `:261`): one `probabilistic` policy at **0.5%**, on the trace-storage branch only so spanmetrics still see every span. Treat sampling as a cost control you opt into — not as a privacy control | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | #### Account Address Hashing @@ -429,20 +537,40 @@ failure mode. Protection is applied in two independent layers: 1. **SDK-side** (this node): the path-finding RPC handlers call `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the - `pathfind_source_account` / `pathfind_dest_account` span attributes. The - helper emits the first 16 characters of `sha512Half(address)` as - lowercase hex — deterministic (spans for one account still correlate) - but non-reversible. + `pathfind_source_account` / `pathfind_dest_account` span attributes. For a + non-empty address the helper emits the first 16 characters of + `sha512Half(address)` as lowercase hex — deterministic (spans for one + account still correlate) but non-reversible. An empty address returns empty + rather than the hash of the empty string (`Redaction.cpp:18-19`). 2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in the OpenTelemetry Collector re-hashes those same attributes, so any node that emitted a raw value is still redacted before storage. #### Collector-Level Data Protection -The OpenTelemetry Collector can be configured (via an `attributes` processor) -to hash or redact sensitive attributes before export — for example, hashing -`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` -to drop IP addresses, and deleting `params` to redact request parameters. +The shipped base config does exactly one thing here, and it is the +defense-in-depth layer described above: an `attributes/hash` processor +(`otel-collector-config.yaml:105-110`) hashing `pathfind_source_account` and +`pathfind_dest_account`. + +**No `peer_address` or `params` scrubbing rule is needed on the trace pipeline, +and none is shipped.** Earlier drafts prescribed `delete` actions for both. +Neither attribute is ever emitted: `peer_address` has zero set-sites in the code +(peer spans carry `peer_id`, an int64 process-local slot number — not an IP and +not a key), and no span sets a `params` attribute — RPC spans carry `command`, +`version`, `rpc_role`, `rpc_status`, `request_payload_size`, `is_batch`, +`batch_size` and `load_type`, never the request body. Adding delete rules for +absent keys would be harmless but misleading: it would imply the node emits IPs +and request parameters in spans when it does not. + +This says nothing about the **log** pipeline, which is where peer IPs actually +do leave the node (see the note under "Data NOT Collected" above). An +`attributes` processor cannot help there — the addresses are inside free-text +log bodies, not in structured attributes. + +If a future span _does_ introduce an IP-bearing or payload-bearing attribute, +the `attributes` processor is the right place to strip it — and the attribute +should be added to the §2.4 catalogue in the same change. #### Configuration Options for Privacy @@ -555,8 +683,8 @@ flowchart TB proto["message TraceContext {
bytes trace_id = 1; // 16 bytes
bytes span_id = 2; // 8 bytes
uint32 trace_flags = 3;
string trace_state = 4;
}"] end - subgraph jobqueue["JobQueue (Internal Async)"] - job["Context captured at job creation,
restored at execution

class Job {
otel::context::Context
traceContext_;
};"] + subgraph jobqueue["JobQueue / Coroutines (Internal Async)"] + job["CoroAwareContextStorage
(RuntimeContextStorage override)

Per-coroutine context stack,
installed globally at startup.
Job itself carries no context."] end style http fill:#0d47a1,stroke:#082f6a,color:#ffffff @@ -568,7 +696,7 @@ flowchart TB - **HTTP/WebSocket - RPC (blue)**: For client-facing RPC requests, trace context is propagated using the W3C `traceparent` header. This is the standard approach and works with any OTel-compatible client. - **Protocol Buffers - P2P (green)**: For peer-to-peer messages between xrpld nodes, trace context is embedded as a protobuf `TraceContext` message carrying trace_id, span_id, flags, and optional trace_state. -- **JobQueue - Internal Async (red)**: For asynchronous work within a single node, the OTel context is captured when a job is created and restored when the job executes on a worker thread. This bridges the async gap so spans remain linked. +- **JobQueue / Coroutines - Internal Async (red)**: For asynchronous work within a single node, the ambient OTel context follows the coroutine rather than being carried on the work item. `include/xrpl/core/Job.h` has **no** telemetry include and no `traceContext_` member — an earlier draft of this diagram showed one, and that was never built. Instead `xrpl::telemetry::CoroAwareContextStorage` (`include/xrpl/telemetry/CoroAwareContextStorage.h:84`) overrides the SDK's `RuntimeContextStorage` with a per-coroutine context stack, and is installed as the global storage in `Telemetry::start()` (`Telemetry.cpp:416-419`) before the tracer provider and before the first span. That fixes the wrong-thread scope pop across coroutine yield/resume and keeps log↔trace correlation intact. The storage is never reset — tearing it down while spans may still exist is undefined behaviour in the SDK — so it lives for the process lifetime. --- @@ -651,7 +779,10 @@ parent through the active context. ### 2.6.4 Coexistence Strategy -> **Note**: Phase 7 replaces the StatsD bridge with native OTel Metrics SDK export. The diagram below shows the Phase 6 intermediate state. See [Phase7_taskList.md](./Phase7_taskList.md) for the migration design where Beast Insight emits via OTLP instead of StatsD. +> **Note**: Phase 7 **added** a native OTel Metrics export path alongside the +> StatsD bridge; it did not replace it. The diagram below shows the Phase 6 +> state, which is still reachable today via `[insight] server=statsd`. See +> [Phase7_taskList.md](./Phase7_taskList.md) for the design. ```mermaid flowchart TB @@ -681,17 +812,54 @@ flowchart TB - **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. -**Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details. +**Phase 7 outcome (as shipped)**: Beast Insight gained an `OTelCollector` +`Collector` implementation that rides the global MeterProvider and exports via +OTLP/HTTP to the same collector as traces. It is selected with +`[insight] server=otel`. -### 2.6.5 Correlation with PerfLog +The three back ends are **co-equal branches of one `if/else` chain** in +`makeCollectorManager()` (`CollectorManager.cpp:37-75`), not a migration path: -Trace IDs can be correlated with existing PerfLog entries for comprehensive -debugging. The design is for `RPCHandler.cpp` to start an `rpc.command.` -span alongside the existing PerfLog `rpcStart`/`rpcFinish`/`rpcError` calls, -extract the span's `trace_id` (when valid), and eventually stamp it onto the -PerfLog entry (a planned `setTraceId` hook) so logs and traces share a key. The -span status is set to OK on success or to error (recording the exception) on -failure. +| `[insight] server=` | Collector | Status | +| ---------------------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `otel` | `OTelCollector` | OTLP/HTTP to the OTel Collector — the recommended setting | +| `statsd` | `StatsDCollector` | Unchanged from before Phase 7. **Not deprecated**: no warning is logged, no removal is scheduled, and the code path is not marked legacy | +| absent / anything else | `NullCollector` | **The default.** A node with no `[insight]` section emits no metrics at all | + +Two corrections to earlier drafts, both of which matter operationally: StatsD +is not a "deprecated fallback", and `otel` is not the default — you must set it +explicitly. See [06-implementation-phases.md §6.8](./06-implementation-phases.md), +[Phase7_taskList.md](./Phase7_taskList.md), and +[05 §5.8.6](./05-configuration-reference.md) for which `[insight]` keys are live +under `server=otel` (most are inert). + +### 2.6.5 Correlation with Logs + +**Shipped in Phase 8 — and not the way this section originally planned it.** +The design here was a `setTraceId` hook on PerfLog, fed from the +`rpc.command.` span in `RPCHandler.cpp`. That hook was never built: +`setTraceId` has zero occurrences in **source** — the only hits in the tree are +in these plan documents, describing the design that was dropped — and PerfLog's +JSON output carries no trace ID. + +What shipped instead is broader and needs no per-call-site wiring: the **journal +sink** stamps the IDs onto _every_ log line written while a span is active. +`Logs::format()` (`src/libxrpl/basics/Log.cpp:304-338`, inside +`#ifdef XRPL_ENABLE_TELEMETRY`) reads the thread-local OTel context, and when +the active span context is valid it prefixes the message with +`trace_id=<32 hex> span_id=<16 hex>`. It inspects the context value directly +rather than calling `GetSpan()`, so the common no-span path costs no heap +allocation. + +Because the IDs land in the ordinary `debug.log` stream, correlation is +end-to-end without touching PerfLog: the collector's `filelog` receiver parses +`trace_id`/`span_id` as optional capture groups and ships the lines to Loki, and +Grafana links both directions (Tempo `tracesToLogs` → Loki, Loki derived fields +→ Tempo). Details in [05 §5.8.5](./05-configuration-reference.md). + +RPC spans still exist and still set status (OK on success, error with the +recorded exception on failure) — that part of the original design is intact. +Only the PerfLog-stamping mechanism was replaced. --- diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md index f11af7a929..5eb5750905 100644 --- a/OpenTelemetryPlan/03-implementation-strategy.md +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -7,26 +7,54 @@ ## 3.1 Directory Structure -The telemetry implementation follows xrpld's existing code organization pattern: +The telemetry implementation follows xrpld's existing code organization +pattern. The tree below is the current on-disk contents of the three telemetry +directories, and it has three differences from the original design sketch worth +calling out: `TelemetryConfig.h`, `TraceContext.h`, `SpanAttributes.h` and +`TraceContext.cpp` were never created (config structs live inside +`Telemetry.h`, propagation lives in `TraceContextPropagator.h`, and attribute +constants live in the `*SpanNames.h` headers next to their owning class); the +metrics work of Phase 7/9 added a whole second module under +`src/xrpld/telemetry/`, which the sketch predated. ``` -include/xrpl/ -├── telemetry/ -│ ├── Telemetry.h # Main telemetry interface (global singleton) -│ ├── TelemetryConfig.h # Configuration structures -│ ├── TraceContext.h # Context propagation utilities -│ ├── SpanGuard.h # RAII span management with factory methods + discard() -│ ├── DiscardFlag.h # Thread-local discard flag -│ └── SpanAttributes.h # Attribute helper functions +include/xrpl/telemetry/ # libxrpl layer: tracing SDK wrapper +├── Telemetry.h # Interface + Setup config struct + factories +├── SpanGuard.h # RAII span management, factory methods, discard() +├── SpanNames.h # StaticStr/join() + shared span & attr constants +├── DiscardFlag.h # Thread-local discard flag +├── CoroAwareContextStorage.h # RuntimeContextStorage override for coroutines +├── DeterministicIdGenerator.h # trace_id from txHash / prevLedgerHash +├── TraceContextPropagator.h # protobuf TraceContext inject/extract (P2P) +├── TraceContextValidation.h # Validation of peer-supplied trace context +├── Redaction.h # redactAccount() — unconditional address hashing +└── GetObjectMetricNames.h # getobject_* metric name constants -src/libxrpl/ -├── telemetry/ -│ ├── Telemetry.cpp # Implementation + FilteringSpanProcessor -│ ├── TelemetryConfig.cpp # Config parsing -│ ├── TraceContext.cpp # Context serialization -│ └── NullTelemetry.cpp # No-op implementation +src/libxrpl/telemetry/ +├── Telemetry.cpp # TelemetryImpl + FilteringSpanProcessor + initMetrics() +├── TelemetryConfig.cpp # [telemetry] section parsing (makeTelemetrySetup) +├── SpanGuard.cpp # Span/scope guard implementation +├── CoroAwareContextStorage.cpp +├── DeterministicIdGenerator.cpp +├── Redaction.cpp +└── NullTelemetry.cpp # No-op impl — ALWAYS compiled (in-source #ifdef) + +src/xrpld/telemetry/ # xrpld layer: native metrics + tx tracing helpers +├── MetricsRegistry.h / .cpp # Owns the XRPL_METRIC_* instruments + MeterProvider +├── MetricMacros.h # XRPL_METRIC_COUNTER_ADD / _HISTOGRAM_RECORD / ... +├── ValidationTracker.h # Validation-agreement tracking (impl in detail/) +├── detail/ValidationTracker.cpp +├── ConsensusReceiveTracing.h # Peer proposal/validation receive spans +├── PropagationHelpers.h # Context inject/extract call-site helpers +├── TxSpanNames.h # tx.* span + attribute constants +└── TxTracing.h # Transaction span helpers ``` +Per-class span-name headers deliberately live next to their owning class rather +than in `telemetry/` — see `ConsensusSpanNames.h`, `TxApplySpanNames.h`, +`LedgerSpanNames.h`, `RpcSpanNames.h`, `PathFindSpanNames.h`, +`PeerSpanNames.h`, `TxQSpanNames.h`, `GrpcSpanNames.h`. + --- ## 3.2 Implementation Approach @@ -100,13 +128,22 @@ flowchart TB | --------------------- | --------- | ---------------------- | ---------- | | Span creation | 500-1000 | Every traced operation | Low | | Span end | 100-200 | Every traced operation | Low | -| SetAttribute (string) | 80-120 | 3-5 per span | Low | -| SetAttribute (int) | 40-60 | 2-3 per span | Negligible | +| SetAttribute (string) | 80-120 | 3-5 per span (typical) | Low | +| SetAttribute (int) | 40-60 | 2-3 per span (typical) | Negligible | | AddEvent | 100-200 | 0-2 per span | Low | | Context injection | 150-250 | Per outgoing message | Low | | Context extraction | 100-180 | Per incoming message | Low | | GetCurrent context | 10-20 | Thread-local access | Negligible | +> **"3-5 attributes per span" is a typical case, not a bound.** The frequency +> column above describes the median span (`tx.receive`, `rpc.command.*`). A few +> spans are deliberately attribute-rich: `consensus.accept.apply` sets **13** +> attributes (`RCLConsensus.cpp:600-674`), and `consensus.round` / +> `consensus.establish` are of the same order. Use ~15 as the worst case when +> sizing per-span attribute cost and memory; the consensus spans that hit it fire +> once per ~3-second round, so their absolute cost stays in the noise +> (see §3.4.3). + **Source**: Span creation based on OTel C++ SDK `BM_SpanCreation` benchmark (AlwaysOnSampler + SimpleSpanProcessor + InMemoryExporter), median ~1,000 ns on CI hardware. AddEvent includes timestamp read + string copy + vector push + mutex acquisition. Context injection/extraction @@ -120,8 +157,8 @@ confirmed by `BM_SpanCreationWithScope` benchmark delta (~160 ns). %%{init: {'pie': {'textPosition': 0.75}}}%% pie showData "tx.receive (1400ns)" : 1400 - "tx.validate (1200ns)" : 1200 - "tx.relay (1200ns)" : 1200 + "tx.process (1200ns)" : 1200 + "tx.apply (1200ns)" : 1200 "Context inject (200ns)" : 200 ``` @@ -131,9 +168,17 @@ pie showData **Overhead percentage**: 4.0 μs / 200 μs (avg tx processing) = **~2.0%** -> **Breakdown**: Each span (tx.receive, tx.validate, tx.relay) costs ~1,000 ns for creation plus +> **Breakdown**: Each span (tx.receive, tx.process, tx.apply) costs ~1,000 ns for creation plus > ~200-400 ns for 3-5 attribute sets. Context injection is ~200 ns (confirmed by benchmarks). > On production hardware, expect ~2.6 μs total (~1.3% overhead) due to faster span creation (~500-600 ns). +> +> This three-span model predates the apply-pipeline instrumentation. The shipped +> transaction path also emits `tx.preflight`, `tx.preclaim` and `tx.transactor` +> (the spans planned here as `tx.validate`), and never emits `tx.relay`. Scale +> the estimate by span count for a current figure: ~6 spans ≈ 7-8 μs on CI +> hardware, ~4-5 μs on server hardware. The measured end-to-end cost is in +> §3.5.3 (~3-4% throughput at head sampling 1.0), which supersedes this +> bottom-up estimate. ### 3.4.3 Consensus Round Overhead @@ -148,18 +193,26 @@ pie showData > **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for 1-2 attributes, totaling ~1,100-1,200 ns. > Context operations remain ~200 ns (confirmed by benchmarks). On production hardware, expect ~24 μs total. +> +> The "1-2 attributes" figure understates the shipped consensus spans, which are +> the attribute-rich ones: `consensus.accept.apply` alone sets 13 +> (`RCLConsensus.cpp:600-674`). Adding ~1 μs per such span still leaves the +> round total under ~40 μs against a ~3 s round, so the conclusion below is +> unaffected. Note also that the `consensus.phase` row covers the shipped names +> `consensus.phase.open`, `consensus.establish` and `consensus.accept` — see +> [02 §2.3.2](./02-design-decisions.md). **Overhead percentage**: 36 μs / 3s (typical round) = **~0.001%** (negligible) ### 3.4.4 RPC Request Overhead -| Operation | Cost (ns) | -| ---------------- | ------------ | -| rpc.request span | ~1200 | -| rpc.command span | ~1100 | -| Context extract | ~250 | -| Context inject | ~200 | -| **TOTAL** | **~2.75 μs** | +| Operation | Cost (ns) | +| ------------------------------------------ | ------------ | +| `rpc.http_request` / `rpc.ws_message` span | ~1200 | +| `rpc.command.*` span | ~1100 | +| Context extract | ~250 | +| Context inject | ~200 | +| **TOTAL** | **~2.75 μs** | > **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for attributes (command name, > version, role). Context extract/inject costs are confirmed by OTel C++ benchmarks. @@ -263,12 +316,20 @@ The overhead estimates in Sections 3.3-3.5 are derived from the following source > compression (~60-70% of raw) and batching (amortized headers), ~350 bytes/span is more realistic. > The table uses the conservative estimate for capacity planning. -| Sampling Rate | Spans/sec | Bandwidth | Notes | -| ------------- | --------- | --------- | ---------------- | -| 100% | ~500 | ~250 KB/s | Development only | -| 10% | ~50 | ~25 KB/s | Staging | -| 1% | ~5 | ~2.5 KB/s | Production | -| Error-only | ~1 | ~0.5 KB/s | Minimal overhead | +**Node → collector bandwidth is always the 100% row.** Head sampling is a +`static constexpr` 1.0 (`Telemetry.h:234`) with no config key, so every node +exports every span and the export bandwidth is not tunable from `xrpld.cfg`. + +| Sampling Rate | Spans/sec | Bandwidth | Where it applies | +| --------------------- | --------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| 100% | ~500 | ~250 KB/s | **The only reachable node→collector figure.** Plan capacity against this row | +| 0.5% | ~2.5 | ~1.25 KB/s | Collector→backend only, and only with the Grafana Cloud overlay's `tail_sampling` (`otel-collector-config.grafanacloud.yaml:60-67`) | +| 10% / 1% / error-only | — | — | **Not implemented.** No shipped config produces these ratios; treat them as illustrative of what a tail-sampling policy could do | + +The rows below 100% therefore reduce _storage_ cost at the backend, never the +node's egress. Note also that the shipped 0.5% policy is applied to the +trace-storage branch only, so the spanmetrics-derived RED metrics still see +100% of spans and stay exact. ### 3.6.2 Trace Context Propagation @@ -285,7 +346,26 @@ The overhead estimates in Sections 3.3-3.5 are derived from the following source ### 3.7.1 Sampling Strategies -#### Tail Sampling +#### Head Sampling (node) — fixed, not a decision point + +There is no sampling decision on the node. `samplingRatio` is a +`static constexpr double = 1.0` (`Telemetry.h:234`) and `TelemetryConfig.cpp:139` +records why nothing is parsed: a per-node ratio would let two nodes make +opposite keep/drop decisions for the same distributed trace, yielding partial +traces. The ratio sampler is wrapped in a `ParentBasedSampler` so a span with a +remote parent honours the upstream flag. The only node-local way to drop a span +is the explicit, per-call-site `SpanGuard::discard()`, enforced downstream by +`FilteringSpanProcessor`. + +#### Tail Sampling (collector) — aspirational shape + +The flowchart below is a **design sketch of a multi-policy tail sampler. It is +not what ships.** The base collector config has no `tail_sampling` processor at +all; the Grafana Cloud overlay has exactly one `probabilistic` policy at 0.5% +with no error or latency carve-outs. Read it as a template for a policy you +might write, not as a description of this repo — and note that adding +error/latency policies would need `decision_wait` tuning, since a policy can +only see spans that arrived within that window. ```mermaid flowchart TD @@ -299,13 +379,18 @@ flowchart TD consensus -->|No| slow{"Is Slow?"} slow -->|Yes| sample - slow -->|No| prob{"Random < 10%?"} + slow -->|No| prob{"Probabilistic keep?
(shipped policy: 0.5%)"} prob -->|Yes| sample prob -->|No| drop["DROP"] - style sample fill:#4caf50,stroke:#388e3c,color:#fff - style drop fill:#f44336,stroke:#c62828,color:#fff + style sample fill:#1b5e20,stroke:#0d3d14,color:#fff + style drop fill:#b71c1c,stroke:#7f1d1d,color:#fff + style trace fill:#0d47a1,stroke:#082f6a,color:#fff + style errors fill:#334155,stroke:#1e293b,color:#fff + style consensus fill:#334155,stroke:#1e293b,color:#fff + style slow fill:#334155,stroke:#1e293b,color:#fff + style prob fill:#334155,stroke:#1e293b,color:#fff ``` ### 3.7.2 Batch Tuning Recommendations @@ -318,7 +403,17 @@ flowchart TD ### 3.7.3 Conditional Instrumentation -Instrumentation is gated on two levels. A compile-time feature flag (`XRPL_ENABLE_TELEMETRY`) reduces the trace macros to no-ops when telemetry is built out, so disabled builds carry zero cost. At runtime, per-component guards (e.g. `shouldTracePeer()`) skip span creation for components whose tracing is turned off, incurring no overhead beyond a single boolean check. +Instrumentation is gated on two levels. A compile-time feature flag reduces the trace macros to no-ops when telemetry is built out, so disabled builds carry zero cost. At runtime, per-component guards (e.g. `shouldTracePeer()`) skip span creation for components whose tracing is turned off, incurring no overhead beyond a single boolean check. + +> The compile-time gate is the macro `XRPL_ENABLE_TELEMETRY`, but that macro is +> **not** the switch you flip. It is a compile definition added by +> `CMakeLists.txt` (`add_compile_definitions(XRPL_ENABLE_TELEMETRY)`) when the CMake option `telemetry` is ON. +> That option is declared ON today (`option(telemetry "Enable OpenTelemetry tracing" ON)`) +> only so that CI compiles the instrumented build while the telemetry branches are +> in review; **OFF is the intended default once merged**, flipped in a separate +> change. Select the value explicitly instead of relying on the default: +> `-Dtelemetry=ON|OFF` (CMake) or `-o telemetry=True|False` (Conan). See +> [05 §5.4.2](./05-configuration-reference.md). --- @@ -372,39 +467,78 @@ quadrantChart ### 3.9.4 Architectural Impact Assessment -| Aspect | Impact | Justification | -| -------------------- | ------- | -------------------------------------------------------------------------------- | -| **Data Flow** | Minimal | Read-only instrumentation; no modification to consensus or transaction data flow | -| **Threading Model** | Minimal | Context propagation uses thread-local storage (standard OTel pattern) | -| **Memory Model** | Low | Bounded queues prevent unbounded growth; RAII ensures cleanup | -| **Network Protocol** | Low | Optional fields in protobuf (high field numbers); backward compatible | -| **Configuration** | None | New config section; existing configs unaffected | -| **Build System** | Low | Optional CMake flag; builds work without OpenTelemetry | -| **Dependencies** | Low | OpenTelemetry SDK is optional; null implementation when disabled | +| Aspect | Impact | Justification | +| -------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Data Flow** | Minimal | Read-only instrumentation; no modification to consensus or transaction data flow | +| **Threading Model** | Minimal | Context propagation uses thread-local storage (standard OTel pattern) | +| **Memory Model** | Low | Bounded queues prevent unbounded growth; RAII ensures cleanup | +| **Network Protocol** | Low | Optional fields in protobuf (high field numbers); backward compatible | +| **Configuration** | None | New config section; existing configs unaffected | +| **Build System** | Low | A single CMake option (`telemetry`) selects the whole feature in or out, and builds work either way (`-Dtelemetry=ON` / `-Dtelemetry=OFF`). It is declared ON today only so CI compiles the instrumented paths; **OFF is the intended default once merged**, so the shipped build is opt-in | +| **Dependencies** | Medium | `opentelemetry-cpp/1.28.0` is a **conditional** requirement, never a hard one: `conanfile.py:152-153` adds it only `if self.options.telemetry`, and `:238-239` adds the matching `libxrpl` component requirement the same way. The option's declared default is `True` today (`conanfile.py:59`), so a default `conan install` does resolve it; with `-o telemetry=False` it never enters the graph and the null implementation supplies the factory | ### 3.9.5 Backward Compatibility -| Compatibility | Status | Notes | -| --------------- | ------- | ----------------------------------------------------- | -| **Config File** | ✅ Full | New `[telemetry]` section is optional | -| **Protocol** | ✅ Full | Optional protobuf fields with high field numbers | -| **Build** | ✅ Full | `XRPL_ENABLE_TELEMETRY=OFF` produces identical binary | -| **Runtime** | ✅ Full | `enabled=0` produces zero overhead | -| **API** | ✅ Full | No changes to public RPC or P2P APIs | +| Compatibility | Status | Notes | +| --------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Config File** | ✅ Full | New `[telemetry]` section is optional | +| **Protocol** | ✅ Full | Optional protobuf fields with high field numbers | +| **Build** | ✅ Full | `-Dtelemetry=OFF` (or `-o telemetry=False`) produces a binary with all tracing compiled out, whatever the option's declared default happens to be. **Not** `-DXRPL_ENABLE_TELEMETRY=OFF`, which does not disable anything — it is not a CMake option, only a compile definition that `CMakeLists.txt:152` adds inside the `if(telemetry)` block. CMake does flag it (`Manually-specified variables were not used by the project`) at the end of configuration, so it is not literally silent — but the warning is easy to scroll past and the resulting binary still has telemetry compiled in. See [05 §5.4.2](./05-configuration-reference.md) | +| **Runtime** | ✅ Full | `enabled=0` produces zero overhead | +| **API** | ✅ Full | No changes to public RPC or P2P APIs | ### 3.9.6 Rollback Strategy If issues are discovered after deployment: -1. **Immediate**: Set `enabled=0` in config and restart (zero code change) -2. **Quick**: Rebuild with `XRPL_ENABLE_TELEMETRY=OFF` +1. **Immediate**: Set `enabled=0` in `[telemetry]` and restart (zero code change). + Also set `[insight] server=` to something other than `otel` if metrics must + stop too — `enabled=0` governs tracing, and the metrics pipeline is selected + separately ([02 §2.6.4](./02-design-decisions.md)). +2. **Quick**: Rebuild with `-Dtelemetry=OFF` (CMake) or `-o telemetry=False` + (Conan). Pass the flag explicitly — an omitted flag resolves to the option's + declared default, which is ON today and OFF once the feature is merged; a + build that already has telemetry off needs no rebuild at all. + **Do not use `-DXRPL_ENABLE_TELEMETRY=OFF`** — it is not a CMake option, so + it is ignored (CMake reports it under `Manually-specified variables were not +used by the project`) and the rebuilt binary still has telemetry compiled in. + This step also drops the `opentelemetry-cpp` dependency, so expect a full + rebuild rather than an incremental one. 3. **Complete**: Revert telemetry commits (clean separation makes this easy) ### 3.9.7 Code Change Examples **Minimal RPC Instrumentation (Low Intrusiveness):** Instrumenting an RPC handler adds roughly 3-4 lines: one macro to start the span and one or two `setAttribute` calls (command name, status). The span ends automatically via RAII, so the existing control flow — process the request, send the result — is untouched. -**Consensus Instrumentation (Medium Intrusiveness):** Consensus is slightly more intrusive because child spans in later phase transitions need the round's context. Beyond the span-start and attribute macros, this requires storing the active context in a new member variable (`currentRoundContext_`) at round start. The existing round logic itself remains unchanged. +**Consensus Instrumentation (Medium Intrusiveness):** Consensus is slightly more intrusive because child spans in later phase transitions need the round's context. Beyond the span-start and attribute macros, this requires **four** new member variables on the adaptor rather than the single `currentRoundContext_` this section originally sketched (`RCLConsensus.h:103,113,123,143`): + +- `std::optional roundSpan_` (`:103`) — the round span + itself. It is **created and ended in one place**, `startRoundTracing()`: the + previous round's guard is released at `RCLConsensus.cpp:1288-1289` + (`if (roundSpan_) roundSpan_.reset();`) and the new one is emplaced a few + lines later — at `:1306` or `:1310` on the `"attribute"` strategy, at `:1319` + on the default `"deterministic"` one. `preStartRound()` does not create it; it + calls `startRoundTracing()` at `:1229`. There is no `reset()` method — the + span simply lives until the next round begins. A `SpanGuard` owns no + thread-local scope, so emplacing and resetting on different job workers is + safe. +- `telemetry::SpanContext roundSpanContext_` (`:113`) — a lightweight value-type + snapshot, captured at the end of `startRoundTracing()` (`:1350`). Child spans + link through this, not through an ambient parent, so code running on another + worker (e.g. `createValidationSpan()` on `jtACCEPT`) never touches + `roundSpan_` cross-thread. +- `telemetry::SpanContext prevRoundSpanContext_` (`:123`) — the prior round's + context, saved at `:1282` **before** the new span overwrites + `roundSpanContext_`, so the new round span can carry a follows-from link and + consecutive rounds stay navigable. +- `telemetry::SpanContext acceptSpanContext_` (`:143`) — the current round's + accept-span context, set at `:544` and cleared at `:1286` on each new round. + `createValidationSpan()` prefers it as the parent and falls back to + `roundSpanContext_` (`:1363-1373`), so a stale value must not survive into the + next round. + +The split is the point: the guard is owned by one thread, the contexts are +copied freely. The existing round logic itself remains unchanged. --- diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index cc3a83b347..e3b8ae5fe6 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -11,43 +11,115 @@ ### 5.1.1 Configuration File Section -The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Telemetry is disabled by default (`enabled=0`); enabling it turns on distributed tracing for transaction flow, consensus, and RPC calls, with traces exported to an OpenTelemetry Collector over OTLP. Head sampling is intentionally fixed at 1.0 (sample everything) and is not configurable — per-node head-sampling would produce broken/partial distributed traces, so volume reduction is delegated to the collector's tail sampling (see Section 7.4.2). The full option reference follows. +The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Telemetry is disabled by default (`enabled=0`); enabling it turns on distributed tracing for transaction flow, consensus, and RPC calls, with traces exported to an OpenTelemetry Collector over OTLP. Head sampling is intentionally fixed at 1.0 (sample everything) and is not configurable — per-node head-sampling would produce broken/partial distributed traces, so volume reduction is delegated to the collector's tail sampling (see Section 7.4.2). Transaction trace IDs are always deterministic (`trace_id = txHash[0:16]`); there is no strategy switch for the transaction path. The full option reference follows. + +> **`service_instance_id` is effectively required for `beast::insight` +> metrics — and only for those.** Three producers resolve the instance id +> independently, and exactly one of them lacks a node-key fallback: +> +> | Producer | Resource built by | Unset `service_instance_id` yields | +> | ------------------------------------------- | -------------------------------------------- | ---------------------------------------------- | +> | Traces (and therefore all `span_*` metrics) | `Telemetry::start()` | Base58 node public key | +> | Native `XRPL_METRIC_*` (`MetricsRegistry`) | `MetricsRegistry::initExporterAndProvider()` | Base58 node public key | +> | `beast::insight` (`[insight] server=otel`) | `TelemetryImpl` **constructor** | **`service.instance.id` absent** — no fallback | +> +> - **Traces**: the tracer resource is built in `Telemetry::start()` +> (`Telemetry.cpp:380-387`), which runs after `ApplicationImp::setup()` has +> called `setServiceInstanceId()` (`Application.cpp:1323`) with the Base58 +> node public key. An unset key therefore still yields the node key. The +> `spanmetrics` connector derives `span_calls_total` / +> `span_duration_milliseconds_*` from those spans, so span metrics inherit +> the correct id too. +> - **Native `XRPL_METRIC_*` metrics** build their **own** MeterProvider +> resource in `MetricsRegistry::initExporterAndProvider()` +> (`MetricsRegistry.cpp:280`, `:296-304`, provider created at `:339`), and +> `ApplicationImp::startTelemetry()` supplies the id with an explicit node-key +> fallback (`Application.cpp:1674-1679`: read the config key, and +> `if (instanceId.empty() && nodeIdentity_)` substitute +> `toBase58(TokenType::NodePublic, …)`). By then `setup()` has resolved +> `nodeIdentity_` (`Application.cpp:1315`), so these metrics carry the node +> key even with the config key unset. +> - **`beast::insight` metrics** are the exception. They use the **global** +> MeterProvider, whose resource is built in the `TelemetryImpl` +> **constructor** (`Telemetry.cpp:321-338`, `initMetrics()` at `:447`), +> because insight instruments are created eagerly in subsystem constructors +> and would otherwise bind to the noop provider forever. At that point +> `serviceInstanceId` is still `""` (`Application.cpp:348` passes an empty +> node key), and the code comment at `Telemetry.cpp:333-336` states plainly +> that the later setter "cannot change this immutable resource". Worse, +> `initMetrics()` sets the attribute **unconditionally** +> (`Telemetry.cpp:488`), so the resource carries `service.instance.id=""` +> rather than omitting it — whereas `MetricsRegistry` guards the same write +> with `if (!instanceId.empty())` (`MetricsRegistry.cpp:302-303`). +> +> Result: with `service_instance_id` unset, `beast::insight` metrics — and only +> those — export with an empty `service.instance.id`. Every shipped Grafana +> dashboard filters on `service_instance_id=~"$node"`, so **insight-backed +> panels** lose their per-node dimension; span-metric and `XRPL_METRIC_*` +> panels are unaffected. Set the key explicitly on any node whose insight +> metrics are dashboarded. +> +> **Known issue.** The asymmetry is a defect, not a design: `MetricsRegistry` +> already demonstrates the node-key fallback that the global provider needs. +> A fix would have to resolve the node identity before `TelemetryImpl` is +> constructed, or make the insight metrics use a late-built provider. ### 5.1.2 Configuration Options Summary -| Option | Type | Default | Description | -| -------------------------- | ------ | --------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `enabled` | bool | `false` | Enable/disable telemetry | -| `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint | -| `use_tls` | bool | `false` | Enable TLS for exporter connection | -| `tls_ca_cert` | string | `""` | Path to CA certificate file | -| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | -| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | -| `batch_size` | uint | `512` | Spans per export batch | -| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | -| `max_queue_size` | uint | `2048` | Maximum queued spans | -| `trace_transactions` | bool | `true` | Enable transaction tracing | -| `trace_consensus` | bool | `true` | Enable consensus tracing | -| `trace_rpc` | bool | `true` | Enable RPC tracing | -| `trace_peer` | bool | `true` | Enable peer message tracing (high volume) | -| `trace_ledger` | bool | `true` | Enable ledger tracing | -| `tx_trace_strategy` | string | `"deterministic"` | TX trace ID strategy: `"deterministic"` (trace_id = txHash[0:16]) or `"attribute"` (random) | -| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"attribute"` (random) | -| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics | -| `service_instance_id` | string | `` | Instance identifier | +| Option | Type | Default | Description | +| -------------------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | bool | `false` | Enable/disable telemetry | +| `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint for **traces** | +| `metrics_endpoint` | string | `http://localhost:4318/v1/metrics` | OTLP/HTTP collector endpoint for the native metrics pipeline (`MetricsRegistry`). Read in `Application.cpp:1670` | +| `use_tls` | bool | `false` | Enable TLS for exporter connection | +| `tls_ca_cert` | string | `""` | Path to CA certificate file | +| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | +| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | +| `batch_size` | uint | `512` | Spans per export batch | +| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | +| `max_queue_size` | uint | `2048` | Maximum queued spans | +| `trace_transactions` | bool | `true` | Enable transaction tracing | +| `trace_consensus` | bool | `true` | Enable consensus tracing | +| `trace_rpc` | bool | `true` | Enable RPC tracing | +| `trace_peer` | bool | `true` | Enable peer message tracing (high volume) | +| `trace_ledger` | bool | `true` | Enable ledger tracing | +| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"attribute"` (random). Parsed at `TelemetryConfig.cpp:155-156`, consumed at `RCLConsensus.cpp:1291,1296`. **Not validated** — see the note below | +| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics | +| `service_instance_id` | string | node public key (base58) | Instance identifier (`service.instance.id`). Traces, span metrics and native `XRPL_METRIC_*` metrics all fall back to the node key; **`beast::insight` metrics do not** — see the note in §5.1.1 | + +**`consensus_trace_strategy` is not validated.** `TelemetryConfig.cpp:155-156` +copies the raw string into `Setup::consensusTraceStrategy` without checking it +against an allowed set, and the only comparison in the code is +`strategy == "attribute"` (`RCLConsensus.cpp:1296`). Any unrecognised value — +including a typo — silently takes the deterministic branch with no log warning. +The two accepted values are documented at `include/xrpl/telemetry/Telemetry.h:287-292`. + +**Not a config key — deterministic transaction trace IDs are unconditional.** +Earlier drafts of this document listed a `tx_trace_strategy` option +(`"deterministic"` \| `"attribute"`). No such key exists: `TelemetryConfig.cpp` +parses no transaction-strategy key, and the transaction trace ID is always +derived from the transaction hash. Only the **consensus** path has a +switchable strategy. **Planned (not yet implemented)**: the following options appear in the design -documents but are not parsed by `TelemetryConfig.cpp` in Phase 1b and later -phases. They will be added as the corresponding subsystems are instrumented: +documents but are not parsed by `TelemetryConfig.cpp`. They will be added as +the corresponding subsystems are instrumented: -| Option | Planned Phase | Purpose | -| -------------------------- | ------------- | ----------------------------------------------------------------------- | -| `exporter` | Future | Select between OTLP/HTTP and OTLP/gRPC | -| `trace_pathfind` | Phase 2 | Path computation tracing toggle | -| `trace_txq` | Phase 3 | Transaction queue tracing toggle | -| `trace_validator` | Future | Validator list / manifest update tracing | -| `trace_amendment` | Future | Amendment voting tracing | -| `consensus_trace_strategy` | Phase 4 | Trace ID strategy for consensus rounds (`deterministic` \| `attribute`) | +| Option | Planned Phase | Purpose | +| ----------------- | ------------- | ---------------------------------------- | +| `exporter` | Future | Select between OTLP/HTTP and OTLP/gRPC | +| `trace_pathfind` | Phase 2 | Path computation tracing toggle | +| `trace_txq` | Phase 3 | Transaction queue tracing toggle | +| `trace_validator` | Future | Validator list / manifest update tracing | +| `trace_amendment` | Future | Amendment voting tracing | + +> **`exporter` is not read, so do not set it.** Both shipped sample configs +> (`docker/telemetry/xrpld-telemetry.cfg`, +> `docker/telemetry/xrpld-telemetry-mainnet.cfg`) used to carry +> `exporter=otlp_http`; the line had no effect and has since been replaced with +> a comment saying so. OTLP/HTTP is the only transport that exists (§2.2.1), and +> `endpoint` / `metrics_endpoint` are the only transport knobs, until the §2.2.2 +> gRPC work lands. --- @@ -55,7 +127,16 @@ phases. They will be added as the corresponding subsystems are instrumented: > **TxQ** = Transaction Queue -The parser `makeTelemetrySetup()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` reads the `[telemetry]` `Section` and populates a `Telemetry::Setup` struct, applying the defaults listed in Section 5.1.2 via `section.value_or(...)`. It derives `serviceInstanceId` from the node public key when not overridden, selects the exporter endpoint default by exporter type, and leaves the sampling ratio at its fixed 1.0 default (not read from config — see Section 7.4.2). +The parser `makeTelemetrySetup()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` reads the `[telemetry]` `Section` and populates a `Telemetry::Setup` struct, applying the defaults listed in Section 5.1.2 via `section.valueOr(...)`. It takes `serviceInstanceId` from the `nodePublicKey` argument when the key is absent, applies one unconditional `endpoint` default (`dflt::endpoint`, `TelemetryConfig.cpp:61`, used at `:108`) — the parser has no notion of exporter type — and leaves the sampling ratio at its fixed 1.0 default (a `static constexpr` member, so there is nothing to parse; `TelemetryConfig.cpp:139`, `Telemetry.h:234`). It also rejects two contradictory mTLS configurations outright (`tls_client_cert` without `tls_client_key`, and either without `use_tls=1`) rather than failing open at handshake time. + +`metrics_endpoint` is deliberately **not** handled here: it is read separately in `ApplicationImp::startTelemetry()` (`Application.cpp:1670`) and passed to `MetricsRegistry::start()`. Note the consequence — the two metric exporters resolve their URL differently: + +| Metric source | Exporter built by | URL comes from | +| ------------------------------------------ | -------------------------------------------- | -------------------------------------------------------------------- | +| `beast::insight` (`[insight] server=otel`) | `Telemetry::initMetrics()` (global provider) | `endpoint` with a trailing `/v1/traces` rewritten to `/v1/metrics` | +| Native `XRPL_METRIC_*` (`MetricsRegistry`) | `MetricsRegistry::initExporterAndProvider()` | `metrics_endpoint`, defaulting to `http://localhost:4318/v1/metrics` | + +Setting a non-default `endpoint` therefore moves the insight metrics with it, but leaves the native metrics on localhost unless `metrics_endpoint` is set too. --- @@ -68,6 +149,13 @@ The parser `makeTelemetrySetup()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` > resolved later in `setup()`. The `Telemetry` object is therefore > constructed with an empty `serviceInstanceId` and patched via > `setServiceInstanceId()` once `setup()` has called `getNodeIdentity()`. +> **This patch reaches traces only.** The **global** MeterProvider resource — +> the one `beast::insight` metrics use — is already frozen by then (§5.1.1), so +> those metrics keep whatever `service_instance_id` the config supplied (`""` +> if it supplied none). Native `XRPL_METRIC_*` metrics do not go through this +> patch at all: `startTelemetry()` re-reads the config key and applies its own +> node-key fallback when building `MetricsRegistry`'s separate resource +> (`Application.cpp:1674-1679`). `ApplicationImp` (in `src/xrpld/app/main/Application.cpp`) owns a `std::unique_ptr telemetry_`. It is built in the member initializer list via `makeTelemetry(makeTelemetrySetup(...))` with an empty `serviceInstanceId`, then patched in `setup()` by calling `setServiceInstanceId()` with the Base58 node public key (unless the user supplied a custom `service_instance_id`). `start()` and `run()` forward to `telemetry_->start()` / `telemetry_->stop()`, and `getTelemetry()` returns the owned instance. @@ -87,13 +175,72 @@ The parser `makeTelemetrySetup()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` > **OTLP** = OpenTelemetry Protocol -### 5.4.1 Find OpenTelemetry Module +### 5.4.1 Locating the OpenTelemetry SDK -A `cmake/FindOpenTelemetry.cmake` module locates the OpenTelemetry C++ SDK. It first tries `find_package(opentelemetry-cpp CONFIG)`, aliasing the imported targets `OpenTelemetry::api`, `OpenTelemetry::sdk`, and `OpenTelemetry::otlp_grpc_exporter`, and falls back to `pkg-config` when no CMake config package is present. +> **Superseded design.** Earlier drafts described a hand-written +> `cmake/FindOpenTelemetry.cmake` module that aliased `OpenTelemetry::api`, +> `OpenTelemetry::sdk` and `OpenTelemetry::otlp_grpc_exporter` with a +> `pkg-config` fallback. That module was never written — it exists in no +> commit — and the aliasing approach it described does not work with the +> package the build actually consumes. + +The SDK is located by the Conan-generated CMake config package, nothing else: + +- `CMakeLists.txt` — `find_package(opentelemetry-cpp CONFIG REQUIRED)`, + guarded by the `telemetry` option (§5.4.2). The dependency itself is + declared in `conanfile.py:153` (`opentelemetry-cpp/1.28.0`), also guarded — + `requirements()` adds it only `if self.options.telemetry` (`:152`), so with + the option off the package never enters the dependency graph. +- Linking goes through the **umbrella** target + `opentelemetry-cpp::opentelemetry-cpp`, never the per-component targets. + `cmake/XrplCore.cmake:221-225` and `:83-91` record why: the Conan package + under-declares its inter-component dependencies, so naming `::api` / `::sdk` + individually produces the wrong static-link order and fails at executable + link time. The umbrella target supplies both the trace and metrics + components with the correct ordering. ### 5.4.2 CMakeLists.txt Changes -The top-level `CMakeLists.txt` adds an `XRPL_ENABLE_TELEMETRY` option (default `OFF`). When enabled, it runs `find_package(OpenTelemetry REQUIRED)`, defines the `XRPL_ENABLE_TELEMETRY` compile flag, and builds the `xrpl_telemetry` library from the real telemetry sources linked against the OpenTelemetry targets; when disabled, it builds the same target from a no-op `NullTelemetry.cpp` so call sites compile unchanged. +The build flag is `telemetry`: + +``` +option(telemetry "Enable OpenTelemetry tracing" ON) # top-level CMakeLists.txt +``` + +The declared value is ON **temporarily**, so that CI compiles the telemetry code +paths while the feature branches are in review. **OFF is the intended default +once merged**, and the flip is a separate change. Set the value explicitly +rather than relying on the default: + +| To … | Use (CMake) | Use (Conan) | +| ------------------------- | ----------------- | -------------------- | +| Build telemetry in | `-Dtelemetry=ON` | `-o telemetry=True` | +| Build it out (all no-ops) | `-Dtelemetry=OFF` | `-o telemetry=False` | + +When the option is ON, the guarded block below it runs +`find_package(opentelemetry-cpp CONFIG REQUIRED)` and adds the +**compile definition** `XRPL_ENABLE_TELEMETRY`. + +> **`XRPL_ENABLE_TELEMETRY` is not a CMake option.** It is only ever _added_ +> as a compile definition by `add_compile_definitions(XRPL_ENABLE_TELEMETRY)` in that same block. Passing +> `-DXRPL_ENABLE_TELEMETRY=OFF` on the CMake command line disables **nothing** — +> it defines an unused cache variable and telemetry stays compiled in. CMake does +> report it, at the end of configuration under `Manually-specified variables were +not used by the project`, so it is not literally silent — but that line is easy +> to scroll past. Any procedure that relies on it (including the rollback path in +> [§3.9.6](./03-implementation-strategy.md)) must use `-Dtelemetry=OFF`. + +The target is `xrpl.libxrpl.telemetry`, created by `add_module(xrpl telemetry)` +at `cmake/XrplCore.cmake:231` from `include/xrpl/telemetry/` + +`src/libxrpl/telemetry/`. There is no `xrpl_telemetry` target. + +Selection between the real and the no-op implementation is an **in-source +`#ifdef`, not a source swap**: `NullTelemetry.cpp` is compiled into the target +unconditionally (see its header comment, `NullTelemetry.cpp:1-12`). It provides +the `makeTelemetry()` factory when `XRPL_ENABLE_TELEMETRY` is undefined; when +the macro is defined, `Telemetry.cpp` provides the factory instead and +`NullTelemetry`'s virtuals only serve as noop tracer/span fallbacks. Call sites +compile unchanged either way. --- @@ -105,13 +252,101 @@ The top-level `CMakeLists.txt` adds an `XRPL_ENABLE_TELEMETRY` option (default ` The authoritative collector config lives in the repo at `docker/telemetry/otel-collector-config.yaml` (with Tempo backend config in `docker/telemetry/tempo.yaml`). The sections below summarize the development and production shapes of that pipeline. -### 5.5.1 Development Configuration +### 5.5.1 Development / Base Configuration -The development collector enables an OTLP receiver on both gRPC (`0.0.0.0:4317`) and HTTP (`0.0.0.0:4318`), a single `batch` processor (1s timeout, batch size 100), and two exporters: a `logging` exporter for console debugging and `otlp/tempo` (insecure) for trace visualization. The single `traces` pipeline wires receiver → batch → both exporters. +`docker/telemetry/otel-collector-config.yaml` is the base config used by the +local stack and by CI. It carries **three** pipelines, not one: + +| Pipeline | Receivers | Processors | Exporters | +| --------- | --------------------- | ---------------------------------------------------------------- | ------------------------------------ | +| `traces` | `otlp` | `resource/tier`, `resource/stripsdk`, `attributes/hash`, `batch` | `debug`, `otlp/tempo`, `spanmetrics` | +| `metrics` | `otlp`, `spanmetrics` | `resource/tier`, `resource/stripsdk`, `batch` | `prometheus` | +| `logs` | `filelog` | `resource/logs`, `resource/tier`, `resource/stripsdk`, `batch` | `otlphttp/loki` | + +Component detail: + +- **Receivers.** `otlp` on gRPC `0.0.0.0:4317` and HTTP `0.0.0.0:4318` (both + traces and native metrics arrive on 4318). `filelog` tails + `/var/log/xrpld/*/debug.log` and runs a `regex_parser` that lifts + `timestamp`, `partition`, `severity` and the optional `trace_id`/`span_id` + emitted by the journal sink (§5.8.5). +- **Processors.** `batch` (1s timeout, `send_batch_size: 100`); + `resource/tier` (`action: upsert` on `deployment.environment`, `action: insert` on + `xrpl.network.type` only when absent); `resource/stripsdk` (drops the + `telemetry.sdk.*` attributes); `resource/logs` (`action: upsert` on + `service.name` and `job` — only the former becomes a Loki stream label, see + the known issue in §5.8.5); `attributes/hash` (hashes + `pathfind_source_account` and `pathfind_dest_account`). +- **Connector.** `spanmetrics` with `namespace: "span"` + (`otel-collector-config.yaml:114`) — this is why the derived RED metrics are + `span_calls_total` / `span_duration_milliseconds_*`. The connector's own + default namespace is **empty**, so without this setting the names would be + the bare `calls_total` / `duration_milliseconds_*`. The + `traces_spanmetrics_*` family is **not** the connector's default and is not + produced here at all — it comes from a different producer, Tempo's + `metrics_generator` `span-metrics` processor (`tempo.yaml:75`), whose + `remote_write` is commented out in this repo (see §5.8.6). Histogram + `unit: ms` + with sub-millisecond buckets from `0.01ms`, plus explicit `2s`–`30s` + boundaries for consensus and `ledger.acquire`. ~25 low-cardinality + dimensions are promoted to labels (`command`, `rpc_status`, `tx_type`, + `ter_result`, `stage`, `consensus_mode`, `outcome`, …). +- **Exporters.** `debug` (console, `verbosity: detailed`), `otlp/tempo` + (`tempo:4317`, `tls.insecure: true`), `otlphttp/loki` + (`http://loki:3100/otlp` — Loki 3.x native OTLP; the old `loki` exporter was + removed in collector-contrib v0.147.0), and `prometheus` on + `0.0.0.0:8889` with `resource_to_telemetry_conversion.enabled: true` so the + tier and instance resource attributes become Prometheus labels. +- **Extensions.** `health_check` on `0.0.0.0:13133` only. There is **no** + `zpages` extension. + +Deliberately absent from the base config — do not document them as present: +no `memory_limiter`, no `tail_sampling`, no Elastic APM exporter, and no +`tx_account` attribute rule (the hashed keys are the two `pathfind_*_account` +ones). ### 5.5.2 Production Configuration -The production collector adds TLS on the OTLP gRPC receiver and a richer processor chain: a `memory_limiter` (OOM guard), `batch` (5s timeout, size 512), `tail_sampling`, and an `attributes` processor that hashes sensitive fields (e.g. `tx_account`) and stamps `deployment.environment`. Tail sampling keeps all `ERROR` traces, slow consensus rounds (>5s) and slow RPC requests (>1s), and probabilistically samples the remainder at 10%. Exporters target Grafana Tempo (TLS) and Elastic APM; `health_check` and `zpages` extensions are enabled for operability. +There is no separate "production" collector config in this repo. The one +overlay that exists is `docker/telemetry/otel-collector-config.grafanacloud.yaml`. +It is **not** the base config plus one processor — it restructures the service +graph. The full delta: + +| Added by the overlay | Where | Purpose | +| ------------------------ | ------ | ------------------------------------------------------------------------- | +| `basicauth/grafanacloud` | `:29` | Extension; instance id / API token from the container environment | +| `tail_sampling` | `:60` | One `probabilistic` policy at **0.5%**, `decision_wait: 10s` | +| `transform/cloudlabels` | `:119` | Copies three resource attrs onto datapoint labels for Cloud (OTLP) ingest | +| `otlphttp/grafanacloud` | `:236` | Single OTLP/HTTP exporter fanning all three signals to Grafana Cloud | +| `metrics_flush_interval` | `:136` | `spanmetrics` flushes every 15s instead of the 60s default | + +| Removed by the overlay | Consequence | +| ---------------------- | ---------------------------------------------------------------------------- | +| `attributes/hash` | **Pathfinding account attributes are not hashed on this config** — see below | +| `debug` | No console span dump; collector logs alone when diagnosing ingest | + +Pipelines go from **three** (`traces`, `metrics`, `logs`) to **five** +(`:253-280`): `traces/metrics`, `traces/store`, `metrics/local`, +`metrics/cloud`, `logs`. `tail_sampling` is applied in **`traces/store`** +(`:259-261`) — the branch feeding Tempo and Grafana Cloud — not in a pipeline +named `traces`, which does not exist in the overlay. The `traces/metrics` +branch feeds `spanmetrics` unsampled, so the derived RED metrics stay exact +while stored traces are ~1/200 of ingested ones. + +> **Known issue — the cloud path does not hash pathfinding accounts.** The base +> config runs `attributes/hash` on its `traces` pipeline +> (`otel-collector-config.yaml:105-110`), hashing `pathfind_source_account` and +> `pathfind_dest_account` as defense in depth behind the node-side hashing. The +> overlay declares no such processor and lists none on any of its five +> pipelines, so on the Grafana Cloud config those two attributes reach **both** +> Grafana Cloud and the local Tempo with whatever value the node sent. Any node +> that emits raw addresses loses its second line of defense. Adding +> `attributes/hash` to `traces/store` and `traces/metrics` would close the gap. + +Hardening a collector for a real deployment (TLS/mTLS on the receiver, +NetworkPolicy, peer trace-context validation) is covered in +[Securing the OTel Pipeline](./secure-OTel.md) — not by any config file in +`docker/telemetry/`. --- @@ -119,7 +354,31 @@ The production collector adds TLS on the OTLP gRPC receiver and a richer process > **OTLP** = OpenTelemetry Protocol -The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up four services on a shared `xrpld-telemetry` network: an `otel-collector` (otel/opentelemetry-collector-contrib) exposing OTLP gRPC `4317`, OTLP HTTP `4318`, and health check `13133`; `tempo` for trace storage/visualization; `grafana` with provisioned datasources and dashboards (anonymous admin enabled); and an optional `prometheus` for metric correlation. +The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up **six** services on a shared `xrpld-telemetry` bridge network. All images are pinned to exact tags. + +| Service | Image | Published ports | Role | +| ---------------- | ---------------------------------------------- | ---------------------- | ---------------------------------------------------------------- | +| `otel-collector` | `otel/opentelemetry-collector-contrib:0.158.0` | `4317`, `4318`, `8889` | OTLP ingest, spanmetrics, filelog tail, Prometheus scrape target | +| `tempo` | `grafana/tempo:2.9.4` | `3200` | Trace storage and TraceQL | +| `loki` | `grafana/loki:3.7.6` | `3100` | Log storage for log↔trace correlation | +| `prometheus` | `prom/prometheus:v3.13.2` | `9090` | Scrapes the collector's `:8889` | +| `grafana` | `grafana/grafana:13.1.2` | `3000` | Dashboards + provisioned datasources/alerts, anonymous admin | +| `renderer` | `grafana/grafana-image-renderer:v5.12.0` | `8081` | Panel→PNG rendering for image export and alert screenshots | + +Two corrections to earlier drafts: + +- **`prometheus` is not optional.** `grafana` lists it in `depends_on` (along + with `tempo`, `loki` and `renderer`), and 7 of the 15 dashboards query + `span_calls_total` from it. Removing it blanks most panels. +- **Port `13133` is not published.** The collector's `health_check` extension + listens on `13133` inside the container, but the base compose file publishes + only `4317`, `4318` and `8889`. Health checks from the host must either add a + port mapping or run `docker compose exec`. + +The collector also bind-mounts the xrpld log root read-only +(`${XRPLD_LOG_DIR:-./data/logs}` → `/var/log/xrpld`) for the `filelog` +receiver, and the `grafana` service reads Slack/email alert secrets from an +optional gitignored `.env.alerting`. --- @@ -131,7 +390,7 @@ The authoritative development stack lives in the repo at `docker/telemetry/docke flowchart TB subgraph config["Configuration Sources"] cfgFile["xrpld.cfg
[telemetry] section"] - cmake["CMake
XRPL_ENABLE_TELEMETRY"] + cmake["CMake option: telemetry
ON today for CI, OFF once merged
when ON, defines XRPL_ENABLE_TELEMETRY"] end subgraph init["Initialization"] @@ -168,7 +427,7 @@ flowchart TB **Reading the diagram:** -- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake `telemetry` option controls whether telemetry is compiled in at all. That option is declared ON today only so CI compiles the instrumented paths; OFF is the intended default once merged, so treat the build gate as something to pass explicitly, and the runtime gate is opt-in either way (`enabled=0` by default). Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. - **Initialization**: `makeTelemetrySetup()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects. - **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. - **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) @@ -184,29 +443,88 @@ Step-by-step instructions for integrating xrpld traces with Grafana. ### 5.8.1 Data Source Configuration -#### Tempo (Recommended) +Three datasources are provisioned from `docker/telemetry/grafana/provisioning/datasources/`. There is **no** Elastic APM datasource — `elastic-apm.yaml` was described in an earlier draft but never existed. Elastic remains a _possible_ backend (§7.2); nothing in this repo provisions it. -A Tempo datasource (`grafana/provisioning/datasources/tempo.yaml`, provisioned from `docker/telemetry/grafana/`) points at `http://tempo:3200` and enables `tracesToLogs` (linking to Loki on `service.name`/`tx_hash` and mapping `trace_id` → `traceID`), `serviceMap` against Prometheus, the node graph, and Loki search. +| File | Type | URL | uid | Notes | +| ----------------- | ------------ | ------------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| `tempo.yaml` | `tempo` | `http://tempo:3200` | `tempo` | `nodeGraph`, `serviceMap`/`tracesToMetrics` → `prometheus`, `tracesToLogs` → `loki`, plus ~30 Explore search filters | +| `prometheus.yaml` | `prometheus` | `http://prometheus:9090` | `prometheus` | Backs every span-metric and native-metric panel | +| `loki.yaml` | `loki` | `http://loki:3100` | `loki` | Backs `log-derived-insights`; derived fields jump back to Tempo | -#### Elastic APM +The Tempo `tracesToLogs` block is configured as `filterByTraceID: true`, +`filterBySpanID: false`, **`tags: []`**. The empty tag list is deliberate: the +correlation is by trace ID alone, so no span attribute needs to exist on both +sides. Earlier drafts claimed `trace_id` + `tx_hash` tags — that is not what +ships, and adding a tag Tempo cannot resolve blanks the link. -Alternatively, an Elasticsearch datasource (`grafana/provisioning/datasources/elastic-apm.yaml`) of type `elasticsearch` points at `http://elasticsearch:9200` against the `apm-*` index, using `@timestamp` as the time field and mapping the log message/level fields. +The search-filter list is the practical index of queryable span attributes: +resource scope (`service.name`, `service.instance.id`, `service.version`, +`xrpl.network.id`, `xrpl.network.type`), intrinsics (`name`, `status`, +`duration`), and span scope (`command`, `rpc_status`, `rpc_role`, `tx_hash`, +`tx_type`, `tx_status`, `local`, `path`, `suppressed`, `peer_version`, +`consensus_*`, `ledger_seq`, `ledger_hash`, `close_time_correct`, +`close_resolution_ms`, `proposers`, `mode_old`, `mode_new`, `txq_status`, +`ter_code`). ### 5.8.2 Dashboard Provisioning -A dashboard provider (`grafana/provisioning/dashboards/dashboards.yaml`) loads the `xrpld` dashboard folder from disk (`/var/lib/grafana/dashboards/rippled`), polling for changes every 30s with deletion disabled. +`grafana/provisioning/dashboards/dashboards.yaml` declares a single `file` +provider named `xrpld-telemetry`, `orgId: 1`, targeting Grafana folder `xrpld` +from path `/var/lib/grafana/dashboards` (no `/rippled` suffix), with +`disableDeletion: false`, `editable: true`, `foldersFromFilesStructure: false`. +It sets **no** poll interval — Grafana's `updateIntervalSeconds` default +applies; the "every 30s" figure in earlier drafts was invented. -### 5.8.3 Example Dashboard: RPC Performance +`docker-compose.yml` mounts `./grafana/dashboards` read-only at that path, so +the 15 JSON files in `docker/telemetry/grafana/dashboards/` are what gets +provisioned. -An example `xrpld RPC Performance` dashboard (uid `xrpld-rpc-performance`) sourced from Tempo via TraceQL provides four panels: RPC latency by command (heatmap), RPC error rate by command (timeseries), the top 10 slowest RPC commands by average duration (table), and a recent-traces table. +### 5.8.3 Shipped Dashboards -### 5.8.4 Example Dashboard: Transaction Tracing +The dashboards are Prometheus-first, not TraceQL-first, and their uids are +bare (no `xrpld-` prefix). The full inventory and per-panel query reference is +[09-data-collection-reference.md](./09-data-collection-reference.md); the uids +are: -An example `xrpld Transaction Tracing` dashboard (uid `xrpld-tx-tracing`) over Tempo provides three panels: transaction throughput (`tx.receive` rate, stat), cross-node relay count (average `span.relay_count` on `tx.relay`, timeseries), and a table of transaction validation errors (`tx.validate` with `status.code=error`). +`consensus-health`, `fee-market`, `job-queue`, `ledger-data-sync`, +`ledger-operations`, `log-derived-insights`, `network-traffic`, `node-health`, +`overlay-traffic-detail`, `peer-network`, `peer-quality`, `rpc-pathfinding`, +`rpc-performance`, `transaction-overview`, `validator-health`. -### 5.8.5 TraceQL Query Examples +> **Panel-count convention used in these docs**: counts are of **data panels +> only** — `type: "row"` collapsible headers are excluded, because a row is a +> layout element with no query. A board's raw `panels` array is therefore longer +> than its stated count (e.g. `rpc-performance` has 19 array entries: 2 rows + +> 17 data panels). -Common queries for xrpld traces: +Two examples described in earlier drafts do not exist and should not be looked +for: `xrpld-rpc-performance` (the real board is `rpc-performance`, **17** data +panels in 2 rows, mostly Prometheus span metrics) and `xrpld-tx-tracing` (the +transaction board is `transaction-overview`, **18** data panels in 3 rows; its +error panel filters `span_calls_total{span_name="tx.process", +ter_result!~"tesSUCCESS|"}`, since no `tx.validate` span was ever built — see +[02 §2.3.2](./02-design-decisions.md)). + +> **Why `!~"tesSUCCESS|"` and not `!="tesSUCCESS"`.** An absent Prometheus label +> compares equal to the empty string, and `tx.process` can end **without** a +> `ter_result` attribute: `processTransaction()` returns early when +> `preProcessTransaction()` rejects the transaction +> (`NetworkOPs.cpp:1437-1438`) and `doTransactionAsync()` returns early when the +> transaction is already applying (`:1461-1462`); the only setter runs later, at +> `:1674`. Those series carry `ter_result=""`, which `!="tesSUCCESS"` counts as +> an error. The regex form excludes the empty value explicitly (the trailing +> `|` alternative), which is the form `docs/telemetry-runbook.md:1198` and two +> of the three `transaction-overview.json` failure panels already use. + +Every dashboard exposes a `$node` template variable bound to +`service_instance_id`; see the §5.1.1 note on why `service_instance_id` must be +set for metric panels to split per node. + +### 5.8.4 TraceQL Query Examples + +Common queries for xrpld traces. Every span name and attribute below is one +that the code actually emits — check against the `*SpanNames.h` constants +before adding more. ``` # Find all traces for a specific transaction hash @@ -218,57 +536,163 @@ Common queries for xrpld traces: # Find consensus rounds taking >5 seconds {resource.service.name="xrpld" && name="consensus.round"} | duration > 5s -# Find failed transactions with error details -{resource.service.name="xrpld" && name="tx.validate" && status.code=error} +# Find failed transaction processing +{resource.service.name="xrpld" && name="tx.process" && span.ter_result!="tesSUCCESS"} -# Find transactions relayed to many peers -{resource.service.name="xrpld" && name="tx.relay"} | span.relay_count > 10 +# Find failed apply-pipeline stages (preflight / preclaim / transactor) +{resource.service.name="xrpld" && name=~"tx\\.(preflight|preclaim|transactor)" && status=error} + +# Find transactions that arrived from a peer rather than a local client. +# The `local` attribute lives on tx.process, NOT on tx.receive (see the note +# below). +{resource.service.name="xrpld" && name="tx.process" && span.local=false} # Compare latency across nodes {resource.service.name="xrpld" && name="rpc.command.account_info"} | avg(duration) by (resource.service.instance.id) ``` -### 5.8.6 Correlation with PerfLog +> Queries in earlier drafts used `tx.validate`, `tx.relay` and +> `span.relay_count`. None of the three exists: signature/format validation +> ships as `tx.preflight`/`tx.preclaim`, and no relay span or relay-count +> attribute was ever built. See [02 §2.3.2](./02-design-decisions.md). -To correlate OpenTelemetry traces with existing PerfLog data: +> **TraceQL silently returns nothing for an absent attribute.** Unlike PromQL, +> where a missing label compares equal to `""`, a TraceQL attribute predicate +> matches only spans that actually carry the attribute — including negated +> forms such as `!=` and `=~".*"`. So filtering on the wrong span name yields +> zero rows with no error. `local` has exactly one set-site, +> `NetworkOPs.cpp:1417`, and it is on **`tx.process`**: an earlier draft paired +> it with `name="tx.receive"`, which can never match. Check the attribute's +> owning span in +> [09 §1.2](./09-data-collection-reference.md) before combining a `name=` and a +> `span.` predicate. -**Step 1: Configure Loki to ingest PerfLog** +### 5.8.5 Correlation with Logs -Configure a Promtail scrape job (`promtail-config.yaml`) that tails `/var/log/rippled/perf*.log`, parses each JSON line, and promotes `trace_id`, `ledger_seq`, and `tx_hash` to Loki labels. +Log↔trace correlation is **implemented** (Phase 8) and needs no Promtail, +Fluentd or PerfLog change. Two pieces: -**Step 2: Add trace_id to PerfLog entries** +1. **The node stamps the IDs.** The journal sink `Logs::format()` + (`src/libxrpl/basics/Log.cpp:304-338`, guarded by `XRPL_ENABLE_TELEMETRY`) + reads the thread-local OTel context and, when a valid span is active, + prefixes the message with `trace_id=<32 hex> span_id=<16 hex>`. It reads + the context value directly rather than calling `GetSpan()` to avoid a heap + allocation on the (common) no-span path. This is the ordinary `debug.log` + stream — PerfLog is not involved, and the `setTraceId` hook described in + earlier drafts was never built. +2. **The collector ingests them.** The `filelog` receiver tails + `/var/log/xrpld/*/debug.log` and its `regex_parser` lifts `trace_id` and + `span_id` as optional capture groups (§5.5.1). `resource/logs` applies an + `upsert` of `service.name=xrpld`, which Loki promotes to the stream label + `service_name`, so the canonical selector is **`{service_name="xrpld"}`**. + Logs land in Loki via `otlphttp/loki`. -Modify PerfLog so its JSON output includes a `trace_id` field whenever a valid span is active: fetch the current span from the OpenTelemetry runtime context, and if its context is valid, render the trace ID as a 32-character lowercase hex string into the log entry. +> **Known issue — the collector's `job` upsert is ineffective for stream +> selection.** `resource/logs` also applies an `upsert` of a `job=xrpld` attribute +> (`otel-collector-config.yaml:62-70`) with the stated intent that operators +> could paste `{job="xrpld"}`. That does not work. On OTLP ingest Loki promotes +> only an **allow-listed** set of resource attributes to indexed stream labels +> (`service.name`, `service.namespace`, `service.instance.id`, +> `deployment.environment`, the `k8s.*`/`cloud.*` keys); `job` is not on that +> list, and this repo ships no Loki config override — `docker-compose.yml:75` +> starts Loki with the image's built-in `/etc/loki/local-config.yaml`. `job` +> therefore lands in **structured metadata**, which cannot appear in a stream +> selector, so `{job="xrpld"}` returns an empty result rather than an error. +> Corroboration in-repo: `docs/telemetry-runbook.md:2533` states the same +> ("`service_name="xrpld"` (not `job="xrpld"`)"), and **all 38 Loki queries** in +> the shipped dashboards (35 panel targets + 3 Loki-backed template variables) +> select on `service_name` — **zero** use `job`. Either drop the `job` +> upsert or add `job` to Loki's `distributor.otlp_config.resource_attributes` +> allow-list via a mounted Loki config; until then, use `service_name`. -**Step 3: Configure Grafana trace-to-logs link** +Grafana then links the two directions: the Tempo datasource's `tracesToLogs` +(`filterByTraceID: true`, `tags: []`) jumps trace → logs, and `loki.yaml`'s +derived fields jump log → trace. -In the Tempo datasource, set the `tracesToLogs` derived field to link to Loki on the `trace_id` and `tx_hash` tags, with `filterByTraceID: true`. - -### 5.8.7 Correlation with Insight/OTel System Metrics +### 5.8.6 Correlation with Insight/OTel System Metrics To correlate traces with Beast Insight system metrics: **Step 1: Export Insight metrics to Prometheus** Beast Insight metrics are exported natively via OTLP to the OTel Collector, -which exposes them on the Prometheus endpoint alongside spanmetrics. Configure -the `[insight]` section of `xrpld.cfg` with `server=otel`, -`endpoint=http://localhost:4318/v1/metrics`, and `prefix=xrpld`; no separate -StatsD exporter or Prometheus scrape job is needed when using `server=otel`. +which exposes them on its Prometheus endpoint (`:8889`) alongside spanmetrics. +Set `server=otel` in the `[insight]` section of `xrpld.cfg`; no separate StatsD +exporter or Prometheus scrape job is needed. -**Step 2: Add exemplars to metrics** +`makeCollectorManager()` (`src/xrpld/app/main/CollectorManager.cpp`) reads these +`[insight]` keys: -The OpenTelemetry SDK automatically adds exemplars (trace IDs) to metrics when using the Prometheus exporter, linking metric spikes to specific traces. +| Key | Read at | Effect when `server=otel` | +| --------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------- | +| `server` | `:35` | **Live.** `statsd` \| `otel` \| anything else. Selects the collector implementation. | +| `address` | `:39` | StatsD only — the UDP endpoint. | +| `prefix` | `:41`, `:53` | **Inert.** Stored on the OTel collector but `formatName()` prepends nothing (`OTelCollector.cpp:855-866`); only StatsD applies it. | +| `endpoint` | `:50` | **Inert.** Logged for diagnostics (`OTelCollector.cpp:730`), then unused. | +| `service_instance_id` | `:58` | **Inert.** `(void)`-discarded (`OTelCollector.cpp:722`). | +| `service_name` | `:64` | **Inert.** `(void)`-discarded (`OTelCollector.cpp:723`). | -**Step 3: Configure Grafana metric-to-trace link** +> **Where the identity and endpoint actually come from.** `OTelCollector` +> deliberately does **not** own a pipeline: it fetches the Meter from the +> **global** MeterProvider that `Telemetry::initMetrics()` published +> (`OTelCollector.cpp:726-745`). So the resource attributes — including +> `service.instance.id`, which every dashboard filters on — and the exporter +> URL both come from the **`[telemetry]`** section, not `[insight]`. The four +> inert keys above are back-compat leftovers from the StatsD-era signature; +> setting them has no effect. Set `[telemetry] service_instance_id` instead +> (§5.1.1). -In the Prometheus datasource, set `exemplarTraceIdDestinations` to map the `trace_id` exemplar to the Tempo datasource. +> **`server=otel` is not the default.** `CollectorManager.cpp:72-75` falls through +> to `NullCollector` for any unrecognised or absent `server` value, so a node +> with no `[insight]` section emits no metrics at all. -**Step 4: Dashboard panel with exemplars** +**Step 2: Correlate metrics to traces** -Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled. +Today this is a **time-range** correlation, not a click-through one: note the +window from the metric panel, then search Tempo over the same window filtered +by `service.instance.id`. -This allows clicking on metric data points to jump directly to the related trace. +> **Exemplars are NOT implemented.** Earlier drafts of this section instructed +> operators to rely on automatic exemplars, set +> `exemplarTraceIdDestinations` on the Prometheus datasource, and enable +> `exemplar: true` on panels. None of that is wired up: the string `exemplar` +> appears **nowhere** in `src/libxrpl/telemetry/`, `src/xrpld/telemetry/`, or +> `docker/telemetry/`. Concretely, three things are missing — +> +> 1. the SDK's exemplar filter is left at its default and no reservoir is +> configured in `Telemetry::initMetrics()` or `MetricsRegistry`; +> 2. the collector's `prometheus` exporter has no exemplar settings; +> 3. `grafana/provisioning/datasources/prometheus.yaml` has no +> `exemplarTraceIdDestinations` block. +> +> Note also that the query used as an example, `rpc_duration_seconds_bucket`, +> does not exist — RPC latency histograms are `span_duration_milliseconds_bucket` +> (spanmetrics, `unit: ms`) and `rpc_method_us` (native). Wiring exemplars end +> to end is genuine open work; until it lands, do not document a click-through +> that operators cannot perform. + +**Step 3: Jump the other way instead** + +Trace → metrics is available now: the Tempo datasource sets +`tracesToMetrics.datasourceUid: prometheus` with a ±1h time shift, so the +span-metric queries it builds resolve against the `span_*` families the +collector's `spanmetrics` connector produces. Trace → logs and log → trace are +both live (§5.8.5). + +> **Known gap — Service Map is configured but inactive.** The Tempo datasource +> declares `serviceMap.datasourceUid: prometheus`, and `tempo.yaml:70-76` +> enables the `service-graphs` metrics-generator processor, but the generator +> has nowhere to write: its `remote_write` block is **commented out** +> (`tempo.yaml:53-56`), and `prometheus.yml:6-9` defines a single scrape job +> against `otel-collector:8889` — it never scrapes or accepts writes from +> Tempo. `traces_service_graph_request_total` and its siblings are therefore +> never stored, so the Service Map / Node Graph tab renders empty. The same gap +> means Tempo's `span-metrics` processor never lands +> `traces_spanmetrics_*` either (§5.5.1) — every span metric the dashboards use +> comes from the collector's connector instead. Closing it needs both halves: +> uncomment `remote_write` in `tempo.yaml` **and** enable +> `--web.enable-remote-write-receiver` on the Prometheus service (or add a +> scrape job for Tempo). --- diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index fe02d914e2..8283fa92d1 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -62,7 +62,7 @@ gantt section Phase 8 Log-Trace Correlation :p8, after p7, 1w - section Phase 9 (Future) + section Phase 9 Internal Metric Gap Fill :p9, after p8, 2.5w section Phase 10 (Future) @@ -93,11 +93,18 @@ gantt ### Exit Criteria -- [ ] OpenTelemetry SDK compiles and links -- [ ] Telemetry can be enabled/disabled via config -- [ ] Basic span creation works -- [ ] No performance regression when disabled -- [ ] Unit tests passing +- [x] OpenTelemetry SDK compiles and links — `conanfile.py:153` requires + `opentelemetry-cpp/1.28.0` when the `telemetry` option is on (`:152`); + `cmake/XrplCore.cmake:91,245` links the umbrella target + `opentelemetry-cpp::opentelemetry-cpp` +- [x] Telemetry can be enabled/disabled via config — `TelemetryConfig.cpp:103` + parses `[telemetry] enabled` (default 0) +- [x] Basic span creation works — `libxrpl/telemetry/SpanGuard.cpp`, covered by + `src/tests/libxrpl/telemetry/SpanGuardScope.cpp` and `SpanGuardFactory.cpp` +- [ ] No performance regression when disabled — `NullTelemetry.cpp` provides the + no-op path, but the <0.1% claim needs the Phase 10 benchmark suite + (`--with-benchmark`), which is not run in CI +- [x] Unit tests passing — 10 GTest files under `src/tests/libxrpl/telemetry/` --- @@ -124,11 +131,20 @@ gantt ### Exit Criteria -- [ ] All RPC commands traced -- [ ] Trace context propagates from HTTP headers -- [ ] WebSocket and HTTP both instrumented -- [ ] <1ms overhead per RPC call -- [ ] Integration tests passing +- [x] All RPC commands traced — `rpc.command.{name}` built from + `rpc_span::prefix::command` (`RpcSpanNames.h:127`), emitted from + `RPCHandler.cpp` +- [ ] Trace context propagates from HTTP headers — **not implemented**. + `TraceContextPropagator.h` only offers `extractFromProtobuf()` / + `injectToProtobuf()`; there is no `traceparent` header reader anywhere in + the tree (`grep -ri traceparent src/ include/` → 0 hits). Cross-node + correlation is carried by the protobuf `TraceContext` field and by + deterministic trace IDs instead. +- [x] WebSocket and HTTP both instrumented — `rpc.http_request` and + `rpc.ws_message` (`RpcSpanNames.h:133-136`) +- [ ] <1ms overhead per RPC call — needs the Phase 10 benchmark suite +- [ ] Integration tests passing — the end-to-end RPC span assertions live in the + Phase 10 harness (`validate_telemetry.py`), not on this branch --- @@ -162,13 +178,19 @@ and [Phase3_taskList.md Task 3.9](./Phase3_taskList.md) for the full implementat ### Exit Criteria -- [ ] Transaction traces span across nodes -- [ ] Trace context in Protocol Buffer messages -- [ ] HashRouter deduplication visible in traces -- [ ] Multi-node integration tests passing -- [ ] <5% overhead on transaction throughput -- [ ] Deterministic trace_id: all nodes produce same trace_id for same transaction -- [ ] Protobuf span_id propagation preserves parent-child ordering when available +- [ ] Transaction traces span across nodes — needs a live multi-node run (Phase 10 harness) +- [x] Trace context in Protocol Buffer messages — `message TraceContext` + (`include/xrpl/proto/xrpl.proto:101`), carried as optional field `1001` on + three message types (`:130`, `:181`, `:229`) +- [x] HashRouter deduplication visible in traces — `suppressed` attribute + (`TxSpanNames.h:71`) +- [ ] Multi-node integration tests passing — Phase 10 harness +- [ ] <5% overhead on transaction throughput — needs the Phase 10 benchmark suite +- [x] Deterministic trace_id: all nodes produce same trace_id for same transaction + — `libxrpl/telemetry/DeterministicIdGenerator.cpp` +- [x] Protobuf span_id propagation preserves parent-child ordering when available + — `TraceContextPropagator.h` `injectToProtobuf()` / `extractFromProtobuf()` + (`trace_state`, field 4, is reserved and deliberately unwired) --- @@ -187,7 +209,7 @@ and [Phase3_taskList.md Task 3.9](./Phase3_taskList.md) for the full implementat | 4.5 | Add consensus-specific attributes | ✅ Done | | 4.6 | Correlate with transaction traces | ✅ Done | | 4.7 | Build verification and testing | ✅ Done | -| 4.8 | Validation span enrichment (ext. dashboard) | ❌ Not done | +| 4.8 | Validation span enrichment (ext. dashboard) | ✅ Done (partial) | **Note**: The original plan doc listed tasks 4.7-4.11 as "Validator list tracing", "Amendment voting tracing", "SHAMap sync tracing", "Multi-validator integration tests", @@ -212,10 +234,18 @@ SHAMap tracing are not implemented. - [x] Phase transitions visible (open, establish, close, accept) - [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b - [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) -- [x] No impact on consensus timing -- [ ] Multi-validator test network validated +- [ ] No impact on consensus timing — **not measured**. No consensus-timing + benchmark has been run on any branch in the chain; the benchmark suite + lives on the Phase 10 branch and does not isolate consensus round time +- [ ] Multi-validator test network validated — needs a live multi-node run; the + 5-node harness lives on the Phase 10 branch, not here - [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept -- [ ] Validation span enrichment (Task 4.8) — not implemented +- [x] Validation span enrichment (Task 4.8) — send span sets `ledger_seq`, + `ledger_hash`, `proposing`, `full_validation` (`RCLConsensus.cpp:975-981`); + receive span sets `ledger_hash`, `full_validation` (`PeerImp.cpp:2573-2574`); + `consensus.accept` sets `quorum` from `app_.getValidators().quorum()` + (`RCLConsensus.cpp:516`). Still open: `proposers_validated` — never + implemented, no attribute of that name exists in the tree. ### Implementation Status — Phase 4a Complete @@ -285,7 +315,7 @@ with `TraceCategory::Consensus` gating. No macros used — all tracing via direc - [x] Strategy switchable via config (`deterministic` / `attribute`) - [x] Consecutive rounds linked via follows-from spans - [x] Build passes with telemetry ON and OFF -- [x] No impact on consensus timing +- [ ] No impact on consensus timing — **not measured** (see §6.5 Exit Criteria) See [Phase4_taskList.md](./Phase4_taskList.md) for full task details. @@ -376,23 +406,35 @@ The `StatsDMeterImpl` in `StatsDCollector.cpp` sends metrics with `|m` suffix, w ### New Grafana Dashboards -**Node Health** (`statsd-node-health.json`, uid: `xrpld-statsd-node-health`): +**Node Health** (`node-health.json`, uid: `node-health`): - Validated/Published Ledger Age, Operating Mode Duration/Transitions, I/O Latency, Job Queue Depth, Ledger Fetch Rate, Ledger History Mismatches, Key Jobs Execution/Dequeue Time, FullBelowCache Size/Hit Rate, Ledger Publish Gap, State Duration Rate, All Jobs Detail -**Network Traffic** (`statsd-network-traffic.json`, uid: `xrpld-statsd-network`): +**Network Traffic** (`network-traffic.json`, uid: `network-traffic`): - Active Inbound/Outbound Peers, Peer Disconnects, Total Bytes/Messages In/Out, Transaction/Proposal/Validation Traffic, Top Traffic Categories, Duplicate Traffic, All Traffic Categories Detail -**RPC & Pathfinding (StatsD)** (`statsd-rpc-pathfinding.json`, uid: `xrpld-statsd-rpc`): +**RPC & Pathfinding** (`rpc-pathfinding.json`, uid: `rpc-pathfinding`): - RPC Request Rate, Response Time p95/p50, Response Size p95/p50, Pathfinding Fast/Full Duration, Resource Warnings/Drops, Response Time Heatmap ### Exit Criteria -- [ ] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=ledgermaster_validated_ledger_age`) -- [ ] All 3 new Grafana dashboards load without errors +- [x] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=ledgermaster_validated_ledger_age`) + — superseded by Phase 7: the same metric names now arrive over OTLP + (`server=otel`) and the StatsD receiver has been removed from the collector +- [x] All 3 new Grafana dashboards load without errors — shipped as + `node-health.json`, `network-traffic.json`, `rpc-pathfinding.json`, + uids `node-health` / `network-traffic` / `rpc-pathfinding`. These three + were renamed in **two** steps: `statsd-*.json` → `system-*.json` + (`2f7064ace6`), then `system-*.json` → bare (`2c590a47c5`). An + `xrpld-statsd-*` form **never existed** in any commit, and `25868f2740` + did not touch these three — it de-prefixed a different set + (`xrpld-fee-market`, `xrpld-job-queue`, `xrpld-peer-quality`, + `xrpld-validator-health` → bare). §6.7 above now carries the shipped names. - [ ] Integration test verifies at least core StatsD metrics (ledger age, peer counts, RPC requests) + — the metric assertions live in the Phase 10 harness + (`expected_metrics.json`), not on this branch - [ ] ~~Meter metrics (`warn`, `drop`) flow correctly after `|m` → `|c` fix~~ — DEFERRED (breaking change, tracked separately; resolved by Phase 7's OTel Counter mapping) --- @@ -568,12 +610,20 @@ See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. ### Exit Criteria - [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) -- [ ] `server=otel` is the default in development docker-compose -- [ ] `server=statsd` still works as a fallback -- [ ] Existing Grafana dashboards display data correctly -- [ ] Integration test passes with OTLP-only metrics pipeline -- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) -- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant + — the receiver is gone and `OTelCollector` is wired, but the 255+ figure + needs a live scrape to confirm +- [x] `server=otel` is the default in development docker-compose — + `docker/telemetry/xrpld-telemetry.cfg:112`, + `xrpld-telemetry-mainnet.cfg:121`, `integration-test.sh:380` +- [x] `server=statsd` still works as a fallback — `CollectorManager.cpp:37` + still branches on `server == "statsd"` alongside `"otel"` (`:46`) +- [ ] Existing Grafana dashboards display data correctly — needs a live stack +- [ ] Integration test passes with OTLP-only metrics pipeline — Phase 10 harness +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) — needs + the Phase 10 benchmark suite +- [x] Deferred Task 6.1 (`|m` wire format) no longer relevant — `OTelMeterImpl` + (`OTelCollector.cpp:308`) maps meters onto an OTel counter, so the + non-standard `|m` wire type is never emitted on the `server=otel` path --- @@ -675,19 +725,37 @@ flowchart LR ### Exit Criteria -- [ ] Log lines within active spans contain `trace_id= span_id=` -- [ ] Log lines outside spans have no trace context (no empty fields) -- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver -- [ ] Grafana Tempo → Loki one-click correlation works -- [ ] Grafana Loki → Tempo reverse lookup works via derived field -- [ ] Integration test verifies trace_id presence in logs -- [ ] No performance regression from trace_id injection (< 0.1% overhead) +- [x] Log lines within active spans contain `trace_id= span_id=` — + `Log.cpp:304-338`, guarded by `#ifdef XRPL_ENABLE_TELEMETRY` +- [x] Log lines outside spans have no trace context (no empty fields) — the + block reads the thread-local span key and appends nothing when it is + absent or the context is invalid (`Log.cpp:310-318`) +- [x] Loki ingests xrpld logs via OTel Collector filelog receiver — + `otel-collector-config.yaml:38` (`filelog`); `loki` service in + `docker-compose.yml:71` +- [x] Grafana Tempo → Loki one-click correlation works — + `provisioning/datasources/tempo.yaml:32` (`tracesToLogs`) +- [x] Grafana Loki → Tempo reverse lookup works via derived field — + `provisioning/datasources/loki.yaml:16` (`derivedFields`) +- [ ] Integration test verifies trace_id presence in logs — implemented in the + Phase 10 harness, but CI runs it with `--skip-loki`, so it is not gated +- [ ] No performance regression from trace_id injection (< 0.1% overhead) — + needs the Phase 10 benchmark suite --- -## 6.8.2 Phase 9: Internal Metric Instrumentation Gap Fill (Weeks 14-15) — Future Enhancement +## 6.8.2 Phase 9: Internal Metric Instrumentation Gap Fill (Weeks 14-15) -> **Status**: Planned, not yet implemented. +> **Status**: Complete. Merged on `pratik/otel-phase9-metric-gap-fill`. Shipped +> artefacts: `src/xrpld/telemetry/MetricsRegistry.{h,cpp}` (~41 KB + ~71 KB), +> `src/xrpld/telemetry/MetricMacros.h`, `include/xrpl/nodestore/WriteStats.h`, +> `src/xrpld/app/ledger/AcquireStats.h`, +> `include/xrpl/telemetry/GetObjectMetricNames.h`, 10 GTest files under +> `src/tests/libxrpl/telemetry/`, 4 new Grafana dashboards, provisioned Grafana +> alerting (13 rules), and the Phase 9 sections of +> `09-data-collection-reference.md` and `docs/telemetry-runbook.md`. +> Tasks 9.14-9.17 remain open by design — see +> [Phase9_taskList.md](./Phase9_taskList.md). ### Motivation @@ -700,50 +768,64 @@ Hybrid approach — two instrumentation strategies based on proximity to existin ```mermaid flowchart TB subgraph xrpld["xrpld process"] - subgraph existing["Existing beast::insight registrations"] - NS["NodeStore I/O
(Database.cpp)"] + subgraph newreg["New OTel MetricsRegistry (all Phase 9 metrics)"] + NS["NodeStore I/O
async gauge
nodestore_state"] + CR["Cache Hit Rates
async gauge"] + TQ["TxQ Metrics
async gauge"] + PL["PerfLog RPC / Job
counters + histograms"] + CO["CountedObjects
async gauge"] + LF["Load Factors
async gauge"] end - subgraph newreg["New OTel MetricsRegistry"] - CR["Cache Hit Rates
(async gauge callbacks)"] - TQ["TxQ Metrics
(async gauge callbacks)"] - PL["PerfLog RPC/Job
(counters + histograms)"] - CO["CountedObjects
(async gauge callbacks)"] - LF["Load Factors
(async gauge callbacks)"] + subgraph existing["Pre-existing beast::insight
(unchanged by Phase 9)"] + IN["Node state, PeerFinder,
overlay traffic, caches"] end end subgraph export["Export Pipelines"] + OS["OTel Metrics SDK
PeriodicMetricReader
10s interval"] BI["beast::insight
OTelCollector (Phase 7)"] - OS["OTel Metrics SDK
PeriodicMetricReader"] end - NS --> BI + NS --> OS CR --> OS TQ --> OS PL --> OS CO --> OS LF --> OS + IN --> BI - BI --> OTLP["OTLP/HTTP :4318
/v1/metrics"] - OS --> OTLP + OS --> OTLP["OTLP/HTTP :4318
/v1/metrics"] + BI --> OTLP - style xrpld fill:#1a2633,color:#ccc,stroke:#4a90d9 - style existing fill:#2a4a6b,color:#fff,stroke:#4a90d9 - style newreg fill:#2a4a6b,color:#fff,stroke:#4a90d9 - style export fill:#1a3320,color:#ccc,stroke:#5cb85c - style NS fill:#4a90d9,color:#fff,stroke:#2a6db5 - style CR fill:#5cb85c,color:#fff,stroke:#3d8b3d - style TQ fill:#5cb85c,color:#fff,stroke:#3d8b3d - style PL fill:#5cb85c,color:#fff,stroke:#3d8b3d - style CO fill:#5cb85c,color:#fff,stroke:#3d8b3d - style LF fill:#5cb85c,color:#fff,stroke:#3d8b3d - style BI fill:#449d44,color:#fff,stroke:#2d6e2d - style OS fill:#449d44,color:#fff,stroke:#2d6e2d - style OTLP fill:#f0ad4e,color:#000,stroke:#c78c2e + style xrpld fill:#1a2633,color:#e8e8e8,stroke:#4a90d9 + style newreg fill:#22405c,color:#ffffff,stroke:#5cb85c + style existing fill:#22405c,color:#ffffff,stroke:#4a90d9 + style export fill:#1a3320,color:#e8e8e8,stroke:#5cb85c + style NS fill:#5cb85c,color:#000000,stroke:#3d8b3d + style CR fill:#5cb85c,color:#000000,stroke:#3d8b3d + style TQ fill:#5cb85c,color:#000000,stroke:#3d8b3d + style PL fill:#5cb85c,color:#000000,stroke:#3d8b3d + style CO fill:#5cb85c,color:#000000,stroke:#3d8b3d + style LF fill:#5cb85c,color:#000000,stroke:#3d8b3d + style IN fill:#4a90d9,color:#000000,stroke:#2a6db5 + style OS fill:#449d44,color:#ffffff,stroke:#2d6e2d + style BI fill:#449d44,color:#ffffff,stroke:#2d6e2d + style OTLP fill:#f0ad4e,color:#000000,stroke:#c78c2e ``` -- **beast::insight extensions** (blue): NodeStore I/O metrics added near existing `Database.cpp` registrations — exported via Phase 7's `OTelCollector`. -- **OTel MetricsRegistry** (green): New centralized class using `ObservableGauge` async callbacks for cache, TxQ, PerfLog, CountedObjects, and load factors — polled at 10s intervals by `PeriodicMetricReader`. +- **OTel MetricsRegistry** (green): the single home for every Phase 9 metric — + `ObservableGauge` async callbacks for NodeStore I/O, cache, TxQ, CountedObjects + and load factors, plus synchronous counters/histograms for PerfLog RPC and job + data. Polled at 10s intervals by `PeriodicMetricReader` + (`MetricsRegistry.cpp:289`, `export_interval_millis = 10000`). +- **NodeStore I/O is _not_ a beast::insight extension.** The original plan + routed it through `Database.cpp` insight registrations; the shipped code + registers a `nodestore_state` observable gauge instead + (`MetricsRegistry.cpp:957-965`) that reads `Database`'s public accessors + (`getFetchTotalCount()`, `getStoreDurationUs()`, …). `Database.cpp` has no + `beast::insight` members at all. +- **beast::insight** (blue) still carries the pre-Phase-9 metric surface via + Phase 7's `OTelCollector`; Phase 9 added nothing to it. ### Third-Party Consumer Context @@ -758,39 +840,99 @@ flowchart TB ### Tasks -| Task | Description | -| ---- | ----------------------------------------- | -| 9.1 | NodeStore I/O metrics | -| 9.2 | Cache hit rate metrics + MetricsRegistry | -| 9.3 | TxQ metrics | -| 9.4 | PerfLog per-RPC metrics | -| 9.5 | PerfLog per-job metrics | -| 9.6 | Counted object instance metrics | -| 9.7 | Fee escalation & load factor metrics | -| 9.7a | push_metrics.py parity gauges | -| 9.8 | New Grafana dashboards (2 new, 2 updated) | -| 9.9 | Update documentation | -| 9.10 | Integration tests | +| Task | Description | Status | +| ---- | ------------------------------------------------------- | ----------------------------- | +| 9.1 | NodeStore I/O metrics (`nodestore_state` gauge) | ✅ Done | +| 9.2 | Cache hit rate metrics + `MetricsRegistry` | ✅ Done | +| 9.3 | TxQ metrics | ✅ Done | +| 9.4 | PerfLog per-RPC metrics | ✅ Done | +| 9.5 | PerfLog per-job metrics (`job_type` + `handler` labels) | ✅ Done | +| 9.6 | Counted object instance metrics | ✅ Done | +| 9.7 | Fee escalation & load factor metrics | ✅ Done | +| 9.7a | push_metrics.py parity gauges | ✅ Done | +| 9.8 | New Grafana dashboards (4 new, 2 updated) | ✅ Done | +| 9.9 | Update documentation | ✅ Done | +| 9.9a | Provisioned Grafana alerting (13 rules / 5 groups) | ✅ Done | +| 9.10 | Integration tests / `MetricsRegistry` unit tests | ✅ Done (unit tests) | +| 9.11 | Validator Health dashboard | ✅ Done | +| 9.12 | Peer Quality dashboard | ✅ Done | +| 9.13 | Ledger Economy row on `node-health` | ✅ Done | +| 9.14 | Overlay traffic accounting defects (documentation only) | 📄 Documented, not fixed | +| 9.15 | Peer keepalive / discovery instrumentation | ❌ Not implemented | +| 9.16 | PeerFinder slot and cache metrics | ❌ Not implemented | +| 9.17 | Peer span coverage (`peer.connect` / `peer.message.*`) | ❌ Not implemented (deferred) | -See [Phase9_taskList.md](./Phase9_taskList.md) for detailed per-task breakdown. +See [Phase9_taskList.md](./Phase9_taskList.md) for detailed per-task breakdown, +including the four open items (9.14-9.17) and why each is blocked. + +### Provisioned Grafana Alerting (Task 9.9a) + +Phase 9 also ships the first provisioned Grafana alerting for the OTel stack — +**13 rules in 5 groups**, 2 contact points, and a two-level notification policy +tree, auto-loaded from the existing `provisioning/` mount (no docker-compose +change): + +| File | Contents | +| ------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `docker/telemetry/grafana/provisioning/alerting/rules.yaml` | 13 rules across `xrpld-consensus` (3), `xrpld-validator` (2), `xrpld-jobqueue` (3), `xrpld-node-state` (2), `xrpld-overlay` (3) | +| `docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml` | `xrpld-default` (Slack) and `xrpld-critical` (Slack + email) | +| `docker/telemetry/grafana/provisioning/alerting/policies.yaml` | Root route → `xrpld-default`; child route `severity = critical` → `xrpld-critical`. Grouped by `alertname` + `service_instance_id`. | + +Shipped rules: `LedgerHistoryMismatch`, `LedgerCloseStalled`, +`ValidatedLedgerStale`, `ValidationsMissed`, `ValidationsNotChecked`, +`JobQueueTxOverflow`, `JobQueueLatencyHigh`, `NodeStoreIOLatencyHigh`, +`NodeStateFlapping`, `NodeNotFull`, `ManifestJobQueueConvoy`, +`ManifestFloodInbound`, `PeerResourceDisconnects`. Three carry +`severity: critical`, ten `severity: warning`. + +Operator documentation for each alert lives in the **Alerting** section of +`docs/telemetry-runbook.md`. The remaining, genuinely-unshipped rules from the +external-dashboard set are scoped in the appendix under **Task 11.9: Remaining +Alert Rules from External Dashboard**. ### Exit Criteria -- [ ] All ~68 new metrics visible in Prometheus via OTLP pipeline -- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK -- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) -- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) -- [ ] Documentation updated with full new metric inventory +- [ ] All ~68 new metrics visible in Prometheus via OTLP pipeline — every + instrument is registered (`MetricsRegistry.cpp`), but end-to-end + visibility is asserted by the Phase 10 harness, not on this branch +- [x] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK — + covered by `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` + (`async_gauges_start_after_start_is_safe`, + `async_gauges_before_start_does_not_break_start`, + `async_gauges_respect_the_compile_time_guard`, `destructor_calls_stop`, + `disabled_construction`, `disabled_start_stop`, `disabled_recording_methods`) +- [x] 4 new Grafana dashboards operational (Fee Market, Job Queue, Validator + Health, Peer Quality) + 2 updated (Node Health, RPC Performance) — all + present under `docker/telemetry/grafana/dashboards/` +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) — needs + the Phase 10 benchmark suite; not measured +- [x] Documentation updated with full new metric inventory — + `09-data-collection-reference.md` §5b "Internal Metric Gap Fill (Phase 9)" + and "Phase 9: OTel SDK-Exported Metrics (MetricsRegistry)"; + `docs/telemetry-runbook.md` § Alerting +- [x] Provisioned Grafana alerting shipped (13 rules / 5 groups, 2 contact + points, nested notification policy) --- -## 6.8.3 Phase 10: Synthetic Workload Generation & Telemetry Validation (Weeks 16-17) — Future Enhancement +## 6.8.3 Phase 10: Synthetic Workload Generation & Telemetry Validation (Weeks 16-17) -> **Status**: Planned, not yet implemented. +> **Status**: Implemented on `pratik/otel-phase10-workload-validation`; **not +> merged into this branch**, so none of its files +> (`docker/telemetry/workload/`, `.github/workflows/telemetry-validation.yml`) +> are present here. The exit criteria below are tracked on that branch. ### Motivation -Before the telemetry stack (Phases 1-9) can be considered production-ready, we need automated proof that all 16 spans, 22 attributes, 300+ metrics, 10 Grafana dashboards, and log-trace correlation work correctly under realistic load. This phase establishes a reusable CI-integrated validation suite and performance benchmark baseline. +Before the telemetry stack (Phases 1-9) can be considered production-ready, we need automated proof that all spans, attributes, metrics, Grafana dashboards, and log-trace correlation work correctly under realistic load. This phase establishes a reusable CI-integrated validation suite and performance benchmark baseline. + +> **Inventory note**: the "16 spans / 22 attributes / 10 dashboards" figures this +> section used to quote are stale. As of this branch there are **15 dashboards on +> disk** (`ls docker/telemetry/grafana/dashboards/*.json`), of which **14** are +> asserted by the Phase 10 harness — `log-derived-insights` is provisioned but +> unasserted. The span and attribute totals are computed dynamically by +> `validate_telemetry.py` from `expected_spans.json`; see +> [Phase10_taskList.md](./Phase10_taskList.md) for the live figures. ### Architecture @@ -849,13 +991,61 @@ flowchart LR See [Phase10_taskList.md](./Phase10_taskList.md) for detailed per-task breakdown. +### CI Deliverable (Task 10.6) + +The Phase 10 CI entry point is `.github/workflows/telemetry-validation.yml` +(348 lines, on the Phase 10 branch). It runs three jobs — `linux-image-tag`, +`build-xrpld`, `validate-telemetry` — and is triggered by `workflow_dispatch` +plus `push` on `pratik/otel-phase*`, `feature/otel-*` and +`feature/telemetry-*`. **There is no cron schedule**, so nothing runs this +workflow on a timer. + +> **Caveat — the `push` trigger's `paths` filter excludes the C++ telemetry +> sources.** The branch filter is only half the trigger; `push` also carries: +> +> ```yaml +> paths: +> - ".github/workflows/telemetry-validation.yml" +> - "docker/telemetry/**" +> - "include/xrpl/basics/Telemetry*.h" +> - "src/xrpld/app/misc/Telemetry*" +> ``` +> +> The last two globs match **nothing** on the Phase 10 branch — neither +> `include/xrpl/basics/Telemetry*.h` nor `src/xrpld/app/misc/Telemetry*` exists +> (0 tracked paths). The telemetry code actually lives in +> `src/xrpld/telemetry/**` (9 files, including `MetricsRegistry.cpp`) and +> `src/libxrpl/telemetry/**` (7 files), and **neither is listed**. Consequence: a +> pure C++ telemetry change — new instrument, renamed metric, changed span +> attribute — never triggers this workflow on push. Only edits under +> `docker/telemetry/**` or to the workflow file itself do. Fix: replace the two +> dead globs with `src/xrpld/telemetry/**`, `src/libxrpl/telemetry/**` and +> `include/xrpl/telemetry/**`. + +> **Caveat — four inert inputs.** The workflow declares five +> `workflow_dispatch` inputs, but only `run_benchmark` changes behaviour. +> `rpc_rate`, `rpc_duration`, `tx_tps` and `tx_duration` are forwarded as +> `--rpc-rate` / `--rpc-duration` / `--tx-tps` / `--tx-duration` to +> `run-full-validation.sh`, which parses them into shell variables and then +> never reads them again: load shape comes entirely from +> `--profile` / `workload-profiles.json` (the orchestrator is invoked with +> `--profile` only). Changing those four inputs has no effect on the generated +> workload. + ### Exit Criteria -- [ ] 5-node validator cluster starts and reaches consensus in docker-compose -- [ ] Validation suite confirms all 16 spans, 22 attributes, 300+ metrics -- [ ] All 10 Grafana dashboards render data (no empty panels) +- [ ] 5-node validator cluster starts and reaches consensus — note that + `docker-compose.workload.yaml` contains only the observability backend + (collector, Tempo, Prometheus, Loki, Grafana); the 5 validators are native + `xrpld` processes started by `run-full-validation.sh` (`NUM_NODES=5`) +- [ ] Validation suite confirms the full span / attribute / metric inventory + (counts computed dynamically from `expected_spans.json` and + `expected_metrics.json`) +- [ ] All 14 harness-asserted Grafana dashboards render data (15 on disk; + `log-derived-insights` is provisioned but unasserted) - [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead - [ ] CI workflow runs validation on telemetry branch changes + (`.github/workflows/telemetry-validation.yml`) --- @@ -1110,7 +1300,8 @@ flowchart TB - ~~Validator list and manifest tracing~~ — descoped - ~~Amendment voting tracing~~ — descoped - ~~SHAMap sync tracing~~ — descoped -- Full end-to-end traces (client → RPC → TX → consensus → ledger) — partial (tx-consensus correlation not yet done) +- Full end-to-end traces (client → RPC → TX → consensus → ledger) — tx-consensus + correlation shipped as `tx.included` events in `doAccept` (Task 4.6) **Code Changes**: ~100 lines across 3 consensus files @@ -1151,13 +1342,13 @@ Clear, measurable criteria for each phase. ### 6.12.1 Phase 1: Core Infrastructure -| Criterion | Measurement | Target | -| --------------- | ---------------------------------------------------------- | ---------------------------- | -| SDK Integration | `cmake --build` succeeds with `-DXRPL_ENABLE_TELEMETRY=ON` | ✅ Compiles | -| Runtime Toggle | `enabled=0` produces zero overhead | <0.1% CPU difference | -| Span Creation | Unit test creates and exports span | Span appears in Tempo | -| Configuration | All config options parsed correctly | Config validation tests pass | -| Documentation | Developer guide exists | PR approved | +| Criterion | Measurement | Target | +| --------------- | ---------------------------------------------- | ---------------------------- | +| SDK Integration | `cmake --build` succeeds with `-Dtelemetry=ON` | ✅ Compiles | +| Runtime Toggle | `enabled=0` produces zero overhead | <0.1% CPU difference | +| Span Creation | Unit test creates and exports span | Span appears in Tempo | +| Configuration | All config options parsed correctly | Config validation tests pass | +| Documentation | Developer guide exists | PR approved | **Definition of Done**: All criteria met, PR merged, no regressions in CI. @@ -1215,19 +1406,19 @@ Clear, measurable criteria for each phase. ### 6.12.6 Success Metrics Summary -| Phase | Primary Metric | Secondary Metric | Deadline | Status | -| -------- | ------------------------------------------------------------------ | --------------------------- | -------------- | ------------------ | -| Phase 1 | SDK compiles and runs | Zero overhead when disabled | End of Week 2 | Active | -| Phase 2 | 100% RPC coverage | <1ms latency overhead | End of Week 4 | Active | -| Phase 3 | Cross-node traces work | <5% throughput impact | End of Week 6 | Active | -| Phase 4 | Consensus fully traced | No consensus timing impact | End of Week 8 | Active | -| Phase 5 | Production deployment | Operators trained | End of Week 9 | Active | -| Phase 6 | StatsD metrics in Prometheus | 3 dashboards operational | End of Week 10 | Active | -| Phase 7 | All metrics via OTLP | No StatsD dependency | End of Week 12 | Active | -| Phase 8 | trace_id in logs + Loki | Tempo↔Loki correlation | End of Week 13 | Active | -| Phase 9 | 68+ new internal metrics in Prom | 2 new dashboards | End of Week 15 | Future Enhancement | -| Phase 10 | Full telemetry stack validated; OTel-sourced regression gate in CI | < 3% CPU overhead proven | End of Week 17 | Future Enhancement | -| Phase 11 | Third-party metrics via receiver | 4 new dashboards + alerting | End of Week 20 | Future Enhancement | +| Phase | Primary Metric | Secondary Metric | Deadline | Status | +| -------- | ------------------------------------------------------------------ | --------------------------------------------- | -------------- | ------------------ | +| Phase 1 | SDK compiles and runs | Zero overhead when disabled | End of Week 2 | Active | +| Phase 2 | 100% RPC coverage | <1ms latency overhead | End of Week 4 | Active | +| Phase 3 | Cross-node traces work | <5% throughput impact | End of Week 6 | Active | +| Phase 4 | Consensus fully traced | No consensus timing impact | End of Week 8 | Active | +| Phase 5 | Production deployment | Operators trained | End of Week 9 | Active | +| Phase 6 | StatsD metrics in Prometheus | 3 dashboards operational | End of Week 10 | Active | +| Phase 7 | All metrics via OTLP | No StatsD dependency | End of Week 12 | Active | +| Phase 8 | trace_id in logs + Loki | Tempo↔Loki correlation | End of Week 13 | Active | +| Phase 9 | 68+ new internal metrics in Prom | 4 new dashboards + 13 provisioned alert rules | End of Week 15 | Complete | +| Phase 10 | Full telemetry stack validated; OTel-sourced regression gate in CI | < 3% CPU overhead proven | End of Week 17 | On Phase 10 branch | +| Phase 11 | Third-party metrics via receiver | 4 new dashboards + 14 remaining alert rules | End of Week 20 | Not started | --- @@ -1322,7 +1513,6 @@ flowchart TB > **Date**: 2026-03-30 > **Status**: Draft > **Source**: [realgrapedrop/xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard) -> **Jira Epic**: RIPD-5060 ### Summary @@ -1351,7 +1541,7 @@ Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from | Upgrade Awareness | `peers_higher_version_pct`, `upgrade_recommended` | 2 | | Storage / Other | `ledger_nudb_bytes`, `jq_trans_overflow_total`, `initial_sync_duration_seconds` | 3 | -#### Alert Rules (18 total, from external dashboard) +#### Alert Rules (18 in the external dashboard; 4 addressed by Phase 9 — 2 fully, 2 partially) | Group | Count | Rules | | ----------- | ----- | ----------------------------------------------------------------------------------------------------------------------- | @@ -1359,6 +1549,12 @@ Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from | Network | 3 | Peer drop >10%/30%, P90 latency + disconnect correlation | | Performance | 7 | CPU >80%, memory >90%, disk >85%, job queue overflow, upgrade recommended, tx rate drop, stale ledger | +> Phase 9 ships **13 provisioned rules in 5 groups** against xrpld's own metric +> surface; 4 of them address external rules — **fully** for unhealthy state and +> job queue overflow, only **partially** for IO latency and stale ledger (looser +> thresholds and longer windows; see the coverage table under Task 11.9). The 14 +> genuinely-remaining rules are scoped under Task 11.9 below. + --- ### Branch-to-Change Mapping @@ -1371,20 +1567,37 @@ Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from Add node-level health context to every `rpc.command.*` span so operators can correlate RPC behavior with node state. -New span attributes on `rpc.command.*`: +> **Status: NOT IMPLEMENTED as span attributes.** Neither key was ever added to +> a span. The dotted `xrpl.*` **span-attribute** namespace was dropped in favour +> of bare/underscore keys (`9e27120a15`), and these two were never re-added under +> any name. Falsifiable check: `grep -rn 'seg::xrpl' src/ include/` → exactly **2** +> hits, both in `include/xrpl/telemetry/SpanNames.h:117-118` +> (`attr::networkId` / `attr::networkType`, i.e. `xrpl.network.id` and +> `xrpl.network.type`), and both are **resource** attributes set on the OTel +> resource at startup, not span attributes. (Do not use +> `grep 'makeStr("xrpl\.'` as evidence — the keys were always composed with +> `join(seg::…)`, never that literal, so it has returned 0 hits since day one and +> proves nothing.) +> The **values** are exported instead as `MetricsRegistry` metric label values: +> `server_info{metric="server_state"}` (`MetricsRegistry.cpp:1014`) and +> `validator_health{metric="amendment_blocked"}` (`MetricsRegistry.cpp:1216`). +> Correlating an RPC with node state therefore requires a metric join, not a +> span filter. Kept here as an open item. -| Attribute | Type | Source | Value Example | -| ----------------------------- | ------ | ------------------------------------ | --------------------- | -| `xrpl.node.amendment_blocked` | bool | `app_.getOPs().isAmendmentBlocked()` | `true` | -| `xrpl.node.server_state` | string | `app_.getOPs().strOperatingMode()` | `"full"`, `"syncing"` | +Proposed (never built) span attributes on `rpc.command.*`: + +| Attribute (proposed) | Type | Source | Value Example | Status | +| -------------------- | ------ | ------------------------------------ | --------------------- | ---------------------------------------------- | +| `amendment_blocked` | bool | `app_.getOPs().isAmendmentBlocked()` | `true` | ❌ Never implemented — metric label value only | +| `server_state` | string | `app_.getOPs().strOperatingMode()` | `"full"`, `"syncing"` | ❌ Never implemented — metric label value only | **File**: `src/xrpld/rpc/detail/RPCHandler.cpp` (in the `rpc.command.*` span creation block, after existing setAttribute calls) -**Rationale**: RPC is the operator's primary interaction point. When a node is amendment-blocked or degraded, every RPC response is suspect. Tagging spans with this state enables Jaeger queries like `{name=~"rpc.command.*"} | xrpl.node.amendment_blocked = true` to find all RPCs served during a blocked period. +**Rationale**: RPC is the operator's primary interaction point. When a node is amendment-blocked or degraded, every RPC response is suspect. Tagging spans with this state would enable TraceQL queries like `{name=~"rpc.command.*" && span.amendment_blocked = true}` to find all RPCs served during a blocked period. **Exit Criteria**: -- [ ] `rpc.command.server_info` spans carry `xrpl.node.amendment_blocked` and `xrpl.node.server_state` attributes +- [ ] `rpc.command.server_info` spans carry `amendment_blocked` and `server_state` attributes — **open**, never implemented - [ ] No measurable latency impact (attribute values are cached atomics, not computed per-call) --- @@ -1399,18 +1612,27 @@ Add the relaying peer's xrpld version to transaction receive spans to enable ver New span attribute on `tx.receive`: -| Attribute | Type | Source | Value Example | -| ------------------- | ------ | -------------------- | --------------- | -| `xrpl.peer.version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | +| Attribute | Type | Source | Value Example | Defined at | +| -------------- | ------ | -------------------- | --------------- | ------------------ | +| `peer_version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | `TxSpanNames.h:79` | -**File**: `src/xrpld/overlay/detail/PeerImp.cpp` (in the `tx.receive` span block, after existing `xrpl.peer.id` setAttribute) +> The dotted `xrpl.peer.version` form in the original spec was never emitted; the +> live key is the bare `peer_version` (`9e27120a15` dropped the `xrpl.*` +> namespace repo-wide). + +**File**: `src/xrpld/overlay/detail/PeerImp.cpp` (in the `tx.receive` span block, after the existing `peer_id` setAttribute) **Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues during network upgrades. **Exit Criteria**: -- [ ] `tx.receive` spans carry `xrpl.peer.version` attribute with a non-empty version string -- [ ] Attribute is omitted (not empty-string) when `getVersion()` returns empty +- [x] `tx.receive` spans carry `peer_version` attribute with a non-empty version + string — `PeerImp.cpp:1341-1342` sets `tx_span::attr::peerVersion` on the + `txReceiveSpan` created at `:1330` +- [x] Attribute is omitted (not empty-string) when `getVersion()` returns empty — + the call site is guarded: + `if (auto const version = getVersion(); !version.empty())` + (`PeerImp.cpp:1341`), so no attribute is set at all on the empty path --- @@ -1422,26 +1644,37 @@ New span attribute on `tx.receive`: Add ledger hash and validation type to validation spans on both send and receive paths. This enables trace-level agreement analysis — filter by ledger hash to see which validators agreed. -New span attributes on `consensus.validation.send`: +> **Status: SHIPPED**, with one exception noted below. All keys are bare / +> underscore — the dotted `xrpl.*` forms in the original spec were never emitted +> as **span** attributes. Check: `grep -rn 'seg::xrpl' src/ include/` → 2 hits, +> both `SpanNames.h:117-118` resource attributes (`xrpl.network.{id,type}`). -| Attribute | Type | Source | Value Example | -| ----------------------------- | ------ | --------------------------------------- | --------------------------- | -| `xrpl.validation.ledger_hash` | string | Ledger hash from `validate()` call args | `"A1B2C3..."` (64-char hex) | -| `xrpl.validation.full` | bool | Whether this is a full validation | `true` | +Span attributes on `consensus.validation.send` (`RCLConsensus.cpp:975-981`): -New span attributes on `peer.validation.receive`: +| Attribute | Type | Source | Value Example | Defined at | +| ----------------- | ------ | --------------------------------------- | --------------------------- | ----------------- | +| `ledger_hash` | string | Ledger hash from `validate()` call args | `"A1B2C3..."` (64-char hex) | `SpanNames.h:147` | +| `full_validation` | bool | Whether this is a full validation | `true` | `SpanNames.h:148` | +| `ledger_seq` | int64 | `ledger.seq()` | `93110248` | shared consensus | +| `proposing` | bool | `proposing` argument | `true` | shared consensus | -| Attribute | Type | Source | Value Example | -| ---------------------------------- | ------ | ------------------------------------- | --------------------------- | -| `xrpl.peer.validation.ledger_hash` | string | From deserialized STValidation object | `"A1B2C3..."` (64-char hex) | -| `xrpl.peer.validation.full` | bool | From STValidation flags | `true` | +Span attributes on `peer.validation.receive` (`PeerImp.cpp:2573-2574`): -New span attributes on `consensus.accept`: +| Attribute | Type | Source | Value Example | Defined at | +| ----------------- | ------ | ------------------------------------- | --------------------------- | -------------------- | +| `ledger_hash` | string | From deserialized STValidation object | `"A1B2C3..."` (64-char hex) | `PeerSpanNames.h:35` | +| `full_validation` | bool | `val->isFull()` | `true` | `PeerSpanNames.h:34` | -| Attribute | Type | Source | Value Example | -| ------------------------------------ | ----- | ---------------------------------------- | ------------- | -| `xrpl.consensus.validation_quorum` | int64 | `app_.validators().quorum()` | `28` | -| `xrpl.consensus.proposers_validated` | int64 | `result.proposers` from consensus result | `35` | +Span attributes on `consensus.accept`: + +| Attribute | Type | Source | Value Example | Status | +| --------------------- | ----- | ---------------------------------------- | ------------- | ------------------------------------------------------------------- | +| `quorum` | int64 | `app_.getValidators().quorum()` | `28` | ✅ `RCLConsensus.cpp:516`, `ConsensusSpanNames.h:219` | +| `proposers_validated` | int64 | `result.proposers` from consensus result | `35` | ❌ **Never implemented** — no attribute of this name exists in code | + +> `proposers` is already set on `consensus.accept` (`RCLConsensus.cpp:513`), so a +> separate `proposers_validated` key would be a duplicate under a different +> name; that is why it was never added. It stays open only as a naming decision. **Files**: @@ -1452,10 +1685,11 @@ New span attributes on `consensus.accept`: **Exit Criteria**: -- [ ] `consensus.validation.send` spans carry `xrpl.validation.ledger_hash` and `xrpl.validation.full` -- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` -- [ ] `consensus.accept` spans carry `xrpl.consensus.validation_quorum` and `xrpl.consensus.proposers_validated` -- [ ] Ledger hash attributes match between send and receive for the same ledger +- [x] `consensus.validation.send` spans carry `ledger_hash` and `full_validation` — `RCLConsensus.cpp:975-981` +- [x] `peer.validation.receive` spans carry `ledger_hash` and `full_validation` — `PeerImp.cpp:2573-2574` +- [x] `consensus.accept` spans carry `quorum` — `RCLConsensus.cpp:516` +- [ ] `consensus.accept` spans carry `proposers_validated` — **open**, never implemented (see note above) +- [ ] Ledger hash attributes match between send and receive for the same ledger — needs a live multi-node run --- @@ -1650,7 +1884,7 @@ New MetricsRegistry observable gauge for node state duration. | Gauge Name | Label `metric=` | Type | Source | | ---------------- | ------------------------------- | ------ | ------------------------------------------------ | -| `state_tracking` | `state_value` | int64 | 0-7 numeric encoding matching external dashboard | +| `state_tracking` | `state_value` | double | 0-6 numeric encoding matching external dashboard | | | `time_in_current_state_seconds` | double | `now - lastModeChangeTime` | **State value encoding**: @@ -1761,9 +1995,12 @@ Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats. > **Ref**: Adds to existing Phase 9 task list. Depends on Phase 7 gauges/counters. Consumed by Phase 10 (dashboard load checks). -**Task 9.11: Validator Health Dashboard** +**Task 9.11: Validator Health Dashboard** — ✅ shipped -New Grafana dashboard: `validator-health.json` +New Grafana dashboard: `validator-health.json` (uid `validator-health`). The +shipped dashboard has **17 panels** across 3 rows — Validation Agreement, +Validation Rates, Server State & Consensus — i.e. 4 more than the 13 planned +below. | Panel | Type | PromQL | | -------------------------- | ---------- | -------------------------------------------------------- | @@ -1785,7 +2022,7 @@ New Grafana dashboard: `validator-health.json` --- -**Task 9.12: Peer Quality Dashboard** +**Task 9.12: Peer Quality Dashboard** — ✅ shipped (6 panels, uid `peer-quality`) New Grafana dashboard: `peer-quality.json` @@ -1800,9 +2037,10 @@ New Grafana dashboard: `peer-quality.json` --- -**Task 9.13: Ledger Economy Dashboard Panels** +**Task 9.13: Ledger Economy Dashboard Panels** — ✅ shipped -Add a "Ledger Economy" row to the existing `node-health.json` dashboard: +The "Ledger Economy" row is present on `node-health.json` with all 5 +`ledger_economy` panels: | Panel | Type | PromQL | | -------------------- | ---------- | --------------------------------------------- | @@ -1822,18 +2060,23 @@ Add a "Ledger Economy" row to the existing `node-health.json` dashboard: Add checks to `validate_telemetry.py` for all new span attributes and metrics. -**New span attribute checks (~8)**: +**New span attribute checks** — bare/underscore keys; the dotted `xrpl.*` forms +were never emitted: -| Span Name | New Attribute | -| --------------------------- | ------------------------------------ | -| `rpc.command.server_info` | `xrpl.node.amendment_blocked` | -| `rpc.command.server_info` | `xrpl.node.server_state` | -| `tx.receive` | `xrpl.peer.version` | -| `consensus.validation.send` | `xrpl.validation.ledger_hash` | -| `consensus.validation.send` | `xrpl.validation.full` | -| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | -| `consensus.accept` | `xrpl.consensus.validation_quorum` | -| `consensus.accept` | `xrpl.consensus.proposers_validated` | +| Span Name | New Attribute | Emitted? | +| --------------------------- | --------------------- | ---------------------------------------------- | +| `rpc.command.server_info` | `amendment_blocked` | ❌ never implemented — metric label value only | +| `rpc.command.server_info` | `server_state` | ❌ never implemented — metric label value only | +| `tx.receive` | `peer_version` | ✅ `TxSpanNames.h:79` | +| `consensus.validation.send` | `ledger_hash` | ✅ `RCLConsensus.cpp:975-981` | +| `consensus.validation.send` | `full_validation` | ✅ `RCLConsensus.cpp:975-981` | +| `peer.validation.receive` | `ledger_hash` | ✅ `PeerImp.cpp:2573` | +| `peer.validation.receive` | `full_validation` | ✅ `PeerImp.cpp:2574` | +| `consensus.accept` | `quorum` | ✅ `RCLConsensus.cpp:516` | +| `consensus.accept` | `proposers_validated` | ❌ never implemented | + +Only the ✅ rows are checkable; the ❌ rows must not be added to +`expected_spans.json` as required attributes. **New metric existence checks (~13)**: @@ -1868,9 +2111,10 @@ Add checks to `validate_telemetry.py` for all new span attributes and metrics. | `validation_agreement_pct_1h` | in [0, 100] | | `unl_expiry_days` | > 0 (not expired) | | `peer_latency_p90_ms` | > 0 (peers exist) | -| `state_value` | in [0, 7] | +| `state_value` | in [0, 6] | -**Total new checks: ~28** (bringing total from 73 to ~101) +**Total new checks: ~28** — the harness computes its check total dynamically, so +no fixed "N of N" figure is asserted here. --- @@ -1878,50 +2122,89 @@ Add checks to `validate_telemetry.py` for all new span attributes and metrics. > **Ref**: Adds to existing Phase 11 task list. Depends on Phase 7 metrics and Phase 9 dashboards. -**Task 11.9: Alert Rules from External Dashboard** +**Task 11.9: Remaining Alert Rules from External Dashboard** -Port 18 alert rules from the external `xrpl-validator-dashboard` to Grafana alerting provisioning. +> **Ownership correction.** Provisioned Grafana alerting is **not** a Phase 11 +> deliverable and does **not** live at +> `docker/telemetry/grafana/alerting/{alert-rules,contact-points,notification-policies}.yaml` +> — that directory has never existed. It shipped on **Phase 9** (`7cabf91a0d`) +> at `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` +> with **13 rules in 5 groups**, 2 contact points and a nested notification +> policy. See §6.8.2 → "Provisioned Grafana Alerting (Task 9.9a)". -**Critical Group** (8 rules, eval interval 10s): +Of the 18 external-dashboard rules originally listed here, **4 are addressed** by +the Phase 9 set (under different names and against xrpld's own metric surface) — +but only 2 of those 4 are a like-for-like match. The other 2 are **partially +covered**: the Phase 9 rule watches the same failure mode at a materially looser +threshold and a longer `for` window, so the external rule's sensitivity is _not_ +reproduced. -| Rule | Condition | For | -| ------------------- | ------------------------------------------------------- | --- | -| Agreement Below 90% | `validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | -| Not Proposing | `state_tracking{metric="state_value"} < 6` | 10s | -| Unhealthy State | `state_tracking{metric="state_value"} < 4` | 10s | -| Amendment Blocked | `validator_health{metric="amendment_blocked"} == 1` | 1m | -| UNL Expiring | `validator_health{metric="unl_expiry_days"} < 14` | 1h | -| High IO Latency | `histogram_quantile(0.95, ios_latency_bucket) > 50` | 1m | -| High Load Factor | `load_factor_metrics{metric="load_factor"} > 1000` | 1m | -| Peer Count Critical | `server_info{metric="peers"} < 5` | 1m | +| External rule | Addressed by (Phase 9 rule) | Group | Coverage | +| ------------------ | ---------------------------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Unhealthy State | `NodeNotFull` | `xrpld-node-state` | Full | +| High IO Latency | `NodeStoreIOLatencyHigh` (`ios_latency_milliseconds_bucket` p95) | `xrpld-jobqueue` | **Partial** — Phase 9 fires at p95 **> 1000 ms for 10m**; the external rule fires at **> 50 for 1m**. A 20× looser threshold and a 10× longer window | +| Job Queue Overflow | `JobQueueTxOverflow` (`jq_trans_overflow_total`) | `xrpld-jobqueue` | Full | +| Stale Ledger | `ValidatedLedgerStale` (`ledgermaster_validated_ledger_age`) | `xrpld-consensus` | **Partial** — different metric and threshold: Phase 9 uses `ledgermaster_validated_ledger_age > 60` for 5m; the external rule uses `ledger_economy{ledger_age_seconds} > 30` for 1m | -**Network Group** (3 rules, eval interval 10s): +> The two partial rows are **not** closed by Phase 9. Either re-baseline the +> Phase 9 thresholds against the measured evidence, or add the tighter external +> variants alongside them under Task 11.12 — do not treat them as done. -| Rule | Condition | For | -| ------------------------- | ----------------------------------------------------------- | --- | -| Peer Drop >10% | `delta(server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | -| Peer Drop >30% | Same formula, threshold -30 | 30s | -| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | +Phase 9 additionally ships 9 rules with no external counterpart: +`LedgerHistoryMismatch`, `LedgerCloseStalled`, `ValidationsMissed`, +`ValidationsNotChecked`, `JobQueueLatencyHigh`, `NodeStateFlapping`, +`ManifestJobQueueConvoy`, `ManifestFloodInbound`, `PeerResourceDisconnects`. -**Performance Group** (7 rules, eval interval 10s): +**Remaining open work for Phase 11 — 14 rules that genuinely do not exist yet:** -| Rule | Condition | For | -| ------------------- | ------------------------------------------------------ | --- | -| CPU High | Per-core CPU > 80% | 2m | -| Memory Critical | Memory usage > 90% | 1m | -| Disk Warning | Disk usage > 85% | 2m | -| Job Queue Overflow | `rate(jq_trans_overflow_total[5m]) > 0` | 1m | -| Upgrade Recommended | `peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | -| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | -| Stale Ledger | `ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | +**Critical** (6 remaining): -**Notification channels**: Template configs for Email/SMTP, Discord, Slack, PagerDuty. +| Rule | Condition | Blocked on | +| ------------------- | ------------------------------------------------------- | ---------- | +| Agreement Below 90% | `validation_agreement{metric="agreement_pct_24h"} < 90` | — | +| Not Proposing | `state_tracking{metric="state_value"} < 6` | — | +| Amendment Blocked | `validator_health{metric="amendment_blocked"} == 1` | — | +| UNL Expiring | `validator_health{metric="unl_expiry_days"} < 14` | — | +| High Load Factor | `load_factor_metrics{metric="load_factor"} > 1000` | — | +| Peer Count Critical | `server_info{metric="peers"} < 5` | — | -**Files**: +> **"Not Proposing" is unblocked.** The `state_tracking` gauge **is** +> implemented: `MetricsRegistry::registerStateTrackingGauge()` +> (`MetricsRegistry.cpp:1461-1510`) creates +> `CreateDoubleObservableGauge("state_tracking", …)` at `:1466` and observes +> `state_value` (`:1497`) and `time_in_current_state_seconds` (`:1502`). It is +> already consumed by `validator-health.json:765,971` and +> `ledger-data-sync.json:869`, and documented in +> [09-data-collection-reference.md](./09-data-collection-reference.md) § +> "State Tracking". Only **3** of the 14 remaining rules are blocked on anything — +> CPU High, Memory Critical and Disk Warning, all needing `node_exporter`. -- `docker/telemetry/grafana/alerting/alert-rules.yaml` (new or extend existing) -- `docker/telemetry/grafana/alerting/contact-points.yaml` -- `docker/telemetry/grafana/alerting/notification-policies.yaml` +**Network** (3 remaining): + +| Rule | Condition | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Peer Drop >10% | `delta(server_info{metric="peers"}[30s]) / ... * 100 < -10` | +| Peer Drop >30% | Same formula, threshold -30 | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` — partially covered by `PeerResourceDisconnects`, which has no latency term | + +**Performance** (5 remaining): + +| Rule | Condition | Blocked on | +| ------------------- | ------------------------------------------------------ | ---------------------------------------- | +| CPU High | Per-core CPU > 80% | needs `node_exporter` — not in the stack | +| Memory Critical | Memory usage > 90% | needs `node_exporter` | +| Disk Warning | Disk usage > 85% | needs `node_exporter` | +| Upgrade Recommended | `peer_quality{metric="peers_higher_version_pct"} > 60` | — | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | — | + +**Notification channels**: the shipped `contactpoints.yaml` provides Slack and +email. Templates for Discord and PagerDuty remain open. + +**Files** (extend the Phase 9 location; do **not** create a second `alerting/` tree): + +- `docker/telemetry/grafana/provisioning/alerting/rules.yaml` (add groups) +- `docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml` (add receivers) +- `docker/telemetry/grafana/provisioning/alerting/policies.yaml` (add routes) --- @@ -1939,18 +2222,44 @@ Document the external dashboard's "fast path" pattern as a future optimization f ### Documentation Updates -#### `docs/telemetry-runbook.md` (on Phase 9 branch) +#### `docs/telemetry-runbook.md` (on Phase 9 branch) — partially done -Add new sections after "Phase 9: OTel Metrics Alerting Rules": +- [x] **Alerting** section — shipped; documents all 13 provisioned rules, + thresholds, likely causes, and how to point a contact point at a real + receiver. + Six dashboard reference sections remain unwritten (`fee-market`, `job-queue`, + `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality`, + `validator-health`), plus one operator explainer: -1. **Validator Health Monitoring** — explains agreement tracking, amendment blocked, UNL expiry, with example PromQL queries -2. **Peer Quality Monitoring** — explains P90 latency, insane peers, version awareness -3. **Ledger Economy Monitoring** — explains fee/reserve gauges, transaction rate, ledger age -4. **Validation Agreement Explained** — operator-facing explanation of the reconciliation algorithm (8s grace, 5m late repair), what "missed" means, and when to worry +- [ ] **`validator-health` guide** — explains agreement tracking, amendment blocked, UNL expiry, with example PromQL queries +- [ ] **`peer-quality` guide** — explains P90 latency, insane peers, version awareness +- [ ] **`fee-market` guide** — explains TxQ depth vs capacity, fee escalation levels, load factor breakdown +- [ ] **`job-queue` guide** — explains per-job-type rates, queue wait vs execution time, concurrency limits +- [ ] **`ledger-data-sync` guide** — explains sync state, ledger acquisition, I/O latency +- [ ] **`overlay-traffic-detail` guide** — explains per-category traffic accounting (note the §6 defects that flatline some panels) +- [ ] **Validation Agreement Explained** — operator-facing explanation of the reconciliation algorithm (8s grace, 5m late repair), what "missed" means, and when to worry -#### `OpenTelemetryPlan/09-data-collection-reference.md` (on Phase 9 branch) +> Ledger economy is a **row on `node-health`**, not a dashboard of its own, so it +> falls under that already-documented section rather than the six above. -Add new metric tables in a "Phase 7+: External Dashboard Parity" section covering all 29 new metrics with their gauge names, label values, types, and sources. +> Still open. The runbook itself records the gap at its dashboard reference +> section, and it names **six** dashboards, not four: "Nine dashboards have a +> reference section below. `fee-market`, `job-queue`, `ledger-data-sync`, +> `overlay-traffic-detail`, `peer-quality`, and `validator-health` are +> provisioned but not yet documented here — their panel descriptions carry the +> same six-heading reference format, so open the panel info icon in Grafana until +> a section is written." (15 dashboards on disk − 6 undocumented = 9 documented.) +> So the remaining runbook work is **six** dashboard guides, plus the Validation +> Agreement explainer listed above. + +#### `OpenTelemetryPlan/09-data-collection-reference.md` (on Phase 9 branch) — done + +- [x] "Phase 7+: External Dashboard Parity Metrics" section with gauge names, + label values, types and sources. +- [x] §5b "Internal Metric Gap Fill (Phase 9)" and "Phase 9: OTel SDK-Exported + Metrics (MetricsRegistry)". +- [x] "New Grafana Dashboards (Phase 9)" and "Updated Grafana Dashboards + (Phase 9)" reference tables. --- @@ -1966,11 +2275,13 @@ Phase 6 (StatsD bridge: peerDisconnectsCharges) │ Phase 7 (ValidationTracker + 7 gauges + 7 counters + agreement gauge) │ -Phase 9 (3 dashboards + ledger economy panels + runbook + data-collection-ref) +Phase 9 (4 new dashboards + ledger economy panels + 13 provisioned + alert rules + data-collection-ref; runbook Alerting only) │ -Phase 10 (28 new validation checks in validate_telemetry.py) +Phase 10 (new validation checks in validate_telemetry.py + + .github/workflows/telemetry-validation.yml) │ -Phase 11 (18 alert rules + dual-datasource docs) +Phase 11 (14 remaining alert rules + dual-datasource docs) ``` ### Rebase Strategy diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index 4ebb6028fd..daa7fa9693 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -17,14 +17,33 @@ ### Quick Start with Tempo ```bash -# Start Tempo with OTLP support +# Start Tempo with OTLP support. +# Version pinned to match docker/telemetry/docker-compose.yml:55 — keep the +# two in step, since Tempo config keys change between minor releases. +# +# Only 4317 (OTLP/gRPC) is published: docker/telemetry/tempo.yaml:28-33 +# declares a single distributor receiver, `otlp.protocols.grpc` on +# 0.0.0.0:4317. There is no `http` protocol block, so nothing listens on 4318 +# and publishing it would give you a port that silently refuses connections. +# 3200 is Tempo's HTTP API/query port (tempo.yaml:17-18), not an ingest port. docker run -d --name tempo \ -p 3200:3200 \ -p 4317:4317 \ - -p 4318:4318 \ - grafana/tempo:2.6.1 + grafana/tempo:2.9.4 ``` +> Note that xrpld itself exports OTLP/**HTTP** only (§2.2.1), so it cannot send +> to this container directly — the collector is what bridges HTTP ingest to +> Tempo's gRPC receiver (`otlp/tempo` → `tempo:4317`). A bare Tempo container is +> useful for replaying traces from another OTLP/gRPC producer, not as an xrpld +> endpoint. + +> In practice, prefer the full stack — +> `docker compose -f docker/telemetry/docker-compose.yml up -d` — over a bare +> Tempo container. Most shipped dashboards query Prometheus span metrics, which +> need the collector and Prometheus services too. See +> [05 §5.6](./05-configuration-reference.md). + --- ## 7.2 Production Backends @@ -168,47 +187,100 @@ flowchart TB ### 7.4.2 Sampling Strategy +An earlier version of this section described a three-policy tail sampler (keep +all errors / keep anything >5s / keep 10% of the rest). **No such sampler +exists in this repo.** What ships is below. + ```mermaid flowchart LR - subgraph head["Head Sampling (Node)"] - hs[Node-level head sampling
fixed at 100%
not configurable] + subgraph head["Head Sampling (Node) — fixed"] + hs["ParentBased(TraceIdRatio 1.0)
samplingRatio is static constexpr
no config key exists
100% of spans exported"] end - subgraph tail["Tail Sampling (Collector)"] - ts1[Keep all errors] - ts2[Keep slow >5s] - ts3[Keep 10% rest] + subgraph tail["Tail Sampling (Collector) — opt-in"] + base["Base config:
NO tail_sampling processor
100% of traces stored"] + cloud["grafanacloud overlay only:
one probabilistic policy
sampling_percentage: 0.5"] end head --> tail + base --> final["Stored Traces"] + cloud --> final - ts1 --> final[Final Traces] - ts2 --> final - ts3 --> final - - style head fill:#0d47a1,stroke:#082f6a,color:#fff - style tail fill:#1b5e20,stroke:#0d3d14,color:#fff - style hs fill:#0d47a1,stroke:#082f6a,color:#fff - style ts1 fill:#1b5e20,stroke:#0d3d14,color:#fff - style ts2 fill:#1b5e20,stroke:#0d3d14,color:#fff - style ts3 fill:#1b5e20,stroke:#0d3d14,color:#fff - style final fill:#bf360c,stroke:#8c2809,color:#fff + style head fill:#0d47a1,stroke:#082f6a,color:#ffffff + style tail fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style hs fill:#0d47a1,stroke:#082f6a,color:#ffffff + style base fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style cloud fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style final fill:#bf360c,stroke:#8c2809,color:#ffffff ``` **Reading the diagram:** -- **Head Sampling (Node)**: xrpld pins head sampling at 100% (sample everything) and does not expose a configurable ratio. This is intentional: a per-node ratio would let different nodes make divergent keep/drop decisions for the same distributed trace, producing broken/partial traces. xrpld uses a `ParentBased` sampler so spans inheriting a remote parent honor the upstream decision. Volume reduction is delegated to the collector's tail sampling. -- **Tail Sampling (Collector)**: The second filter -- the collector inspects completed traces and applies rules: keep all errors, keep anything slower than 5 seconds, and keep 10% of the remainder. -- **Arrow head → tail**: All head-sampled traces flow to the collector, where tail sampling further reduces volume while preserving the most valuable data. -- **Final Traces**: The output after both sampling stages; this is what gets stored and queried. The two-stage approach balances cost with debuggability. +- **Head Sampling (Node)** — fixed at 100% and genuinely not configurable: + `Telemetry.h:234` declares `static constexpr double samplingRatio = 1.0;` and + `TelemetryConfig.cpp:139` records that there is nothing to parse. This is + intentional: a per-node ratio would let different nodes make divergent + keep/drop decisions for the same distributed trace, producing broken/partial + traces. The ratio sampler is wrapped in a `ParentBased` sampler so spans + inheriting a remote parent honour the upstream decision. +- **Tail Sampling (Collector)** — the base config + (`docker/telemetry/otel-collector-config.yaml`) has **no** `tail_sampling` + processor, so the local and CI stacks keep 100% of traces. The only shipped + policy lives in `otel-collector-config.grafanacloud.yaml:60-67`, wired into + the **`traces/store`** pipeline (`:259-261`) — the overlay has no pipeline + named `traces`; it splits the trace stream into `traces/metrics` (unsampled, + feeds `spanmetrics`) and `traces/store` (sampled, feeds Tempo and Grafana + Cloud). See [05 §5.5.2](./05-configuration-reference.md) for the full overlay + delta. The policy is a single `probabilistic` at **0.5%**, + `decision_wait: 10s`, `num_traces: 50000`. There are no error or latency + carve-outs. +- **Why 0.5% does not damage the dashboards**: the policy is applied on the + trace-storage branch only. The `spanmetrics` connector runs on a separate + branch that still sees every span, so `span_calls_total` and + `span_duration_milliseconds_*` remain exact. Sampling costs you individual + example traces in Tempo, not metric accuracy. +- **If you want the error/latency policies**: they are a reasonable thing to + add, but they must be written — and `decision_wait` sized so a trace's spans + have all arrived before the policy evaluates it. + +#### Companion guard: `memory_limiter` (recommended, not configured) + +Tail sampling bounds what the collector **stores**; it does not bound what the +collector **buffers**. `tail_sampling` is the opposite of cheap here — it holds +up to `num_traces` (50 000) traces in memory for `decision_wait` before +deciding — and the `spanmetrics` connector keeps a live series cache on top of +that. A production gateway collector should therefore also run a +[`memory_limiter`](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md) +processor as an OOM guard: it applies backpressure (refusing new data with a +retryable error, which the node's `sending_queue` will retry) instead of letting +the process be killed and losing every buffered trace. + +> **Not currently configured anywhere in this repo.** Neither +> `otel-collector-config.yaml` nor +> `otel-collector-config.grafanacloud.yaml` declares a `memory_limiter`, and +> neither compose file sets a container memory limit — so today a traffic spike +> is bounded only by host RAM. This is a recommendation for real deployments, +> recorded here because [05 §5.5.1](./05-configuration-reference.md) lists +> `memory_limiter` among the processors deliberately **absent** from the shipped +> config and that must not be read as "not needed". Placement rules if you add +> it: it must be the **first** processor in every pipeline (ahead of `batch`), +> and `limit_mib` must sit below the container/cgroup limit with headroom for +> the sampling and spanmetrics caches. ### 7.4.3 Data Retention -| Environment | Hot Storage | Warm Storage | Cold Archive | -| ----------- | ----------- | ------------ | ------------ | -| Development | 24 hours | N/A | N/A | -| Staging | 7 days | N/A | N/A | -| Production | 7 days | 30 days | many years | +| Environment | Hot Storage | Warm Storage | Cold Archive | Source | +| --------------------------- | ----------- | ------------ | ------------ | ------------------------------------------------------------ | +| Development (local stack) | **1 hour** | N/A | N/A | `tempo.yaml:40` — `compactor.compaction.block_retention: 1h` | +| Staging (recommendation) | 7 days | N/A | N/A | Not configured in this repo | +| Production (recommendation) | 7 days | 30 days | many years | Not configured in this repo | + +> **The local stack keeps traces for 1 hour, not 24.** `block_retention: 1h` +> is deliberate — it bounds disk for a long-running dev node — but it means a +> trace you found this morning is gone by lunchtime. Raise +> `block_retention` in `docker/telemetry/tempo.yaml` before starting any +> investigation that needs to span a working day. The staging and production +> rows are recommendations only; nothing in this repo provisions them. --- @@ -224,56 +296,106 @@ flowchart LR --- -## 7.6 Grafana Dashboard Examples +## 7.6 Grafana Dashboards and Alerts -Pre-built dashboards for xrpld observability. +> **Superseded.** This section was written in Phase 1a, before any dashboard +> shipped, and described three hypothetical boards (`xrpld-consensus-health`, +> `xrpld-node-overview`, `xrpld-unified`) and three TraceQL alert rules in a +> group called `xrpld-tracing-alerts`. **None of those uids or rule names exist +> anywhere in the repo.** What actually ships is 15 dashboards and 13 alert +> rules, and both are Prometheus-first rather than TraceQL-first. The +> authoritative references are: +> +> | For | See | +> | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------- | +> | Dashboard and panel inventory, per-panel queries | [09-data-collection-reference.md](./09-data-collection-reference.md) | +> | Alert catalogue, thresholds and response steps | `docs/telemetry-runbook.md` | +> | Files on disk | `docker/telemetry/grafana/dashboards/*.json`, `docker/telemetry/grafana/provisioning/alerting/rules.yaml` | +> +> The rest of this section records only the facts a reader needs so as not to +> chase the removed names. -### 7.6.1 Consensus Health Dashboard +### 7.6.1 Shipped Dashboards -A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all driven by TraceQL: +15 JSON dashboards are provisioned into Grafana folder `xrpld`. The uids are +bare — there is no `xrpld-` prefix: -- **Consensus Round Duration** (timeseries, ms): average `consensus.round` span duration per node instance, with yellow/red thresholds at 4s/5s. -- **Phase Duration Breakdown** (barchart): average duration of `consensus.phase.*` spans grouped by span name. -- **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. -- **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. +`consensus-health`, `fee-market`, `job-queue`, `ledger-data-sync`, +`ledger-operations`, `log-derived-insights`, `network-traffic`, `node-health`, +`overlay-traffic-detail`, `peer-network`, `peer-quality`, `rpc-pathfinding`, +`rpc-performance`, `transaction-overview`, `validator-health`. -Each panel's TraceQL query is described inline in its bullet above. +> **Panel-count convention** (shared with [05 §5.8.3](./05-configuration-reference.md)): +> counts are of **data panels only**. `type: "row"` collapsible headers are +> excluded because a row carries no query, so a board's raw `panels` array is +> longer than its stated count. -### 7.6.2 Node Overview Dashboard +`consensus-health.json` is a useful calibration for how far this section drifted: +where the removed text described "four TraceQL panels", the real board carries **22 +data panels** in 4 rows (26 `panels` array entries) — 19 Prometheus targets +against `${DS_PROMETHEUS}` and 9 TraceQL targets against `${DS_TEMPO}`. Tempo is +used for trace _drill-down_; the time series come from span metrics. -A Tempo-backed dashboard (uid `xrpld-node-overview`) with four panels: +### 7.6.2 Shipped Alert Rules -- **Active Nodes** (stat): count of distinct `resource.service.instance.id` values seen for the `xrpld` service. -- **Total Transactions (1h)** (stat): count of `tx.receive` spans. -- **Error Rate** (gauge, percent): ratio of `status.code=error` spans to all spans, with yellow/red thresholds at 1%/5%. -- **Service Map** (nodeGraph): Tempo-generated service dependency graph. +`docker/telemetry/grafana/provisioning/alerting/rules.yaml` provisions **13 +rules in 5 groups**, all in folder `xrpld`, all `interval: 1m`, and all +**PromQL** — there are zero TraceQL alert rules. -### 7.6.3 Alert Rules +| Group | Rules | +| ------------------ | --------------------------------------------------------------------------- | +| `xrpld-consensus` | `LedgerHistoryMismatch`, `LedgerCloseStalled`, `ValidatedLedgerStale` | +| `xrpld-validator` | `ValidationsMissed`, `ValidationsNotChecked` | +| `xrpld-jobqueue` | `JobQueueTxOverflow`, `JobQueueLatencyHigh`, `NodeStoreIOLatencyHigh` | +| `xrpld-node-state` | `NodeStateFlapping`, `NodeNotFull` | +| `xrpld-overlay` | `ManifestJobQueueConvoy`, `ManifestFloodInbound`, `PeerResourceDisconnects` | -Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts`, evaluated every 1m) against the Tempo datasource: +> Two placements are worth noting because they are not what the rule name +> suggests. `ValidatedLedgerStale` is grouped under `xrpld-consensus`, not +> `xrpld-validator` — it fires on any node whose validated-ledger sequence stops +> advancing, which is a chain-progress symptom rather than a validator-identity +> one. `NodeStoreIOLatencyHigh` is grouped under `xrpld-jobqueue`, not +> `xrpld-node-state` — slow NodeStore I/O manifests first as job-queue backlog, +> so grouping it there keeps the cause and its effect in one notification. -- **Consensus Round Slow** (warning, `for: 5m`): fires when average `consensus.round` duration exceeds 5s. +Thresholds, measured baselines and response procedures are in the runbook's +alert catalogue, not here. - ``` - {resource.service.name="xrpld" && name="consensus.round"} | avg(duration) > 5s - ``` +### 7.6.3 Writing New Rules: the metric name -- **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: +If you add a span-metric alert, the metric is **`span_calls_total`**. This stack +sets the `spanmetrics` connector's `namespace: "span"` +(`otel-collector-config.yaml:114`); the connector's own default namespace is +**empty**, so without that setting the names would be the bare `calls_total` / +`duration_milliseconds_*`. 7 of the 15 dashboards already query the `span_` +names. Durations are likewise `span_duration_milliseconds_bucket`. - ``` - sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) - / - sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) - > 0.05 - ``` +> **`traces_spanmetrics_*` is a different producer, not the connector's +> default.** That family is emitted by **Tempo's** `metrics_generator` +> `span-metrics` processor (`tempo.yaml:70-76`), which is a separate +> implementation from the collector connector. It does not exist in this stack +> either: the generator's `remote_write` is commented out (`tempo.yaml:53-56`) +> and `prometheus.yml:6-9` scrapes only `otel-collector:8889`, so nothing stores +> what Tempo generates. Do not write a rule against `traces_spanmetrics_*` and +> do not describe `namespace: "span"` as overriding it. -- **Transaction Throughput Drop** (warning, `for: 10m`): fires when the `tx.receive` span rate falls below 10/s. +An RPC error-rate rule, written against the real metric name, looks like this. +Note that error _rate_ is a ratio, so it must divide the error-span rate by the +total-span rate — a bare rate returns calls/second and would fire on traffic +volume alone: - ``` - {resource.service.name="xrpld" && name="tx.receive"} | rate() < 10 - ``` +``` +sum(rate(span_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) +/ +sum(rate(span_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) +> 0.05 +``` -> **Note**: The Consensus Round Slow and Transaction Throughput Drop rules use TraceQL aggregates (`avg(duration)`, `rate()`), which require Tempo 2.3+ with TraceQL metrics enabled. Verify aggregate query support in your Tempo version before provisioning. The RPC Error Rate Spike rule instead queries Prometheus span metrics (collector `spanmetrics` connector), so it needs that connector enabled in the collector pipeline. +> **Prefer PromQL over TraceQL for alerting.** TraceQL aggregates +> (`avg(duration)`, `rate()`) need Tempo 2.3+ with TraceQL metrics enabled, are +> slower, and are distorted by any tail sampling in the path (§7.4.2). Span +> metrics are computed pre-sampling and cost nothing extra to query. That is +> why all 13 shipped rules are PromQL. --- @@ -285,81 +407,73 @@ How to correlate OpenTelemetry traces with existing xrpld observability. ### 7.7.1 Correlation Architecture +There is **one** collection agent, not three. Earlier drafts of this diagram +routed logs through "Promtail/Fluentd" and metrics through a "StatsD Exporter"; +neither exists in this stack. Logs are read by the OTel Collector's own +`filelog` receiver, and `beast::insight` metrics arrive at the same collector +over OTLP (`[insight] server=otel`). The single-agent shape is the point: one +process, one config file, one place to add redaction or tier tagging. + ```mermaid flowchart TB subgraph xrpld["xrpld Node"] - otel[OpenTelemetry
Spans] - perflog[PerfLog
JSON Logs] - insight[Beast Insight
StatsD Metrics] + otel["OpenTelemetry Spans"] + journal["Journal debug.log
trace_id= span_id= prefix
(Log.cpp:304-338)"] + insight["Beast Insight + XRPL_METRIC_*
native OTLP metrics"] end - subgraph collectors["Data Collection"] - otelc[OTel Collector] - promtail[Promtail/Fluentd] - statsd[StatsD Exporter] - end + otelc["OTel Collector
receivers: otlp, filelog
connector: spanmetrics
3 pipelines"] subgraph storage["Storage"] - tempo[(Tempo)] - loki[(Loki)] - prom[(Prometheus)] + tempo[("Tempo")] + loki[("Loki")] + prom[("Prometheus")] end - subgraph grafana["Grafana"] - traces[Trace View] - logs[Log View] - metrics[Metrics View] - corr[Correlation
Panel] - end + dashboards["Grafana
Tempo to Loki via tracesToLogs
Loki to Tempo via derived fields"] - otel -->|OTLP| otelc --> tempo - perflog -->|JSON| promtail --> loki - insight -->|StatsD| statsd --> prom + otel -->|"OTLP/HTTP :4318"| otelc + journal -->|"filelog tails
/var/log/xrpld"| otelc + insight -->|"OTLP/HTTP :4318"| otelc - tempo --> traces - loki --> logs - prom --> metrics + otelc -->|"otlp/tempo"| tempo + otelc -->|"otlphttp/loki"| loki + otelc -->|"prometheus :8889"| prom - traces --> corr - logs --> corr - metrics --> corr + tempo --> dashboards + loki --> dashboards + prom --> dashboards - style xrpld fill:#0d47a1,stroke:#082f6a,color:#fff - style collectors fill:#bf360c,stroke:#8c2809,color:#fff - style storage fill:#1b5e20,stroke:#0d3d14,color:#fff - style grafana fill:#4a148c,stroke:#2e0d57,color:#fff - style otel fill:#0d47a1,stroke:#082f6a,color:#fff - style perflog fill:#0d47a1,stroke:#082f6a,color:#fff - style insight fill:#0d47a1,stroke:#082f6a,color:#fff - style otelc fill:#bf360c,stroke:#8c2809,color:#fff - style promtail fill:#bf360c,stroke:#8c2809,color:#fff - style statsd fill:#bf360c,stroke:#8c2809,color:#fff - style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff - style loki fill:#1b5e20,stroke:#0d3d14,color:#fff - style prom fill:#1b5e20,stroke:#0d3d14,color:#fff - style traces fill:#4a148c,stroke:#2e0d57,color:#fff - style logs fill:#4a148c,stroke:#2e0d57,color:#fff - style metrics fill:#4a148c,stroke:#2e0d57,color:#fff - style corr fill:#4a148c,stroke:#2e0d57,color:#fff + style xrpld fill:#0d47a1,stroke:#082f6a,color:#ffffff + style storage fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style otel fill:#0d47a1,stroke:#082f6a,color:#ffffff + style journal fill:#0d47a1,stroke:#082f6a,color:#ffffff + style insight fill:#0d47a1,stroke:#082f6a,color:#ffffff + style otelc fill:#bf360c,stroke:#8c2809,color:#ffffff + style tempo fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style loki fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style prom fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style dashboards fill:#4a148c,stroke:#2e0d57,color:#ffffff ``` **Reading the diagram:** -- **xrpld Node (three sources)**: A single node emits three independent data streams -- OpenTelemetry spans, PerfLog JSON logs, and Beast Insight StatsD metrics. -- **Data Collection layer**: Each stream has its own collector -- OTel Collector for spans, Promtail/Fluentd for logs, and a StatsD exporter for metrics. They operate independently. -- **Storage layer (Tempo, Loki, Prometheus)**: Each data type lands in a purpose-built store optimized for its query patterns (trace search, log grep, metric aggregation). -- **Grafana Correlation Panel**: The key integration point -- Grafana queries all three stores and links them via shared fields (`trace_id`, `tx_hash`, `ledger_seq`), enabling a single-pane debugging experience. +- **xrpld Node (three signals, one transport)**: spans and metrics both leave over OTLP/HTTP on port 4318. Logs do not leave the node at all — the node just writes `debug.log`, and the journal sink prefixes `trace_id=`/`span_id=` whenever a span is active (`Log.cpp:304-338`). +- **OTel Collector (single agent)**: an `otlp` receiver takes spans and metrics; a `filelog` receiver tails `/var/log/xrpld/*/debug.log` and regex-parses the trace/span IDs out of each line. A `spanmetrics` connector derives RED metrics from the trace stream and feeds them into the metrics pipeline. Three pipelines, three exporters — see [05 §5.5.1](./05-configuration-reference.md). +- **PerfLog is not in this picture.** It still writes `perf.log`, but nothing collects it and it carries no trace ID; the `setTraceId` hook once planned for it was never built ([02 §2.6.5](./02-design-decisions.md)). +- **StatsD is not in this picture either.** It remains a supported `[insight] server=` choice, but selecting it takes metrics _out_ of this pipeline and requires a StatsD receiver you would have to add yourself — the compose file's StatsD port mapping is commented out. +- **Grafana**: correlation is bidirectional and configured in the datasources, not in a bespoke panel — Tempo's `tracesToLogs` (`filterByTraceID: true`) jumps trace → logs, and `loki.yaml`'s derived fields jump log → trace. ### 7.7.2 Correlation Fields -| Source | Field | Link To | Purpose | -| ----------- | ------------------- | ------------- | -------------------------- | -| **Trace** | `trace_id` | Logs | Find log entries for trace | -| **Trace** | `tx_hash` | Logs, Metrics | Find TX-related data | -| **Trace** | `ledger_seq` | Logs | Find ledger-related logs | -| **PerfLog** | `trace_id` (new) | Traces | Jump to trace from log | -| **PerfLog** | `ledger_seq` | Traces | Find consensus trace | -| **Insight** | `exemplar.trace_id` | Traces | Jump from metric spike | +| Source | Field | Link To | Status | +| --------------- | --------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Trace** | `trace_id` | Logs | **Live.** Tempo `tracesToLogs`, `filterByTraceID: true` | +| **Trace** | `tx_hash` | — | Live as a span attribute for search; **not** used as a cross-signal join key (`tags: []`) | +| **Trace** | `ledger_seq` | — | Live as a span attribute; not a join key | +| **Journal log** | `trace_id`, `span_id` | Traces | **Live.** Emitted by `Log.cpp:304-338` into `debug.log`, parsed by the collector's `filelog` receiver, jumped via `loki.yaml` derived fields | +| **PerfLog** | `trace_id` | Traces | **Not implemented.** PerfLog output has no trace ID; the planned `setTraceId` hook was never built. Use the journal log instead | +| **Insight** | `exemplar.trace_id` | Traces | **Not implemented.** No exemplar configuration exists anywhere in the code or collector config — no `exemplar_filter` on the SDK side, no `exemplarTraceIdDestinations` on the Prometheus datasource. Metric spike → trace jumps must be done by time range today | ### 7.7.3 Example: Debugging a Slow Transaction @@ -376,31 +490,82 @@ flowchart TB Trace ID: 4bf92f3577b34da6a3ce929d0e0e4736 ``` -**Step 3: Find related PerfLog entries** +**Step 3: Find related log lines** ``` -# In Grafana Explore with Loki -{job="xrpld"} |= "4bf92f3577b34da6a3ce929d0e0e4736" +# In Grafana Explore with Loki. `service_name` is the promoted stream label; +# do NOT use {job="xrpld"} — see the note below. +{service_name="xrpld"} |= "4bf92f3577b34da6a3ce929d0e0e4736" ``` -**Step 4: Check Insight metrics for the time window** +These are journal (`debug.log`) lines, not PerfLog lines — see §7.7.2. + +> **Known issue — `{job="xrpld"}` does not select anything.** The collector's +> `resource/logs` processor does upsert a `job=xrpld` resource attribute +> (`otel-collector-config.yaml:62-70`), explicitly so that operators could paste +> `{job="xrpld"}`. Loki does not cooperate: on OTLP ingest it promotes only an +> **allow-listed** set of resource attributes to indexed stream labels +> (`service.name`, `service.namespace`, `service.instance.id`, +> `deployment.environment`, `k8s.*`, `cloud.*`), and `job` is not on it. This +> repo mounts no Loki config override (`docker-compose.yml:75` uses the image's +> built-in `local-config.yaml`), so `job` lands in **structured metadata** — +> queryable only with a `|` filter after a selector, never as the selector +> itself. A `{job="xrpld"}` query returns empty with no error, which is why this +> is easy to miss. `docs/telemetry-runbook.md:2533` says the same, and all 38 +> Loki queries in the shipped dashboards (35 panel targets + 3 template +> variables) select on `service_name` — zero use `job`. Fix options: +> drop the ineffective `job` upsert, or mount a Loki config adding `job` to +> `distributor.otlp_config.resource_attributes`. + +**Step 4: Check metrics for the time window** ``` -# In Grafana with Prometheus -rate(xrpld_tx_applied_total[1m]) - @ timestamp_from_trace +# In Grafana with Prometheus. Span-derived RED metrics for the transaction +# pipeline (namespace "span" — see 7.6.3): +sum(rate(span_calls_total{span_name="tx.process"}[1m])) by (service_instance_id) + +# Error share of the same pipeline. Note !~"tesSUCCESS|" — NOT +# !="tesSUCCESS" — so spans that carry no ter_result are excluded: +sum(rate(span_calls_total{span_name="tx.process", ter_result!~"tesSUCCESS|"}[5m])) +/ +sum(rate(span_calls_total{span_name="tx.process"}[5m])) ``` -### 7.7.4 Unified Dashboard Example +> **Why the regex form.** An absent Prometheus label is indistinguishable from +> the empty string, and `tx.process` can end **without** a `ter_result`: the span +> is opened at `NetworkOPs.cpp:1416`, but `processTransaction()` returns early +> when `preProcessTransaction()` rejects the transaction (`:1437-1438`), and +> `doTransactionAsync()` returns early when the transaction is already applying +> (`:1461-1462`) — both before the only setter, at `:1674`. Those series arrive +> with `ter_result=""`, which `!="tesSUCCESS"` happily counts as a failure and +> inflates the ratio. `!~"tesSUCCESS|"` excludes the empty value via the trailing +> `|` alternative. This is the form `docs/telemetry-runbook.md:1198` and the +> `transaction-overview.json` stage-failure panels already use; apply it to any +> new `ter_result` predicate. -A single dashboard (uid `xrpld-unified`) that ties traces, metrics, and logs together across the Tempo, Prometheus, and Loki datasources: +> Earlier drafts used `rate(xrpld_tx_applied_total[1m])` and +> `rate(xrpld_tx_received_total[5m])`. **Neither metric exists** — there is no +> `xrpld_`-prefixed metric family at all, because `OTelCollector::formatName()` +> deliberately prepends no prefix (`OTelCollector.cpp:855-866`); the OTel +> resource `service.name` identifies the service instead. Use the `span_*` +> families above (verified in `transaction-overview.json` and +> `rpc-performance.json`) or the native `XRPL_METRIC_*` instrument names listed +> in [09-data-collection-reference.md](./09-data-collection-reference.md). -- **Transaction Latency (Traces)** (timeseries, Tempo): `histogram_over_time(duration)` of `tx.receive` spans. -- **Transaction Rate (Metrics)** (timeseries, Prometheus): `rate(xrpld_tx_received_total[5m])` per instance, with a data link that opens the matching `tx.receive` traces in Tempo. -- **Recent Logs** (logs, Loki): `{job="xrpld"} | json`. -- **Trace Search** (table, Tempo): all `xrpld` traces, with per-row data links on `traceID` that jump to the trace in Tempo and to the correlated logs in Loki (`{job="xrpld"} |= ""`). +### 7.7.4 Unified Dashboard -The cross-datasource data links are what make this a single-pane debugging view; the correlation fields they rely on are listed in section 7.7.2. +> **Superseded.** No `xrpld-unified` dashboard exists. The single-pane view it +> described is instead delivered by two things that did ship: the +> **`log-derived-insights`** dashboard (31 data panels in 10 rows, all +> Loki-backed — 41 `panels` array entries; see the counting convention in +> §7.6.1) plus the +> bidirectional datasource links (Tempo `tracesToLogs` → Loki, `loki.yaml` +> derived fields → Tempo), which let you cross signals from _any_ board rather +> than only from one dedicated dashboard. +> +> The correlation fields those links rely on — and which of them are actually +> implemented — are in §7.7.2. For the full board inventory see +> [09-data-collection-reference.md](./09-data-collection-reference.md). --- diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md index c75af13d12..3cff77a23f 100644 --- a/OpenTelemetryPlan/08-appendix.md +++ b/OpenTelemetryPlan/08-appendix.md @@ -133,18 +133,25 @@ The full span inventory (names, attributes, parents as instrumented) is in ### Task Lists -| Document | Description | -| -------------------------------------------------------------------------- | --------------------------------------------------- | -| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | -| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | -| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | -| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | -| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | -| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | -| [Phase8_taskList.md](./Phase8_taskList.md) | Log-trace correlation | -| [Phase9_taskList.md](./Phase9_taskList.md) | Internal metric instrumentation gap fill (future) | -| [Phase10_taskList.md](./Phase10_taskList.md) | Synthetic workload generation & validation (future) | -| [Phase11_taskList.md](./Phase11_taskList.md) | Third-party data collection pipelines (future) | +| Document | Description | +| -------------------------------------------------------------------------- | ---------------------------------------------- | +| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | +| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | +| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | +| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | +| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | +| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | +| [Phase8_taskList.md](./Phase8_taskList.md) | Log-trace correlation | +| [Phase9_taskList.md](./Phase9_taskList.md) | Internal metric instrumentation gap fill | +| [Phase10_taskList.md](./Phase10_taskList.md) | Synthetic workload generation & validation | +| [Phase11_taskList.md](./Phase11_taskList.md) | Third-party data collection pipelines (future) | + +> **Only Phase 11 is still "future".** Phase 9 ships on +> `pratik/otel-phase9-metric-gap-fill` (18 task entries, 9.1–9.17 plus 9.7a) and +> Phase 10 on `pratik/otel-phase10-workload-validation` (7 tasks). Their task +> lists are present on every branch from those points forward, so a reader on a +> later branch sees plans that are already implemented, not proposals. Phase 11 +> (13 tasks) has no implementation branch. > **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md). @@ -156,13 +163,21 @@ This guide maps Phase 9–11 content to its location across the documentation. ### Phase 9: Internal Metric Instrumentation Gap Fill -| Content | Location | -| ------------------------------- | ------------------------------------------------------------------------ | -| Plan & architecture | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) | -| Task list (10 tasks) | [Phase9_taskList.md](./Phase9_taskList.md) | -| Future metric definitions (~50) | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) | -| New class: `MetricsRegistry` | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (planned) | -| New dashboards | `fee-market`, `job-queue` (planned) | +| Content | Location | +| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Plan & architecture | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) | +| Task list (18 entries, 9.1–9.17) | [Phase9_taskList.md](./Phase9_taskList.md) | +| Metric definitions | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) | +| New class: `MetricsRegistry` | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` — **shipped** | +| New dashboards (4) | `fee-market`, `job-queue`, `peer-quality`, `validator-health` — **shipped** | +| Updated dashboards (2) | `node-health`, `rpc-performance` | +| Provisioned alert rules | `docker/telemetry/grafana/provisioning/alerting/rules.yaml` — 13 rules in 5 groups ([07 §7.6.2](./07-observability-backends.md)) | + +> **Task numbering**: `Phase9_taskList.md` carries 18 `## Task 9.x` headings — +> 9.1 through 9.17 plus the inserted 9.7a (`push_metrics.py` parity). The "10 +> tasks" figure in earlier revisions predates 9.7a and 9.11–9.17. Tasks 9.8 and +> 9.11–9.13 together produce the four new dashboards; Task 9.17 (peer span +> coverage) is explicitly **deferred to Phase 11**. **Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors. @@ -172,23 +187,45 @@ This guide maps Phase 9–11 content to its location across the documentation. | -------------------- | ------------------------------------------------------------------------ | | Plan & architecture | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md) | | Task list (7 tasks) | [Phase10_taskList.md](./Phase10_taskList.md) | +| Branch | `pratik/otel-phase10-workload-validation` | | Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) | -| Test harness | `docker/telemetry/docker-compose.workload.yaml` (planned) | -| CI workflow | `.github/workflows/telemetry-validation.yml` (planned) | +| Test harness | `docker/telemetry/docker-compose.workload.yaml` (phase-10 branch) | +| CI workflow | `.github/workflows/telemetry-validation.yml` (phase-10 branch) | -**Validates**: 16 spans, 22 attributes, 300+ metrics, 10 dashboards, log-trace correlation. +**Validates** (Phase-10 harness inventory): **40** span types, **67** unique +required span attributes, **36** metric entries, **14** dashboards, log-trace +correlation. + +> **These are the harness manifests' counts, and two of them lag the code.** The +> manifests (`docker/telemetry/workload/expected_spans.json`, +> `expected_metrics.json`) live only on the phase-10 branch. `expected_spans.json` +> holds 40 span entries against the **41** span-name families the code emits +> (`rpc.ws_upgrade` has no entry), and its own `total_unique_attributes: 58` field +> is stale against the 67 attributes its per-span `required_attributes` lists +> actually name. `expected_metrics.json` asserts 14 dashboard uids against the +> **15** dashboard JSONs in `docker/telemetry/grafana/dashboards/`; +> `log-derived-insights` is the unasserted one. The full emitted inventory is in +> [09-data-collection-reference.md §1.1](./09-data-collection-reference.md#11-complete-span-inventory-41-spans) +> and [§5c](./09-data-collection-reference.md#validated-telemetry-inventory). ### Phase 11: Third-Party Data Collection Pipelines | Content | Location | | --------------------------------- | ------------------------------------------------------------------------ | | Plan & architecture | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md) | -| Task list (11 tasks) | [Phase11_taskList.md](./Phase11_taskList.md) | +| Task list (13 tasks) | [Phase11_taskList.md](./Phase11_taskList.md) | | External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | | Custom OTel Collector receiver | `docker/telemetry/otel-rippled-receiver/` (planned) | | Prometheus alerting rules (11) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | | New dashboards (4) | Validator Health, Network Topology, Fee Market (External), DEX & AMM | +> **Two of those names now collide with shipped Phase-9 boards.** Phase 9 +> already ships `validator-health` and `fee-market`, both built from the node's +> **own** telemetry. The Phase-11 entries are the third-party-data variants +> (network-wide validator agreement, external fee/DEX feeds via the custom +> receiver). They need distinct uids, or they will overwrite the Phase-9 boards +> on provisioning. + **Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators. --- diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 1c67d275b2..b03b7e4ceb 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -2,7 +2,7 @@ > **Audience**: Developers and operators. This is the single source of truth for all telemetry data collected by xrpld's observability stack. > -> **Related docs**: [docs/telemetry-runbook.md](../docs/telemetry-runbook.md) (operator runbook with alerting and troubleshooting) | [03-implementation-strategy.md](./03-implementation-strategy.md) (code structure and performance optimization) | [04-code-samples.md](./04-code-samples.md) (C++ instrumentation examples) +> **Related docs**: [docs/telemetry-runbook.md](../docs/telemetry-runbook.md) (operator runbook with alerting and troubleshooting) | [03-implementation-strategy.md](./03-implementation-strategy.md) (code structure and performance optimization) | [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) (authoritative span-flow reference; replaces the deleted `04-code-samples.md`) ## Data Flow Overview @@ -33,7 +33,7 @@ graph LR end subgraph viz["Visualization"] - F["Grafana :3000
13 dashboards"] + F["Grafana :3000
15 dashboards"] end A -->|"OTLP/HTTP :4318
(traces + attributes)"| R1 @@ -72,9 +72,13 @@ There are three independent telemetry pipelines entering a single **OTel Collect A third, narrower metrics path exists for instruments created at their call site through the `XRPL_METRIC_*` macros. These use the OTel Metrics SDK directly and reach the collector's OTLP -receiver rather than the StatsD receiver, so their names carry no `xrpld_` prefix. See -[§2a](#2a-call-site-otel-metrics-metricsregistry). Code in `libxrpl` cannot use these macros and -always goes through `beast::insight` instead. +receiver rather than the StatsD receiver, so their names carry no `xrpld_` prefix. The seven +call-site instruments are documented with the families they belong to: +`rpc_in_flight_requests` in +[§Per-RPC Method Metrics](#per-rpc-method-metrics-synchronous-countershistogram), the five +`getobject_*` in [§GetObject Request Path](#getobject-request-path-synchronous-countershistograms), +and `ledgers_closed_total` in [§Synchronous Counters (Phase 7+)](#synchronous-counters-phase-7). +Code in `libxrpl` cannot use these macros and always goes through `beast::insight` instead. **Trace backend** — The collector exports traces via OTLP/gRPC to: @@ -86,13 +90,22 @@ always goes through `beast::insight` instead. ## 1. OpenTelemetry Spans -### 1.1 Complete Span Inventory (~37 spans) +### 1.1 Complete Span Inventory (41 spans) -> **See also**: [02-design-decisions.md §2.3](./02-design-decisions.md#23-span-naming-conventions) for naming conventions and the full span catalog with rationale. [04-code-samples.md §4.6](./04-code-samples.md#46-span-flow-visualization) for span flow diagrams. +> **41 emitted span-name families.** The count is derived from the `*SpanNames.h` +> headers and their call sites, one family per distinct span name +> (`rpc.command.` and `grpc.` each count once, since the +> command / method name is a parameter of a single family). The tables below list +> all 41: RPC 5, gRPC 1, transaction 6, TxQ 6, consensus 13, ledger 4, peer 2, +> pathfind 4. The Phase-10 validation harness +> (`docker/telemetry/workload/expected_spans.json`) catalogues **40** of them — +> `rpc.ws_upgrade` has no entry. + +> **See also**: [02-design-decisions.md §2.3](./02-design-decisions.md#23-span-naming-conventions) for naming conventions and the full span catalog with rationale. [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) for the span flow diagrams (the former `04-code-samples.md` §4.6 was deleted). > **Span names vs. attribute keys**: span names use dotted `subsystem.operation` > form (e.g. `rpc.http_request`). Span _attribute_ keys use the bare/underscore -> form from the 2026-05-13 naming redesign (e.g. `tx_hash`, not `xrpl.tx.hash`). +> form from the 2026-05-13 naming redesign (e.g. `tx_hash`, not `xrpl.tx.hash`). > The dotted `xrpl.*` form is reserved for OTel **resource** attributes set once > at startup. See §1.2 for the full attribute inventory. @@ -190,26 +203,35 @@ Controlled by `trace_transactions=1` in `[telemetry]` config. Controlled by `trace_consensus=1` in `[telemetry]` config. -| Span Name | Parent | Source File | Description | -| ------------------------------ | ------------------ | ---------------- | ------------------------------------------------------------------- | -| `consensus.round` | — (root) | RCLConsensus.cpp | Root span for one consensus round (deterministic trace per round) | -| `consensus.phase.open` | `consensus.round` | Consensus.h | Open phase — collecting transactions before close | -| `consensus.proposal.send` | `consensus.round` | RCLConsensus.cpp | Node broadcasts its transaction set proposal | -| `consensus.ledger_close` | `consensus.round` | RCLConsensus.cpp | Ledger close event triggered by consensus | -| `consensus.establish` | `consensus.round` | Consensus.h | Establish phase — converging on the transaction set | -| `consensus.update_positions` | `consensus.round` | Consensus.h | Position update with per-dispute vote details | -| `consensus.check` | `consensus.round` | Consensus.h | Consensus threshold check (agree/disagree tally) | -| `consensus.accept` | `consensus.round` | RCLConsensus.cpp | Consensus accepts a ledger (round complete) | -| `consensus.accept.apply` | `consensus.accept` | RCLConsensus.cpp | Ledger application with close-time details (jtACCEPT thread) | -| `consensus.validation.send` | `consensus.round` | RCLConsensus.cpp | Validation message sent after ledger accepted (follows-from link) | -| `consensus.mode_change` | `consensus.round` | RCLConsensus.cpp | Operating-mode transition during the round | -| `consensus.proposal.receive` | (context) | PeerImp.cpp | Proposal received from a peer (context-propagated into the round) | -| `consensus.validation.receive` | (context) | PeerImp.cpp | Validation received from a peer (context-propagated into the round) | +| Span Name | Parent | Source File | Description | +| ------------------------------ | --------------------- | ---------------- | ------------------------------------------------------------------- | +| `consensus.round` | — (root) | RCLConsensus.cpp | Root span for one consensus round (deterministic trace per round) | +| `consensus.phase.open` | `consensus.round` | Consensus.h | Open phase — collecting transactions before close | +| `consensus.proposal.send` | `consensus.round` | RCLConsensus.cpp | Node broadcasts its transaction set proposal | +| `consensus.ledger_close` | `consensus.round` | RCLConsensus.cpp | Ledger close event triggered by consensus | +| `consensus.establish` | `consensus.round` | Consensus.h | Establish phase — converging on the transaction set | +| `consensus.update_positions` | `consensus.establish` | Consensus.h | Position update with per-dispute vote details | +| `consensus.check` | `consensus.establish` | Consensus.h | Consensus threshold check (agree/disagree tally) | +| `consensus.accept` | `consensus.round` | RCLConsensus.cpp | Consensus accepts a ledger (round complete) | +| `consensus.accept.apply` | `consensus.accept` | RCLConsensus.cpp | Ledger application with close-time details (jtACCEPT thread) | +| `consensus.validation.send` | `consensus.round` | RCLConsensus.cpp | Validation message sent after ledger accepted (follows-from link) | +| `consensus.mode_change` | `consensus.round` | RCLConsensus.cpp | Operating-mode transition during the round | +| `consensus.proposal.receive` | (context) | PeerImp.cpp | Proposal received from a peer (context-propagated into the round) | +| `consensus.validation.receive` | (context) | PeerImp.cpp | Validation received from a peer (context-propagated into the round) | The `.receive` spans are created per-message in the overlay and joined to the round trace via context propagation rather than direct parenting. The `consensus.validation.send` span uses a follows-from link off the round. +> **`update_positions` and `check` sit one level below `establish`, not below +> the round.** Both are created with +> `SpanGuard::childSpan(..., establishSpanContext_)` +> (`include/xrpl/consensus/Consensus.h:1628` and `:1837`), and +> `consensus.establish` is itself parented to `roundSpanContext_` +> (`Consensus.h:2099-2101`). An earlier revision of this table showed them as +> direct children of `consensus.round`; queries or trace-shape assertions built +> on that tree are wrong by one level. + **Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"consensus.*"}` **Grafana dashboard**: _Consensus Health_ (`consensus-health`) @@ -251,12 +273,12 @@ under an unrelated transaction's trace. Controlled by `trace_rpc=1` in `[telemetry]` config. -| Span Name | Parent | Source File | Description | -| --------------------- | -------------------- | --------------- | ---------------------------------------------------------- | -| `pathfind.request` | `rpc.command.` | PathFind.cpp | `path_find` RPC entry (`doPathFind`) | -| `pathfind.compute` | `pathfind.request` | PathRequest.cpp | Path computation for one request (`PathRequest::doUpdate`) | -| `pathfind.discover` | `pathfind.compute` | Pathfinder.cpp | Graph exploration (one per RPC call) | -| `pathfind.update_all` | — | PathRequest.cpp | Async recomputation of all active requests at ledger close | +| Span Name | Parent | Source File | Description | +| --------------------- | -------------------- | -------------------------------------- | ---------------------------------------------------------- | +| `pathfind.request` | `rpc.command.` | PathFind.cpp:27, RipplePathFind.cpp:36 | `path_find` / `ripple_path_find` RPC entry | +| `pathfind.compute` | `pathfind.request` | PathRequest.cpp:750 | Path computation for one request (`PathRequest::doUpdate`) | +| `pathfind.discover` | `pathfind.compute` | PathRequest.cpp:599-600 | Graph exploration (one per RPC call) | +| `pathfind.update_all` | — | PathRequestManager.cpp:88-92 | Async recomputation of all active requests at ledger close | > **Note**: `pathfind.request` nests under the active `rpc.command.` span. > Because OTel context storage is coroutine-aware (backed by `LocalValue`), the @@ -367,50 +389,78 @@ Join a transaction's work to its ledger with `{span.current_ledger_seq=}`. #### Consensus Attributes -| Attribute | Type | Set On | Description | -| -------------------------- | ------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | -| `consensus_ledger_id` | string | `consensus.round` | Previous-ledger id anchoring the round | -| `ledger_seq` | int64 | `consensus.round`, `consensus.ledger_close`, `consensus.accept.apply`, `consensus.validation.send` | Ledger sequence number | -| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | Node mode: `"Proposing"`, `"Observing"`, `"Wrong"`, etc. | -| `consensus_round_id` | int64 | `consensus.round` | Round identifier | -| `consensus_phase` | string | `consensus.round` | Current phase name (updated on each transition) | -| `trace_strategy` | string | `consensus.round` | Trace-id strategy (`deterministic` / `random`) | -| `previous_ledger_seq` | int64 | `consensus.round` | Sequence of the previous ledger | -| `previous_proposers` | int64 | `consensus.round` | Proposer count in the previous round | -| `previous_round_time_ms` | int64 | `consensus.round` | Duration of the previous round | -| `consensus_round` | int64 | `consensus.proposal.send` | Proposal sequence number for the broadcast proposal | -| `is_bow_out` | boolean | `consensus.proposal.send` | Whether the proposal is a bow-out (resigning the round) | -| `tx_count_open` | int64 | `consensus.ledger_close` | Transactions in the open ledger at close | -| `close_time_resolution_ms` | int64 | `consensus.ledger_close` | Close-time rounding granularity | -| `converge_percent` | int64 | `consensus.establish`, `consensus.update_positions` | Convergence percentage | -| `establish_count` | int64 | `consensus.establish` | Establish-phase iteration count | -| `proposers` | int64 | `consensus.establish`, `consensus.update_positions`, `consensus.accept` | Number of proposers | -| `disputes_count` | int64 | `consensus.establish`, `consensus.update_positions` | Number of disputed transactions | -| `tx_id` | string | `consensus.update_positions` | Disputed transaction id (per-dispute event) | -| `dispute_our_vote` | boolean | `consensus.update_positions` | Our vote on the disputed tx | -| `dispute_yays` | int64 | `consensus.update_positions` | Yes votes on the disputed tx | -| `dispute_nays` | int64 | `consensus.update_positions` | No votes on the disputed tx | -| `agree_count` | int64 | `consensus.check` | Agreeing proposer count | -| `disagree_count` | int64 | `consensus.check` | Disagreeing proposer count | -| `threshold_percent` | int64 | `consensus.check` | Agreement threshold percentage | -| `consensus_result` | string | `consensus.check` | Check outcome | -| `quorum` | int64 | `consensus.check`, `consensus.accept` | Quorum required | -| `round_time_ms` | int64 | `consensus.accept`, `consensus.accept.apply` | Total consensus round duration in milliseconds | -| `consensus_state` | string | `consensus.accept.apply` | Consensus outcome: `"finished"` or `"moved_on"` | -| `close_time` | int64 | `consensus.accept.apply` | Agreed-upon ledger close time (epoch seconds) | -| `close_time_correct` | boolean | `consensus.accept.apply` | Whether validators agreed on close time | -| `close_resolution_ms` | int64 | `consensus.accept.apply` | Close-time rounding granularity in milliseconds | -| `proposing` | boolean | `consensus.accept.apply`, `consensus.validation.send` | Whether this node was a proposer | -| `parent_close_time` | int64 | `consensus.accept.apply` | Parent ledger close time | -| `close_time_self` | int64 | `consensus.accept.apply` | This node's close-time vote | -| `close_time_vote_bins` | string | `consensus.accept.apply` | Distribution of close-time votes | -| `resolution_direction` | string | `consensus.accept.apply` | Whether close resolution increased/decreased/unchanged | -| `tx_count` | int64 | `consensus.accept.apply` | Transactions in the accepted set | -| `ledger_hash` | string | `consensus.validation.send` | Full hash of the validated ledger (shared with peer) | -| `full_validation` | boolean | `consensus.validation.send` | Whether this is a full validation | -| `validation_sign_time` | int64 | `consensus.validation.send` | Validation signing time | -| `mode_old` | string | `consensus.mode_change` | Operating mode before the transition | -| `mode_new` | string | `consensus.mode_change` | Operating mode after the transition | +| Attribute | Type | Set On | Description | +| --------------------------- | ------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `consensus_ledger_id` | string | `consensus.round` | Previous-ledger id anchoring the round | +| `ledger_seq` | int64 | `consensus.round`, `consensus.ledger_close`, `consensus.accept.apply`, `consensus.validation.send` | Ledger sequence number | +| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | Node mode: `"Proposing"`, `"Observing"`, `"Wrong"`, etc. | +| `consensus_round_id` | int64 | `consensus.round` | Round identifier | +| `consensus_phase` | string | `consensus.round` | Current phase name (updated on each transition) | +| `trace_strategy` | string | `consensus.round` | Trace-id strategy (`deterministic` / `attribute`) | +| `previous_ledger_seq` | int64 | `consensus.round` | Sequence of the previous ledger | +| `previous_proposers` | int64 | `consensus.round` | Proposer count in the previous round | +| `previous_round_time_ms` | int64 | `consensus.round` | Duration of the previous round | +| `consensus_round` | int64 | `consensus.proposal.send` | Proposal sequence number for the broadcast proposal | +| `is_bow_out` | boolean | `consensus.proposal.send` | Whether the proposal is a bow-out (resigning the round) | +| `tx_count_open` | int64 | `consensus.ledger_close` | Transactions in the open ledger at close | +| `close_time_resolution_ms` | int64 | `consensus.ledger_close` | Close-time rounding granularity | +| `converge_percent` | int64 | `consensus.establish`, `consensus.update_positions`, `consensus.check` | Convergence percentage | +| `establish_count` | int64 | `consensus.establish`, `consensus.check` | Establish-phase iteration count | +| `proposers` | int64 | `consensus.establish`, `consensus.update_positions`, `consensus.accept` | Number of proposers | +| `disputes_count` | int64 | `consensus.establish`, `consensus.update_positions` | Number of disputed transactions | +| `tx_id` | string | `consensus.update_positions` | Disputed transaction id (per-dispute event) | +| `dispute_our_vote` | boolean | `consensus.update_positions` | Our vote on the disputed tx | +| `dispute_yays` | int64 | `consensus.update_positions` | Yes votes on the disputed tx | +| `dispute_nays` | int64 | `consensus.update_positions` | No votes on the disputed tx | +| `avalanche_threshold` | int64 | `consensus.update_positions` | Escalated weight needed to change our vote | +| `close_time_threshold` | int64 | `consensus.update_positions` | Close-time agreement threshold percentage | +| `agree_count` | int64 | `consensus.check` | Agreeing proposer count | +| `disagree_count` | int64 | `consensus.check` | Disagreeing proposer count | +| `threshold_percent` | int64 | `consensus.check` | Agreement threshold percentage | +| `have_close_time_consensus` | boolean | `consensus.update_positions`, `consensus.check` | Whether the close time reached consensus | +| `proposers_finished` | int64 | `consensus.check` | Proposers that have already validated the next ledger | +| `consensus_stalled` | boolean | `consensus.check` | Whether `checkConsensus` reported a stall | +| `consensus_result` | string | `consensus.check` | Check outcome | +| `quorum` | int64 | `consensus.accept` | Quorum required | +| `round_time_ms` | int64 | `consensus.accept`, `consensus.accept.apply` | Total consensus round duration in milliseconds | +| `consensus_state` | string | `consensus.accept.apply` | Consensus outcome: `"finished"` or `"moved_on"` | +| `close_time` | int64 | `consensus.accept.apply` | Agreed-upon ledger close time (epoch seconds) | +| `close_time_correct` | boolean | `consensus.accept.apply` | Whether validators agreed on close time | +| `close_resolution_ms` | int64 | `consensus.accept.apply` | Close-time rounding granularity in milliseconds | +| `proposing` | boolean | `consensus.accept.apply`, `consensus.validation.send` | Whether this node was a proposer | +| `parent_close_time` | int64 | `consensus.accept.apply` | Parent ledger close time | +| `close_time_self` | int64 | `consensus.accept.apply` | This node's close-time vote | +| `close_time_vote_bins` | string | `consensus.accept.apply` | Distribution of close-time votes | +| `resolution_direction` | string | `consensus.accept.apply` | Whether close resolution increased/decreased/unchanged | +| `tx_count` | int64 | `consensus.accept.apply` | Transactions in the accepted set | +| `ledger_hash` | string | `consensus.validation.send` | Full hash of the validated ledger (shared with peer) | +| `full_validation` | boolean | `consensus.validation.send` | Whether this is a full validation | +| `validation_sign_time` | int64 | `consensus.validation.send` | Validation signing time | +| `mode_old` | string | `consensus.mode_change` | Operating mode before the transition | +| `mode_new` | string | `consensus.mode_change` | Operating mode after the transition | + +> **`quorum` is on `consensus.accept` only.** Its single set site is +> `RCLConsensus::Adaptor::makeAcceptSpan()` +> (`src/xrpld/app/consensus/RCLConsensus.cpp:516`). `consensus.check` +> (`include/xrpl/consensus/Consensus.h:1899-1926`) never sets it, so +> `{name="consensus.check" && span.quorum>0}` matches nothing. + +> **`consensus.check` carries nine attributes, all set before the early +> returns.** `Consensus::haveConsensus()` sets them at +> `include/xrpl/consensus/Consensus.h:1899-1912` and `consensus_result` at +> `:1925`, deliberately ahead of the `No` / `Expired` branches, so the span is +> fully populated even on rounds that never reach consensus. In set order: +> `agree_count`, `disagree_count`, `converge_percent`, +> `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, +> `consensus_stalled`, `establish_count`, `consensus_result`. +> +> Three of these are shared with sibling spans and were previously scoped too +> narrowly in the table above: `converge_percent` and `establish_count` are set on +> `consensus.check` as well as `consensus.establish` / +> `consensus.update_positions`, and `have_close_time_consensus` is set on both +> `consensus.update_positions` (`Consensus.h:1779`) and `consensus.check` +> (`:1903`). `close_time_threshold` (`:1781`) and `avalanche_threshold` (`:1730`) +> stay `consensus.update_positions`-only. **Tempo query**: `{span.consensus_mode="Proposing"}` to find rounds where the node was proposing. @@ -551,25 +601,44 @@ prefix=xrpld ### 2.1 Gauges -| Prometheus Metric | Source File | Description | Typical Range | -| ------------------------------------------- | --------------------- | ----------------------------------------- | ------------------------------- | -| `ledgermaster_validated_ledger_age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | -| `ledgermaster_published_ledger_age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | -| `state_accounting_disconnected_duration` | NetworkOPs.cpp | Cumulative seconds in Disconnected state | Monotonic | -| `state_accounting_connected_duration` | NetworkOPs.cpp | Cumulative seconds in Connected state | Monotonic | -| `state_accounting_syncing_duration` | NetworkOPs.cpp | Cumulative seconds in Syncing state | Monotonic | -| `state_accounting_tracking_duration` | NetworkOPs.cpp | Cumulative seconds in Tracking state | Monotonic | -| `state_accounting_full_duration` | NetworkOPs.cpp | Cumulative seconds in Full state | Monotonic (should dominate) | -| `state_accounting_disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | -| `state_accounting_connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | -| `state_accounting_syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | -| `state_accounting_tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | -| `state_accounting_full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | -| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | -| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | -| `overlay_peer_disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | -| `overlay_peer_disconnects_charges` | OverlayImpl.cpp | Disconnects due to resource limit charges | Low growth (subset of above) | -| `jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | +| Prometheus Metric | Source File | Description | Typical Range | +| ------------------------------------------- | --------------------- | ------------------------------------------------- | ------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | +| `state_accounting_disconnected_duration` | NetworkOPs.cpp | Cumulative **microseconds** in Disconnected state | Monotonic | +| `state_accounting_connected_duration` | NetworkOPs.cpp | Cumulative **microseconds** in Connected state | Monotonic | +| `state_accounting_syncing_duration` | NetworkOPs.cpp | Cumulative **microseconds** in Syncing state | Monotonic | +| `state_accounting_tracking_duration` | NetworkOPs.cpp | Cumulative **microseconds** in Tracking state | Monotonic | +| `state_accounting_full_duration` | NetworkOPs.cpp | Cumulative **microseconds** in Full state | Monotonic (should dominate) | +| `state_accounting_disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | +| `state_accounting_connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | +| `state_accounting_syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | +| `state_accounting_tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | +| `state_accounting_full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | +| `overlay_peer_disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | +| `jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | + +> **`state_accounting_*_duration` is microseconds, not seconds.** +> `NetworkOPsImp::collectMetrics()` does +> `duration_cast(...)` and publishes `.count()` +> (`src/xrpld/app/misc/NetworkOPs.cpp:4884-4897`). Divide by `1e6` for seconds. +> The `node-health` "State Duration Rate (All States)" panel already does +> (`/ 1000000` on each `rate(...)`), and +> `docker/telemetry/grafana/dashboards/validate_dashboards.py:43` lints the +> family as "cumulative µs". Reading the raw value as seconds overstates time +> in state by a factor of one million. + +> **`overlay_peer_disconnects_charges` was never implemented: NOT IMPLEMENTED.** +> No instrument of that name exists anywhere in `src/`, `include/` or `docker/`. +> The resource-charge disconnect count is exported from the OTel +> `MetricsRegistry` instead, as +> `server_info{metric="peer_disconnects_resources"}` — see +> [§Server Info](#server-info-via-otel-metricsregistry). Use that selector; +> the previously documented `overlay_peer_disconnects_charges` matches nothing. +> `06-implementation-phases.md` still names the old metric in its Phase 6/7 +> task text and panel table. **Grafana dashboard**: _Node Health_ (`node-health`) @@ -758,13 +827,14 @@ for how the tier attributes are set and reach metrics. 1. Open Grafana at **http://localhost:3000** 2. Navigate to **Dashboards → xrpld** folder -3. All 10 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` +3. All 15 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` + (the Phase-10 harness asserts 14 of them — `log-derived-insights` is unasserted) --- ## 4. Tempo Trace Search Guide -> **See also**: [08-appendix.md](./08-appendix.md) §8.2 for span hierarchy visualizations. [05-configuration-reference.md](./05-configuration-reference.md) §5.8.5 for TraceQL query examples. +> **See also**: [08-appendix.md](./08-appendix.md) §8.2 for span hierarchy visualizations. [05-configuration-reference.md](./05-configuration-reference.md) §5.8.4 for TraceQL query examples. ### Finding Traces by Type @@ -796,16 +866,16 @@ A consensus round groups its lifecycle spans under a single root (`consensus.round`); the build/ledger spans run as their own trees: ``` -consensus.round (root — one per round) - ├── consensus.phase.open (open phase) - ├── consensus.proposal.send (broadcast proposal) - ├── consensus.ledger_close (close event) - ├── consensus.establish (establish phase) - ├── consensus.update_positions (position updates) - ├── consensus.check (threshold check) - ├── consensus.accept (accept result) - │ └── consensus.accept.apply (apply, jtACCEPT thread) - └── consensus.validation.send (send validation, follows-from link) +consensus.round (root — one per round) + ├── consensus.phase.open (open phase) + ├── consensus.proposal.send (broadcast proposal) + ├── consensus.ledger_close (close event) + ├── consensus.establish (establish phase) + │ ├── consensus.update_positions (position updates) + │ └── consensus.check (threshold check) + ├── consensus.accept (accept result) + │ └── consensus.accept.apply (apply, jtACCEPT thread) + └── consensus.validation.send (send validation, follows-from link) ledger.build (build new ledger) └── tx.apply (apply transaction set) @@ -817,7 +887,7 @@ ledger.store (persist to DB) ## 5. Prometheus Query Examples -> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8.7 for correlating Prometheus system metrics with trace-derived metrics. +> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8.6 for correlating Prometheus system metrics with trace-derived metrics. ### Span-Derived Metrics @@ -916,18 +986,26 @@ Grafana Loki (v3.7.6) serves as the log storage backend. It receives log entries ### LogQL Query Examples +The stream selector is `{service_name="xrpld"}`, **not** `{job="xrpld"}`. Loki's +OTLP ingestion promotes only a small set of resource attributes to stream labels +(`service_name`, `service_instance_id`, `deployment_environment`); everything else +— including the `job` attribute the collector sets — lands in structured +metadata and must be filtered with `|` after the selector. A `{job="xrpld"}` +selector returns zero rows and no error. All shipped queries and the +`log-derived-insights` dashboard use the `service_name` form. + ```logql # Find all logs for a specific trace -{job="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" +{service_name="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" # Error logs with trace context -{job="xrpld"} |= "ERR" |= "trace_id=" +{service_name="xrpld"} |= "ERR" |= "trace_id=" # Logs from a specific partition with trace context -{job="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" +{service_name="xrpld"} | partition = `LedgerMaster` | trace_id != "" # Count traced log lines over time -count_over_time({job="xrpld"} |= "trace_id=" [5m]) +count_over_time({service_name="xrpld"} |= "trace_id=" [5m]) ``` --- @@ -944,10 +1022,17 @@ async callbacks for new categories. > **Authoritative metric names live in [§ Phase 9: OTel SDK-Exported Metrics](#phase-9-otel-sdk-exported-metrics-metricsregistry) below.** > Most internal metrics are emitted as **labeled** gauges — one instrument carrying many logical -> values via a `metric` label (e.g. `cache_metrics{metric="sle_hit_rate"}`, +> values via a `metric` label (e.g. `cache_metrics{metric="SLE_hit_rate"}`, > `txq_metrics{metric="txq_count"}`, `load_factor_metrics{metric="load_factor"}`, > `nodestore_state{metric="node_reads_total"}`) — not the flat per-name form. Query the -> labeled names; the flat names (`cache_sle_hit_rate`, `txq_count`, …) are **not** emitted. +> labeled names; the flat names (`cache_SLE_hit_rate`, `txq_count`, …) are **not** emitted. +> +> **Label values are case-sensitive and three cache values are not lowercase.** +> The `metric` label carries the string literal passed to `Observe()`, verbatim: +> `SLE_hit_rate`, `AL_hit_rate` and `AL_size` are upper-case +> (`src/xrpld/telemetry/MetricsRegistry.cpp:649`, `:665`, `:691`), while +> `ledger_hit_rate` genuinely is lowercase (`:658`). A selector written as +> `cache_metrics{metric="sle_hit_rate"}` matches nothing. #### Server Info (via OTel MetricsRegistry) @@ -989,16 +1074,33 @@ async callbacks for new categories. | Prometheus Metric | Type | Labels | Description | | --------------------------------- | ----- | -------- | ------------------------- | -| `cache_metrics{metric="al_size"}` | Gauge | `metric` | AcceptedLedger cache size | +| `cache_metrics{metric="AL_size"}` | Gauge | `metric` | AcceptedLedger cache size | #### Extended NodeStore Metrics (additions to existing nodestore_state) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------------------- | ----- | -------- | ----------------------------------- | -| `nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | -| `nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | -| `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | -| `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------------- | ----- | -------- | ------------------------------------ | +| `nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | +| `nodestore_state{metric="node_writes_duration_us"}` | Gauge | `metric` | Cumulative write time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | + +> **The cumulative duration pair truncates to whole microseconds.** Both values +> are accumulated in nanoseconds internally and divided on read — +> `getFetchDurationUs()` returns `fetchDurationNs_ / 1000` and +> `getStoreDurationUs()` returns `storeDurationNs_ / 1000` +> (`include/xrpl/nodestore/Database.h:232-254`). The exported unit is +> microseconds and every doc, metric and dashboard agrees on that — this is +> **not** a unit mismatch. The consequence is only at the low end: a handful of +> sub-microsecond reads on a warm store can leave the gauge reading `0` until +> their nanosecond total passes 1000. Read a flat `0` on a low-traffic node as +> "not yet a microsecond of I/O", not as "no I/O". +> +> `node_writes_duration_us` is covered by +> `validate_dashboards.py`'s `NODESTORE_CUMULATIVE` tuple, so the raw-counter +> lint would catch a misuse, but it has **no dashboard panel** yet — an open +> follow-up, unlike its `node_reads_duration_us` sibling. #### Job Queue and GetObject Additions @@ -1019,7 +1121,12 @@ repeated here: write-serialized stall from a cold-read stall. See [Sync Diagnosis Signals](#sync-diagnosis-signals-observable-gauge--nodestore_state). -### New Grafana Dashboards (Phase 9) +### New Grafana Dashboards for the Phase 9 Gap-Fill Metrics + +These two boards were created specifically to surface the gap-fill metrics above. +For the full Phase-9 dashboard delivery record, including the boards added to the +Phase-7 parity set, see +[New Grafana Dashboards (Phase 9)](#new-grafana-dashboards-phase-9). | Dashboard | UID | Data Source | Key Panels | | ------------------ | ------------ | ----------- | ----------------------------------------------------------------- | @@ -1048,16 +1155,35 @@ Phase 10 builds a 5-node validator docker-compose harness with RPC load generato | Category | Expected Count | Validation Method | | ------------------------- | ------------------- | -------------------------------- | -| Trace spans | 16 | Jaeger/Tempo API query | -| Span attributes | 22 | Per-span attribute assertion | +| Trace spans | 40 of 41 emitted | Jaeger/Tempo API query | +| Span attributes | 67 required | Per-span attribute assertion | | Legacy `*` families | ~270 (≈224 traffic) | Prometheus `__name__` query | | Native MetricsRegistry | 35 instruments | Prometheus query | | Call-site `XRPL_METRIC_*` | 7 instruments | Prometheus query | | Per-job-type gauges | 105 (35 types × 3) | Prometheus `__name__` query | | SpanMetrics RED | 4 per span | Prometheus query | -| Grafana dashboards | 10 | Dashboard API "no data" check | +| Grafana dashboards | 14 of 15 on disk | Dashboard API "no data" check | | Log-trace links | Present | Loki query + Tempo reverse check | +> **These are the harness's numbers, not the code's, and three of them differ.** +> `docker/telemetry/workload/expected_spans.json` carries 40 span entries against +> the **41** families the code emits ([§1.1](#11-complete-span-inventory-41-spans)) — +> `rpc.ws_upgrade` has no entry — and 67 distinct required attributes (the +> manifest's own `total_unique_attributes: 58` field is stale). +> `expected_metrics.json` asserts 36 metric entries and 14 dashboard uids against +> the **15** dashboard JSONs in `docker/telemetry/grafana/dashboards/`; +> `log-derived-insights` is the unasserted one. The 35 native instruments match +> the tables in +> [§Phase 9: OTel SDK-Exported Metrics](#phase-9-otel-sdk-exported-metrics-metricsregistry) +> and the Phase 7+ section exactly, counting each labeled gauge family +> (`nodestore_state`, `cache_metrics`, …) once. +> +> Note that `ledgers_closed_total` appears in **both** instrument rows: it is +> created as a `MetricsRegistry` member (`MetricsRegistry.cpp:369-370`, whose +> `incrementLedgersClosed()` has no callers) and separately incremented at its +> call site via `XRPL_METRIC_COUNTER_INC` (`RCLConsensus.cpp:749`). The distinct +> name count across the two rows is therefore 41, not 42. + The two added rows are the families that do not originate as `MetricsRegistry` members. **Call-site** instruments are declared by the `XRPL_METRIC_*` macros (7 distinct names: `rpc_in_flight_requests`, `ledgers_closed_total`, and the @@ -1185,7 +1311,7 @@ Further label values on the same instrument, added to separate the two bottlenecks that both present as the `ledgerData` job lane pinned at its concurrency cap. Observed in `MetricsRegistry::observeNodeStoreTotals()`, `observeWritePathDetail()`, and `observeAcquireStats()` -(`src/xrpld/telemetry/MetricsRegistry.cpp:805-877`). +(`src/xrpld/telemetry/MetricsRegistry.cpp:854-925`). | Prometheus Metric | Type | Labels | Description | | ---------------------------------------------------- | ----- | -------- | ------------------------------------------------------- | @@ -1254,9 +1380,9 @@ data as uninformative unless the build is known to include the fix. | Prometheus Metric | Type | Labels | Description | | --------------------------------------------- | ----- | -------- | ----------------------------- | -| `cache_metrics{metric="sle_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | +| `cache_metrics{metric="SLE_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | | `cache_metrics{metric="ledger_hit_rate"}` | Gauge | `metric` | Ledger cache hit rate | -| `cache_metrics{metric="al_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | +| `cache_metrics{metric="AL_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | | `cache_metrics{metric="treenode_cache_size"}` | Gauge | `metric` | SHAMap TreeNode cache entries | | `cache_metrics{metric="treenode_track_size"}` | Gauge | `metric` | Tracked tree nodes | | `cache_metrics{metric="fullbelow_size"}` | Gauge | `metric` | FullBelow cache entries | @@ -1274,6 +1400,73 @@ data as uninformative unless the build is known to include the fix. | `txq_metrics{metric="txq_med_fee_level"}` | Gauge | `metric` | Median fee level in queue | | `txq_metrics{metric="txq_open_ledger_fee_level"}` | Gauge | `metric` | Open ledger fee escalation level | +#### TxQ Admission and Ledger Mismatch (Synchronous Counters) + +Three monotonic counters created alongside the Phase 7+ parity counters +(`src/xrpld/telemetry/MetricsRegistry.cpp:377-382`). The gauges above answer +"how deep is the queue"; these answer "what did the queue refuse, and did the +ledger we built match the one the network validated". + +| Prometheus Metric | Type | Labels | Description | Increment Site | +| ------------------------------- | ------- | ----------------- | -------------------------------------------------- | --------------------- | +| `txq_dropped_total` | Counter | `reason=""` | Transactions refused admission to the queue | TxQ.cpp:1302,1347 | +| `txq_expired_total` | Counter | (none) | Transactions abandoned out of the queue on expiry | TxQ.cpp:1428 | +| `ledger_history_mismatch_total` | Counter | `reason=""` | Built-vs-validated ledger hash mismatches, by kind | LedgerHistory.cpp:332 | + +Label domains, as emitted: + +| Label | Values | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `txq_dropped_total{reason}` | `queue_full` | +| `ledger_history_mismatch_total{reason}` | `prior_ledger`, `close_time`, `consensus_txset`, `different_txset`, `same_txset_diff_result`, `unknown` | + +**Grafana dashboards**: _Fee Market & TxQ_ (`fee-market`) — "Queue Admission +Rejections (Dropped)", "Queue Abandonment Rate (Expired)"; _Consensus Health_ +(`consensus-health`) — "Ledger History Mismatch Rate by Reason"; _Node Health_ +(`node-health`) — "Ledger History Mismatches". + +> **Known issue — `ledger_history_mismatch_total` has two producers, so a bare +> `sum()` double-counts.** `LedgerHistory::handleMismatch()` increments **both** +> a `beast::insight` counter registered as `ledger.history` / `mismatch` +> (`src/xrpld/app/ledger/LedgerHistory.cpp:323`, created at `:41`) **and** the +> OTel counter above (`:331-332`). The insight counter carries **no** `reason` +> label, and the Prometheus exporter appends `_total` to both, so the two land in +> one metric family: per-node series carrying a `reason` label, plus per-node +> series with `reason` absent that already total all of them. The dual-producer +> mechanism is verifiable from the code above; the exact series count in any given +> stack depends on how many nodes report and how many distinct reasons they have +> hit, so do not treat a fixed number as an invariant. +> +> Consequence: `sum(rate(ledger_history_mismatch_total[5m]))` counts every +> mismatch twice. Always group or filter by `reason`: +> `sum by (reason) (rate(ledger_history_mismatch_total{reason!=""}[5m]))` for the +> per-reason breakdown, or `reason=""` for the untyped total alone. This is a +> **code** defect, not a documentation one — the fix is to retire one producer; +> until then the shipped panels avoid the trap (`consensus-health` groups +> `by (reason)`, `node-health` plots the series unaggregated), and any new panel +> or alert must do the same. + +#### Reduce-Relay Efficiency (Observable Gauge — `reduce_relay_metrics`) + +Transaction reduce-relay effectiveness, read from `Overlay::txMetrics()` each +collection cycle (`src/xrpld/telemetry/MetricsRegistry.cpp:1353-1385`). A high +`suppressed_peers` : `selected_peers` ratio proves the feature is saving +bandwidth; a high `not_enabled_peers` means stale peers are forcing full relay. + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ----- | -------- | ------------------------------------------------------- | +| `reduce_relay_metrics{metric="selected_peers"}` | Gauge | `metric` | Peers selected to receive a relayed transaction | +| `reduce_relay_metrics{metric="suppressed_peers"}` | Gauge | `metric` | Peer sends suppressed by reduce-relay | +| `reduce_relay_metrics{metric="not_enabled_peers"}` | Gauge | `metric` | Peers without reduce-relay support, so relayed in full | +| `reduce_relay_metrics{metric="missing_tx_freq"}` | Gauge | `metric` | Frequency of transactions this node had to request back | + +Each source field is a decimal **string** in the `txMetrics()` JSON, parsed with +`std::stoll`; a field that is absent or unparseable is skipped rather than +reported as zero, so absent is not zero here either. + +**Grafana dashboard**: _Peer Network_ (`peer-network`) — "Reduce-Relay Peer +Selection", "Reduce-Relay Missing-Tx Frequency". + #### Per-RPC Method Metrics (Synchronous Counters/Histogram) | Prometheus Metric | Type | Labels | Description | @@ -1412,17 +1605,43 @@ spelling would silently drop the override. #### Counted Object Instances (Observable Gauge — `object_count`) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------- | ----- | --------------- | ------------------------------ | -| `object_count{type="transaction"}` | Gauge | `type=""` | Live Transaction objects | -| `object_count{type="ledger"}` | Gauge | `type=""` | Live Ledger objects | -| `object_count{type="nodeobject"}` | Gauge | `type=""` | Live NodeObject instances | -| `object_count{type="sttx"}` | Gauge | `type=""` | Serialized transaction objects | -| `object_count{type="stledgerentry"}` | Gauge | `type=""` | Serialized ledger entries | -| `object_count{type="inboundledger"}` | Gauge | `type=""` | Ledgers being fetched | -| `object_count{type="pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | -| `object_count{type="pathrequest"}` | Gauge | `type=""` | Active path requests | -| `object_count{type="hashrouterentry"}` | Gauge | `type=""` | Hash router entries | +**The `type` label value is the demangled, fully-qualified C++ type name.** It is +not a lowercase word and not a friendly alias. The value is +`beast::typeName()` (`include/xrpl/basics/CountedObject.h:115`), which +demangles `typeid(T).name()` with `abi::__cxa_demangle` +(`include/xrpl/beast/type_name.h:16-45`) and applies no stripping; the observer +copies it through verbatim (`src/xrpld/telemetry/MetricsRegistry.cpp:764-770`). +Values therefore keep their `xrpl::` namespace, nested `::`, and template +arguments. + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ----- | --------------- | ------------------------------ | +| `object_count{type="xrpl::Transaction"}` | Gauge | `type=""` | Live Transaction objects | +| `object_count{type="xrpl::Ledger"}` | Gauge | `type=""` | Live Ledger objects | +| `object_count{type="xrpl::NodeObject"}` | Gauge | `type=""` | Live NodeObject instances | +| `object_count{type="xrpl::STTx"}` | Gauge | `type=""` | Serialized transaction objects | +| `object_count{type="xrpl::STLedgerEntry"}` | Gauge | `type=""` | Serialized ledger entries | +| `object_count{type="xrpl::InboundLedger"}` | Gauge | `type=""` | Ledgers being fetched | +| `object_count{type="xrpl::Pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | +| `object_count{type="xrpl::PathRequest"}` | Gauge | `type=""` | Active path requests | +| `object_count{type="xrpl::HashRouter::Entry"}` | Gauge | `type=""` | Hash router entries | + +The list above is the subset most often queried, not the whole label domain. The +series set is whatever `CountedObject` subclasses have been instantiated, so +it also includes `xrpl::SHAMapItem`, `xrpl::SHAMapInnerNode`, +`xrpl::AcceptedLedger`, `xrpl::Job`, template instantiations such as +`xrpl::STBitString<256>` and `xrpl::STInteger`, and a few types +outside the `xrpl` namespace such as `CachedView::hit`. Enumerate it rather than +guess: + +```promql +# Every type currently reporting on one node +count by (type) (object_count{service_instance_id=~"$node"}) +``` + +Grafana's `$type` template variable on _Node Health_ is populated the same way +(`label_values(object_count, type)`), which is why that dashboard needs no +hardcoded list. #### Load Factor Breakdown (Observable Gauge — `load_factor_metrics`) @@ -1515,30 +1734,51 @@ These metrics fill gaps identified by comparing xrpld's internal observability w | -------------------------------------------------- | ------ | -------- | --------------------------------------- | | `validation_agreement{metric="agreement_pct_1h"}` | Double | `metric` | Rolling 1h agreement percentage (0-100) | | `validation_agreement{metric="agreement_pct_24h"}` | Double | `metric` | Rolling 24h agreement percentage | -| `validation_agreement{metric="agreements_1h"}` | Int64 | `metric` | Agreed validations in 1h window | -| `validation_agreement{metric="missed_1h"}` | Int64 | `metric` | Missed validations in 1h window | -| `validation_agreement{metric="agreements_24h"}` | Int64 | `metric` | Agreed validations in 24h window | -| `validation_agreement{metric="missed_24h"}` | Int64 | `metric` | Missed validations in 24h window | +| `validation_agreement{metric="agreement_pct_7d"}` | Double | `metric` | Rolling 7-day agreement percentage | +| `validation_agreement{metric="agreements_1h"}` | Double | `metric` | Agreed validations in 1h window | +| `validation_agreement{metric="missed_1h"}` | Double | `metric` | Missed validations in 1h window | +| `validation_agreement{metric="agreements_24h"}` | Double | `metric` | Agreed validations in 24h window | +| `validation_agreement{metric="missed_24h"}` | Double | `metric` | Missed validations in 24h window | +| `validation_agreement{metric="agreements_7d"}` | Double | `metric` | Agreed validations in the 7-day window | +| `validation_agreement{metric="missed_7d"}` | Double | `metric` | Missed validations in the 7-day window | Data source: `ValidationTracker` class with 8s grace period and 5m late repair window. +> **Every value on this instrument is a double.** The family is one +> `CreateDoubleObservableGauge` (`src/xrpld/telemetry/MetricsRegistry.cpp:1576`), +> so the integral counts are cast to `double` before `Observe()` — there is no +> Int64 sub-series to filter on. The same holds for `validator_health`, +> `peer_quality` and `state_tracking` below; an earlier revision of these four +> tables split the Type column between Int64 and Double, which the code does not +> do. +> +> The 7-day window is `ValidationTracker::kWindow7d` = 168 hours +> (`src/xrpld/telemetry/ValidationTracker.h:311`) and is observed alongside the 1h +> and 24h windows at `MetricsRegistry.cpp:1606-1609`. Panels exist on _Validator +> Health_ (`validator-health`): "Agreement % (7d)" and "Agreements vs Missed +> (7d)". + #### Validator Health (Observable Gauge — `validator_health`) | Prometheus Metric | Type | Labels | Description | | ---------------------------------------------- | ------ | -------- | ------------------------------ | -| `validator_health{metric="amendment_blocked"}` | Int64 | `metric` | 1 if amendment-blocked, else 0 | -| `validator_health{metric="unl_blocked"}` | Int64 | `metric` | 1 if UNL-blocked, else 0 | +| `validator_health{metric="amendment_blocked"}` | Double | `metric` | 1 if amendment-blocked, else 0 | +| `validator_health{metric="unl_blocked"}` | Double | `metric` | 1 if UNL-blocked, else 0 | | `validator_health{metric="unl_expiry_days"}` | Double | `metric` | Days until UNL list expires | -| `validator_health{metric="validation_quorum"}` | Int64 | `metric` | Validation quorum threshold | +| `validator_health{metric="validation_quorum"}` | Double | `metric` | Validation quorum threshold | + +Single `CreateDoubleObservableGauge` at `MetricsRegistry.cpp:1200`. #### Peer Quality (Observable Gauge — `peer_quality`) | Prometheus Metric | Type | Labels | Description | | ------------------------------------------------- | ------ | -------- | ------------------------------------ | | `peer_quality{metric="peer_latency_p90_ms"}` | Double | `metric` | P90 peer latency in milliseconds | -| `peer_quality{metric="peers_insane_count"}` | Int64 | `metric` | Peers with diverged tracking status | +| `peer_quality{metric="peers_insane_count"}` | Double | `metric` | Peers with diverged tracking status | | `peer_quality{metric="peers_higher_version_pct"}` | Double | `metric` | % of peers on newer xrpld version | -| `peer_quality{metric="upgrade_recommended"}` | Int64 | `metric` | 1 if >60% of peers are newer version | +| `peer_quality{metric="upgrade_recommended"}` | Double | `metric` | 1 if >60% of peers are newer version | + +Single `CreateDoubleObservableGauge` at `MetricsRegistry.cpp:1249`. #### Ledger Economy (Observable Gauge — `ledger_economy`) @@ -1554,10 +1794,12 @@ Data source: `ValidationTracker` class with 8s grace period and 5m late repair w | Prometheus Metric | Type | Labels | Description | | -------------------------------------------------------- | ------ | -------- | -------------------------------------- | -| `state_tracking{metric="state_value"}` | Int64 | `metric` | Numeric state 0-6 (see encoding below) | +| `state_tracking{metric="state_value"}` | Double | `metric` | Numeric state 0-6 (see encoding below) | | `state_tracking{metric="time_in_current_state_seconds"}` | Double | `metric` | Duration in current state | -State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (FULL + validating), 6=proposing (FULL + proposing). +Single `CreateDoubleObservableGauge` at `MetricsRegistry.cpp:1466`. + +State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (FULL + validating), 6=proposing (FULL + proposing). Values 0-4 are `OperatingMode` cast to double (`include/xrpl/server/NetworkOPs.h:60-66`); 5 and 6 are the FULL-only refinements at `MetricsRegistry.cpp:1483-1498`. **The range is 0-6, not 0-7** — there is no seventh state. #### Storage Detail (Observable Gauge — `storage_detail`) @@ -1566,11 +1808,11 @@ State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full | `storage_detail{metric="stored_object_bytes"}` | Int64 | `metric` | Cumulative object-payload bytes written (not on-disk size) | > **`stored_object_bytes` is not a file size.** It observes `getStoreSize()` -> (`src/xrpld/telemetry/MetricsRegistry.cpp:1511`), which sums the object payloads +> (`src/xrpld/telemetry/MetricsRegistry.cpp:1557`), which sums the object payloads > this process has written. It therefore excludes NuDB's keys, bucket padding and > log, and it resets when the process restarts while the files on disk do not. > `node_written_bytes` on the `nodestore_state` gauge calls the same accessor -> (`MetricsRegistry.cpp:836`), so the two series are equal by construction and any +> (`MetricsRegistry.cpp:860`), so the two series are equal by construction and any > write-amplification ratio built from the pair is a constant 1.0. To size the store > on disk, stat the backend's files; no metric reports it today. > @@ -1588,6 +1830,24 @@ State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full | `validations_checked_total` | Counter | Network validations observed | LedgerMaster.cpp | | `state_changes_total` | Counter | Operating mode transitions | NetworkOPs.cpp | +> **Known issue — `ledgers_closed_total` has a dead second producer.** The +> instrument is created twice. `MetricsRegistry::registerCounters()` eagerly +> creates it as the member `ledgersClosedCounter_` +> (`src/xrpld/telemetry/MetricsRegistry.cpp:369-370`), and its only mutator, +> `MetricsRegistry::incrementLedgersClosed()` +> (declared `MetricsRegistry.h:591`, defined `MetricsRegistry.cpp:1686`), has +> **zero callers** — the header says so itself at `MetricsRegistry.h:584-588`. +> The value operators actually see comes from the single live increment, +> the `XRPL_METRIC_COUNTER_INC` call site in +> `RCLConsensus::Adaptor::doAccept()` (`src/xrpld/app/consensus/RCLConsensus.cpp:749`). +> +> No metric is wrong and nothing double-counts: the dead member never adds to the +> series. The cost is a redundant eagerly-created instrument plus a misleading API +> that looks like the increment path. **Code follow-up**: delete +> `incrementLedgersClosed()` and `ledgersClosedCounter_` once the macro path is +> considered proven, per the header note. Tracked here rather than fixed in a doc +> pass — the doc is not reworded to imply the member is used. + Lifetime tallies exported as monotonic **ObservableCounters** (not synchronous counters), observed from an existing cumulative source each collection cycle: @@ -1603,19 +1863,40 @@ counters), observed from an existing cumulative source each collection cycle: > decrease) and additive (`agreements_total + missed_total` = ledgers reconciled). The > repair-aware, windowed view remains on `validation_agreement{metric="…"}`. -#### Span Attribute Enrichments (Phases 2-4) +#### Span Attribute Enrichments (Phases 2-4): REMOVED -| Span Name | New Attribute | Type | Source | -| --------------------------- | ------------------------------------ | ------ | ------------------------ | -| `rpc.command.*` | `xrpl.node.amendment_blocked` | bool | Phase 2 — RPCHandler.cpp | -| `rpc.command.*` | `xrpl.node.server_state` | string | Phase 2 — RPCHandler.cpp | -| `tx.receive` | `xrpl.peer.version` | string | Phase 3 — PeerImp.cpp | -| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Phase 4 — RCLConsensus | -| `consensus.validation.send` | `xrpl.validation.full` | bool | Phase 4 — RCLConsensus | -| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | Phase 4 — PeerImp.cpp | -| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | Phase 4 — PeerImp.cpp | -| `consensus.accept` | `xrpl.consensus.validation_quorum` | int64 | Phase 4 — RCLConsensus | -| `consensus.accept` | `xrpl.consensus.proposers_validated` | int64 | Phase 4 — RCLConsensus | +This section used to list nine dotted `xrpl.node.*` / `xrpl.peer.*` / +`xrpl.validation.*` / `xrpl.consensus.*` **span** attributes. **None of them +exists.** A grep for `xrpl.node.`, `xrpl.peer.`, `xrpl.validation.` and +`xrpl.consensus.` across non-test `src/` and `include/` returns nothing, and the +table also contradicted this document's own rule in +[§1.2](#12-complete-attribute-inventory-bareunderscore-keys): dotted keys are +OTel **resource** attributes, never span attributes. + +The dotted form was dropped by the 2026-05-13 naming redesign, in three commits: + +| Commit | Scope | +| ------------ | ------------------------------------------------------------------------------------------------------------- | +| `e339ba1f6b` | tx / txq — dropped the `xrpl..` prefix (phase-3) | +| `46d1012ad4` | consensus — dropped the `xrpl.consensus.` prefix (phase-4) | +| `9e27120a15` | ledger / peer — simplified the keys, updated dashboards (phase-6) | + +What the code emits today, and where it is documented: + +| Old dotted key (never emitted) | Live equivalent | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `xrpl.peer.version` | `peer_version` — see [§Transaction Attributes](#transaction-attributes) | +| `xrpl.validation.ledger_hash`, `xrpl.peer.validation.ledger_hash` | one bare `ledger_hash` on both `consensus.validation.send` and `peer.validation.receive` | +| `xrpl.validation.full`, `xrpl.peer.validation.full` | one bare `full_validation` on both of those spans | +| `xrpl.consensus.validation_quorum` | `quorum`, on `consensus.accept` only | +| `xrpl.node.amendment_blocked` | **not a span attribute at all** — only the metric `validator_health{metric="amendment_blocked"}` (`MetricsRegistry.cpp:1216`) | +| `xrpl.node.server_state` | **not a span attribute at all** — only the metric `server_info{metric="server_state"}` (`MetricsRegistry.cpp:1014`) | +| `xrpl.consensus.proposers_validated` | **never implemented** in any form | + +The identical nine-row list was deleted from +`docker/telemetry/workload/expected_spans.json` by commit `cb9fce6890` for the +same reason. Anything still asserting these keys — a dashboard filter, a TraceQL +query, an alert — matches nothing and should be pointed at the live keys above. ### New Grafana Dashboards (Phase 9) @@ -1663,20 +1944,24 @@ counters), observed from an existing cumulative source each collection cycle: ## 6. Known Issues -| Issue | Impact | Status | -| --------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | -| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | -| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | -| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | -| `handler="other"` mixes several producers | Cannot separate `GetConsL1` from `GetConsL2` | By design — the cardinality bound; see [§Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram) | -| `overhead_cluster_*` is always zero | 8 dashboard panel references are flatlines by construction; cluster traffic is counted as `unknown` | **NOT IMPLEMENTED** — see [§6.0](#60-mtcluster-is-counted-as-unknown-not-implemented) | -| `squelch_ignored_bytes_in/out` always read zero | Only the `_messages_*` pair carries signal for this category | **NOT IMPLEMENTED** — see [§6.1](#61-squelch_ignored-byte-counts-not-implemented) | -| `total_bytes_in` and `total_bytes_out` use different size bases | In/out byte totals are not directly comparable when compression is on | **NOT IMPLEMENTED** — see [§6.2](#62-inboundoutbound-byte-basis-asymmetry-not-implemented) | -| `overhead` conflates `mtPING` with `mtSTATUS_CHANGE` | Keepalive traffic cannot be isolated from status-change traffic | **NOT IMPLEMENTED** — needs a new category; see [§6.3](#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) | -| No metrics for ping RTT distribution, ping timeouts, or `mtENDPOINTS` | Peer keepalive and discovery health are not observable | **NOT IMPLEMENTED** — see [§6.3](#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) | -| 11 of 13 peer message families have no spans | `02` §2.3.2 catalogs `peer.message.*`, `peer.connect`, `peer.disconnect` that were never built | **NOT IMPLEMENTED** — see [§6.4](#64-peer-span-coverage-gap-not-implemented) | -| PeerFinder exports 2 of ~17 available slot/cache readings | Slot pressure, connection churn and discovery-cache health are not observable | **NOT IMPLEMENTED** — see [§6.5](#65-peerfinder-slot-and-cache-metrics-not-implemented) | +| Issue | Impact | Status | +| --------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | +| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | +| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | +| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | +| `handler="other"` mixes several producers | Cannot separate `GetConsL1` from `GetConsL2` | By design — the cardinality bound; see [§Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram) | +| `overhead_cluster_*` is always zero | 8 dashboard panel references are flatlines by construction; cluster traffic is counted as `unknown` | **NOT IMPLEMENTED** — see [§6.0](#60-mtcluster-is-counted-as-unknown-not-implemented) | +| `squelch_ignored_bytes_in/out` always read zero | Only the `_messages_*` pair carries signal for this category | **NOT IMPLEMENTED** — see [§6.1](#61-squelch_ignored-byte-counts-not-implemented) | +| `total_bytes_in` and `total_bytes_out` use different size bases | In/out byte totals are not directly comparable when compression is on | **NOT IMPLEMENTED** — see [§6.2](#62-inboundoutbound-byte-basis-asymmetry-not-implemented) | +| `overhead` conflates `mtPING` with `mtSTATUS_CHANGE` | Keepalive traffic cannot be isolated from status-change traffic | **NOT IMPLEMENTED** — needs a new category; see [§6.3](#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) | +| No metrics for ping RTT distribution, ping timeouts, or `mtENDPOINTS` | Peer keepalive and discovery health are not observable | **NOT IMPLEMENTED** — see [§6.3](#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) | +| 11 of 13 peer message families have no spans | `02` §2.3.2 catalogs `peer.message.*`, `peer.connect`, `peer.disconnect` that were never built | **NOT IMPLEMENTED** — see [§6.4](#64-peer-span-coverage-gap-not-implemented) | +| PeerFinder exports 2 of ~17 available slot/cache readings | Slot pressure, connection churn and discovery-cache health are not observable | **NOT IMPLEMENTED** — see [§6.5](#65-peerfinder-slot-and-cache-metrics-not-implemented) | +| `ledger_history_mismatch_total` has two producers in one family | A bare `sum()` double-counts every mismatch; one series carries no `reason` label | **CODE BUG** — retire one producer; group by `reason` meanwhile. See [§TxQ Admission and Ledger Mismatch](#txq-admission-and-ledger-mismatch-synchronous-counters) | +| `overlay_peer_disconnects_charges` never existed | The documented selector matches nothing; use `server_info{metric="peer_disconnects_resources"}` | **NOT IMPLEMENTED** — see [§2.1](#21-gauges) | +| Nine dotted `xrpl..*` span attributes never shipped | TraceQL filters and harness assertions on the dotted keys match nothing | **NOT IMPLEMENTED** — renamed to bare keys; see [§Span Attribute Enrichments](#span-attribute-enrichments-phases-2-4-removed) | +| `node_writes_duration_us` has no dashboard panel | Cumulative write latency is exported and linted, but never charted | Open follow-up — see [§Extended NodeStore Metrics](#extended-nodestore-metrics-additions-to-existing-nodestore_state) | ### 6.0 `mtCLUSTER` is counted as `unknown`: NOT IMPLEMENTED @@ -1778,9 +2063,9 @@ messages are traced. **Status**: NOT IMPLEMENTED. The span catalog in `02` §2.3.2 is a design inventory, not a statement of what emits; §2.3.2 now marks which entries are -live. Instrumenting the remaining families would change the "~37 spans" count -asserted in [§1.1](#11-complete-span-inventory-37-spans) and in -`docker/telemetry/workload/expected_spans.json`, so it is scoped as its own +live. Instrumenting the remaining families would change the **41 span families** +counted in [§1.1](#11-complete-span-inventory-41-spans) and the **40** catalogued +in `docker/telemetry/workload/expected_spans.json`, so it is scoped as its own change rather than folded into a metric task. ### 6.5 PeerFinder slot and cache metrics: NOT IMPLEMENTED diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md index b7289d5004..f2855643a8 100644 --- a/OpenTelemetryPlan/OpenTelemetryPlan.md +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -102,12 +102,16 @@ flowchart TB | **2** | [Design Decisions](./02-design-decisions.md) | SDK selection, exporters, span naming, attributes, context propagation | | **3** | [Implementation Strategy](./03-implementation-strategy.md) | Directory structure, key principles, performance optimization | | **5** | [Configuration Reference](./05-configuration-reference.md) | xrpld config, CMake integration, Collector configurations | -| **6** | [Implementation Phases](./06-implementation-phases.md) | 5-phase timeline, tasks, risks, success metrics | +| **6** | [Implementation Phases](./06-implementation-phases.md) | 11-phase timeline, tasks, risks, success metrics | | **7** | [Observability Backends](./07-observability-backends.md) | Backend selection guide and production architecture | | **8** | [Appendix](./08-appendix.md) | Glossary, references, version history | | **9** | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards | | **Sec** | [Securing the OTel Pipeline](./secure-OTel.md) | Threat model and hardening (mTLS, peer trace-context validation) | +> Note there is no document 4: `04-code-samples.md` was removed during the +> rollout, and the numbering was left as-is rather than renumbering every +> cross-reference in the chain. + --- ## 0. Tracing Fundamentals @@ -136,9 +140,9 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). -Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. +Span naming follows a hierarchical `.` convention (e.g., `rpc.command.server_info`, `tx.process`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. -**Data Collection & Privacy**: Telemetry collects only operational metadata (timing, counts, hashes) — never sensitive content (private keys, balances, amounts, raw payloads). Privacy protection includes account hashing, configurable redaction, sampling, and collector-level filtering. Node operators retain full control over telemetry configuration. +**Data Collection & Privacy**: Telemetry collects only operational metadata (timing, counts, hashes) — never sensitive content (private keys, balances, amounts, raw payloads). Account addresses are hashed **unconditionally** by the SDK helper and hashed again at the collector; there is no redaction config key and therefore no insecure-by-default state. Trace volume is _not_ reduced on the node (head sampling is fixed at 100%); reduction, where wanted, is a collector-side tail-sampling decision. Node operators control which subsystems are traced via the `[telemetry]` per-component toggles. ➡️ **[Read full Design Decisions](./02-design-decisions.md)** @@ -146,9 +150,9 @@ Span naming follows a hierarchical `.` convention (e.g., ` ## 3. Implementation Strategy -The telemetry code is organized under `include/xrpl/telemetry/` for headers and `src/libxrpl/telemetry/` for implementation. Key principles include RAII-based span management via `SpanGuard` (with `discard()` for dropping unwanted spans), a `FilteringSpanProcessor` that intercepts `OnEnd()` to prevent discarded spans from entering the export pipeline, conditional compilation with `XRPL_ENABLE_TELEMETRY`, and minimal runtime overhead through batch processing and efficient sampling. +The telemetry code is organized under `include/xrpl/telemetry/` for headers, `src/libxrpl/telemetry/` for implementation, and `src/xrpld/telemetry/` for the native-metrics module added in Phases 7 and 9. Key principles include RAII-based span management via `SpanGuard` (with `discard()` for dropping unwanted spans), a `FilteringSpanProcessor` that intercepts `OnEnd()` to prevent discarded spans from entering the export pipeline, conditional compilation behind the `XRPL_ENABLE_TELEMETRY` compile definition (set by the CMake `telemetry` option, which defaults to **ON** — build it out with `-Dtelemetry=OFF`), and minimal runtime overhead through batch processing. -Performance optimization strategies include head sampling fixed at 100% (intentionally not configurable, so trace keep/drop decisions stay coherent across nodes), tail-based sampling at the collector for errors and slow traces to reduce volume, batch export to reduce network overhead, and conditional instrumentation that compiles to no-ops when disabled. +Performance optimization strategies include head sampling fixed at 100% (intentionally not configurable, so trace keep/drop decisions stay coherent across nodes), optional tail-based sampling at the collector to reduce stored volume (not enabled in the base stack — the only shipped policy is a 0.5% probabilistic one in the Grafana Cloud overlay), batch export to reduce network overhead, and conditional instrumentation that compiles to no-ops when disabled. ➡️ **[Read full Implementation Strategy](./03-implementation-strategy.md)** @@ -158,9 +162,19 @@ Performance optimization strategies include head sampling fixed at 100% (intenti > **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring -Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, TLS/mTLS, batch tuning, and component-level filtering. Exporter selection is _not_ configurable — OTLP/HTTP is the only transport. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration uses the `telemetry` option (default **ON**) for compile-time control. -OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. +Endpoints are spread across **three** keys in two sections, not one "traces and metrics" pair: + +| Signal | Key | Default | Source | +| ---------------------------------------------------- | ------------------------------ | ---------------------------------- | --------------------------- | +| Traces | `[telemetry] endpoint` | `http://localhost:4318/v1/traces` | `TelemetryConfig.cpp:36,61` | +| Native metrics (`XRPL_METRIC_*` / `MetricsRegistry`) | `[telemetry] metrics_endpoint` | `http://localhost:4318/v1/metrics` | `Application.cpp:1670` | +| `beast::insight` metrics (`server=otel`) | `[insight] endpoint` | `http://localhost:4318/v1/metrics` | `CollectorManager.cpp:50` | + +`[telemetry]` itself has exactly **one** `endpoint` key, and it is traces-only. + +The repo ships one collector config (`docker/telemetry/otel-collector-config.yaml`, three pipelines: traces, metrics, logs) plus a Grafana Cloud overlay that adds 0.5% tail sampling. A six-service Docker Compose stack — collector, Tempo, Loki, Prometheus, Grafana, renderer — gives a complete local environment. ➡️ **[View full Configuration Reference](./05-configuration-reference.md)** @@ -168,20 +182,31 @@ OpenTelemetry Collector configurations are provided for development and producti ## 6. Implementation Phases -The implementation spans 13 weeks across 8 phases: +The plan was originally scoped at **13 weeks across 8 phases** — the table below +is that original scope. As delivered it grew to **11 phases through week 20**; +Phases 9-11 were added after the original plan was written. See +[06-implementation-phases.md §6.12.6](./06-implementation-phases.md) for the +authoritative per-phase status, and treat the eight rows below as the +originally-planned subset rather than the current timeline: -| Phase | Duration | Focus | Key Deliverables | -| ----- | ----------- | --------------------- | ----------------------------------------------------------- | -| 1 | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration | -| 2 | Weeks 3-4 | RPC Tracing | HTTP context extraction, Handler instrumentation | -| 3 | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation | -| 4 | Weeks 7-8 | Consensus Tracing | Round spans, Proposal/validation tracing | -| 5 | Week 9 | Documentation | Runbook, Dashboards, Training | -| 6 | Week 10 | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards | -| 7 | Weeks 11-12 | Native OTel Metrics | OTelCollector impl, OTLP metrics export, StatsD deprecation | -| 8 | Week 13 | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking | +| Phase | Duration | Focus | Key Deliverables | +| ----- | ----------- | --------------------- | --------------------------------------------------------- | +| 1 | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration | +| 2 | Weeks 3-4 | RPC Tracing | HTTP context extraction, Handler instrumentation | +| 3 | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation | +| 4 | Weeks 7-8 | Consensus Tracing | Round spans, Proposal/validation tracing | +| 5 | Week 9 | Documentation | Runbook, Dashboards, Training | +| 6 | Week 10 | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards | +| 7 | Weeks 11-12 | Native OTel Metrics | OTelCollector impl, OTLP metrics export (StatsD retained) | +| 8 | Week 13 | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking | -**Total Effort**: 65.1 developer-days with 2 developers +Delivered beyond the original scope: **Phase 9** (weeks 14-15, internal metric +instrumentation gap fill), **Phase 10** (weeks 16-17, synthetic workload +generation and telemetry validation) and **Phase 11** (weeks 18-20, third-party +data-collection pipelines). + +**Total Effort**: 65.1 developer-days with 2 developers, for the eight +originally-planned phases only. ➡️ **[View full Implementation Phases](./06-implementation-phases.md)** @@ -191,9 +216,9 @@ The implementation spans 13 weeks across 8 phases: > **APM** = Application Performance Monitoring | **GCS** = Google Cloud Storage -Grafana Tempo is recommended for all environments due to its cost-effectiveness and Grafana integration, while Elastic APM is ideal for organizations with existing Elastic infrastructure. +Grafana Tempo is recommended for all environments due to its cost-effectiveness and Grafana integration, and it is the only backend this repo provisions. Elastic APM remains a reasonable choice for organizations with existing Elastic infrastructure, but nothing here configures it. -The recommended production architecture uses a gateway collector pattern with regional collectors performing tail-based sampling, routing traces to multiple backends (Tempo for primary storage, Elastic for log correlation, S3/GCS for long-term archive). +The recommended production architecture uses a gateway collector pattern with regional collectors performing tail-based sampling, routing traces to multiple backends (Tempo for primary storage, Elastic for log correlation, S3/GCS for long-term archive). Note that several subsections of doc 7 predate the shipped dashboards and alert rules and are marked superseded in place, pointing at [09-data-collection-reference.md](./09-data-collection-reference.md) and `docs/telemetry-runbook.md`. ➡️ **[View Observability Backend Recommendations](./07-observability-backends.md)** @@ -209,7 +234,7 @@ The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, refe ## 9. Data Collection Reference -A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld. Covers all 16 OpenTelemetry spans with their 22 attributes, all StatsD metrics (gauges, counters, histograms, overlay traffic), SpanMetrics-derived Prometheus metrics, and all 10 Grafana dashboards. Includes Tempo search guides and Prometheus query examples. +A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld: the OpenTelemetry span inventory with per-span attributes, the `beast::insight` and native `XRPL_METRIC_*` instruments (gauges, counters, histograms, overlay traffic), the SpanMetrics-derived Prometheus metrics, and the **15** Grafana dashboards. Includes Tempo search guides and Prometheus query examples. Consult that document rather than this index for any count — it tracks the code, this summary does not. ➡️ **[View Data Collection Reference](./09-data-collection-reference.md)** diff --git a/OpenTelemetryPlan/Phase10_taskList.md b/OpenTelemetryPlan/Phase10_taskList.md index a7716f225b..28a1cd28fb 100644 --- a/OpenTelemetryPlan/Phase10_taskList.md +++ b/OpenTelemetryPlan/Phase10_taskList.md @@ -22,10 +22,13 @@ Before Phases 1-9 can be considered production-ready, we need proof that: -1. All 16 spans fire with correct attributes under real transaction workloads +1. Every emitted span fires with its required attributes under real transaction + workloads (the "16 spans / 22 attributes" figures below are stale; the harness + derives both totals from `expected_spans.json`) 2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values 3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki -4. All 10 Grafana dashboards render meaningful data (no empty panels) +4. The 14 harness-asserted Grafana dashboards render meaningful data (no empty + panels); 15 are on disk 5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory) 6. The telemetry stack survives sustained load without data loss or queue backpressure @@ -37,25 +40,42 @@ Before Phases 1-9 can be considered production-ready, we need proof that: **What to do**: -- Create `docker/telemetry/docker-compose.workload.yaml`: - - 5 xrpld validator nodes with UNL configured for each other - - All telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel` - - Full OTel stack: Collector, Tempo, Prometheus, Loki, Grafana - - Shared network with service discovery +- Create `docker/telemetry/docker-compose.workload.yaml` — **as shipped this file + holds only the observability backend**: `otel-collector`, `tempo`, + `prometheus`, `loki`, `grafana`. It contains **no xrpld services**. + - Shared network (`workload-net`) with service discovery -- Each node should: - - Generate validator keys at startup - - Configure all 5 nodes in its UNL - - Enable all trace categories including `trace_peer=1` - - Write logs to a file tailed by the OTel Collector filelog receiver +- The 5 validators are **native `xrpld` processes**, not containers. + `docker/telemetry/workload/run-full-validation.sh` (`NUM_NODES=5`) generates + keys, writes a per-node `xrpld.cfg`, and launches each node on + `127.0.0.1` with sequential RPC / WS / peer ports. Each node: + - Gets its validator key from `generate-validator-keys.sh` + - Lists the other 4 nodes in `ips_fixed` + - Has all telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel` + - Enables all trace categories including `trace_peer=1` + - Writes logs to a file tailed by the OTel Collector filelog receiver -- Include a `Makefile` target: `make telemetry-workload-up` / `make telemetry-workload-down` +- ❌ **`make telemetry-workload-up` / `make telemetry-workload-down` were never + implemented.** There is no `Makefile` anywhere in the repository. The entry + point is `run-full-validation.sh` (with `--profile`, `--nodes`, + `--skip-loki`, `--skip-regression`, `--with-benchmark`). The node-count flag is + spelled `--nodes`, **not** `--num-nodes` — `run-full-validation.sh:80` (usage) + and `:100` (the `case` arm). `NUM_NODES` is the internal shell variable it + assigns to. **Key files**: -- New: `docker/telemetry/docker-compose.workload.yaml` +- New: `docker/telemetry/docker-compose.workload.yaml` (backend only) - New: `docker/telemetry/workload/generate-validator-keys.sh` -- New: `docker/telemetry/workload/xrpld-validator.cfg.template` +- New: `docker/telemetry/workload/run-full-validation.sh` — writes each node's + cfg **inline** via a heredoc at `run-full-validation.sh:242` + (`cat >"$NODE_DIR/xrpld.cfg" < 0, < 60s) **Metric validation** (queries Prometheus API): - - Assert all SpanMetrics-derived metrics are non-zero: `traces_span_metrics_calls_total`, `traces_span_metrics_duration_milliseconds_bucket` - - Assert all StatsD metrics are non-zero: `xrpld_LedgerMaster_Validated_Ledger_Age`, `xrpld_Peer_Finder_Active_*`, etc. - - Assert all Phase 9 metrics are non-zero: `xrpld_nodestore_*`, `xrpld_cache_*`, `xrpld_txq_*`, `xrpld_rpc_method_*`, `xrpld_object_count`, `xrpld_load_factor*` + - Assert all SpanMetrics-derived metrics are non-zero: `span_calls_total`, + `span_duration_milliseconds_bucket` (the connector's `namespace` is `span`, + not `traces_span_metrics` — `otel-collector-config.yaml:113-114`) + - Assert the insight-sourced metrics are non-zero: `ledgermaster_validated_ledger_age`, + `peer_finder_active_{inbound,outbound}_peers`, etc. — all lowercase, no + `xrpld_` prefix (`77f35c03db` removed the prefix and lowercased names) + - Assert all Phase 9 metrics are non-zero: `nodestore_state`, `cache_metrics`, + `txq_metrics`, `rpc_method_{started,finished,errored}_total`, `object_count`, + `load_factor_metrics` - Assert metric label cardinality is within bounds **Log-trace correlation validation** (queries Loki API): @@ -139,7 +174,9 @@ Before Phases 1-9 can be considered production-ready, we need proof that: - Assert Grafana derived field links are functional **Dashboard validation**: - - For each of the 10 Grafana dashboards, query the dashboard API and assert no panels show "No data" + - For each dashboard, query the dashboard API and assert no panels show "No + data". There are **15 dashboards on disk**; the harness asserts **14** — + `log-derived-insights` is provisioned but unasserted. - Output: JSON report with pass/fail per check, suitable for CI. @@ -231,12 +268,17 @@ Before Phases 1-9 can be considered production-ready, we need proof that: ## Exit Criteria -- [ ] 5-node validator cluster starts and reaches consensus in docker-compose +- [ ] 5-node validator cluster starts and reaches consensus — as native `xrpld` + processes driven by `run-full-validation.sh`, not from docker-compose - [ ] RPC load generator fires all traced RPC commands at configurable rates - [ ] Transaction submitter generates 6+ transaction types at configurable TPS -- [ ] Validation suite confirms all 16 spans, 22 attributes, 300+ metrics are present -- [ ] Log-trace correlation validated end-to-end (Loki ↔ Tempo) -- [ ] All 10 Grafana dashboards render data (no empty panels) +- [ ] Validation suite confirms the full span / attribute / metric inventory + (totals computed dynamically from `expected_spans.json` / + `expected_metrics.json`, not the stale 16 / 22 figures) +- [ ] Log-trace correlation validated end-to-end (Loki ↔ Tempo) — implemented, + but CI runs with `--skip-loki`, so it is not gated +- [ ] All 14 harness-asserted Grafana dashboards render data (no empty panels); + 15 on disk - [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead - [ ] CI workflow runs validation on telemetry branch changes - [ ] Validation report output is CI-parseable (JSON with exit codes) diff --git a/OpenTelemetryPlan/Phase11_taskList.md b/OpenTelemetryPlan/Phase11_taskList.md index 7429063984..15d1510a3f 100644 --- a/OpenTelemetryPlan/Phase11_taskList.md +++ b/OpenTelemetryPlan/Phase11_taskList.md @@ -1,6 +1,17 @@ # Phase 11: Third-Party Data Collection Pipelines — Task List -> **Status**: Future Enhancement +> **Status**: Not started — 0 of 13 tasks complete (`grep -c '^## Task 11\.'` = 13: +> Tasks 11.1 through 11.13). Verified against the tree: +> no `.go` files exist anywhere, `docker/telemetry/otel-rippled-receiver/` does +> not exist, `docker/telemetry/prometheus/` does not exist (so no +> `rippled-alerts.yml`), and no `network-topology` / `dex-amm` dashboards are +> present under `docker/telemetry/grafana/dashboards/`. **No Phase 11 work has +> been done, so no task box below may be ticked.** +> +> One **prerequisite** box is ticked, and only one: Task 11.12's +> "`state_tracking` gauge implemented (Task 7.12)". That is an upstream +> dependency satisfied by Phase 7/9 code, not Phase 11 work — see the citation +> there. > > **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes. > @@ -287,7 +298,35 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h ## Task 11.8: Prometheus Alerting Rules -**Objective**: Create production-ready alerting rules for the metrics exported by this receiver. +**Objective**: Create production-ready alerting rules for the `xrpl_*` metrics +exported by this receiver. + +> **Scope note — do not duplicate Phase 9.** Phase 9 already ships provisioned +> **Grafana** alerting at +> `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` +> — 13 rules in 5 groups, 2 contact points (`xrpld-default` Slack, +> `xrpld-critical` Slack + email), and a nested notification policy keyed on +> `severity = critical`. Four of the rules below overlap it: +> +> | Rule here | Addressed by (Phase 9) | Coverage | +> | ------------------- | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +> | `XRPLServerNotFull` | `NodeNotFull` (group `xrpld-node-state`) | Full | +> | `XRPLLedgerStale` | `ValidatedLedgerStale` (group `xrpld-consensus`) | **Partial** — Phase 9: `ledgermaster_validated_ledger_age > 60` for 5m; the external shape is `> 30` for 1m | +> | `XRPLHighIOLatency` | `NodeStoreIOLatencyHigh` (group `xrpld-jobqueue`) | **Partial** — Phase 9: p95 of `ios_latency_milliseconds_bucket` **> 1000 ms for 10m**; external: **> 50 for 1m** | +> | `XRPLStateFlapping` | `NodeStateFlapping` (group `xrpld-node-state`) | Full | +> +> The remaining 8 (`XRPLAmendmentBlocked`, `XRPLNoPeers`, +> `XRPLUnsupportedAmendmentMajority`, `XRPLLowPeerCount`, `XRPLHighLoadFactor`, +> `XRPLSlowConsensus`, `XRPLValidatorListExpiring`, `XRPLClockDrift`) are +> genuinely new. Note the two sets watch different metric surfaces — the Phase 9 +> rules fire on xrpld's own OTLP metrics, these on the receiver's `xrpl_*` +> metrics — so if both are kept, dedupe the notification policy to avoid +> double-paging on the same underlying condition. +> +> `docker/telemetry/prometheus/` does not exist today. Prefer extending the +> Phase 9 Grafana provisioning tree over introducing a second, Prometheus-native +> alerting mechanism; if a `prometheus/` tree is added anyway, say explicitly in +> its header which alerts it owns. **What to do**: @@ -360,9 +399,22 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h **Objective**: Create 4 new dashboards for the data exported by the receiver. +> **UID COLLISION — pick a different uid.** Phase 9 already ships +> `docker/telemetry/grafana/dashboards/validator-health.json` with +> **uid `validator-health`** (17 panels, backed by xrpld's own +> `validation_agreement` / `validator_health` / `state_tracking` OTLP metrics). +> Provisioning a second dashboard with the same uid makes Grafana overwrite one +> with the other — whichever the provisioner loads last wins, silently. Use a +> distinct uid such as `validator-health-external` (and a distinct filename), the +> same way this task already disambiguates Fee Market as +> `xrpld-fee-market-external` against Phase 9's `fee-market`. Also check +> `peer-quality`, `fee-market`, `job-queue` and `node-health` before adding any +> further uid. + **What to do**: -- **Validator Health** (`validator-health`): +- **Validator Health** (`validator-health-external` — **not** `validator-health`, + see the collision note above): - Server state timeline, state duration breakdown - Proposer count trend, converge time trend, validation quorum - Validator list expiration countdown @@ -386,10 +438,16 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h **Key files**: -- New: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` -- New: `docker/telemetry/grafana/dashboards/rippled-network-topology.json` -- New: `docker/telemetry/grafana/dashboards/rippled-fee-market-external.json` -- New: `docker/telemetry/grafana/dashboards/rippled-dex-amm.json` +- New: `docker/telemetry/grafana/dashboards/validator-health-external.json` + (**must not** reuse Phase 9's `validator-health.json` / uid `validator-health`) +- New: `docker/telemetry/grafana/dashboards/network-topology.json` +- New: `docker/telemetry/grafana/dashboards/fee-market-external.json` + (Phase 9 owns `fee-market.json` / uid `fee-market`) +- New: `docker/telemetry/grafana/dashboards/dex-amm.json` + +> Filenames drop the `rippled-` prefix: `145b1469d6` and `25868f2740` renamed +> every dashboard to bare names with bare uids, so no `rippled-*.json` path +> exists in the tree. --- @@ -446,20 +504,55 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h > **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards). > **Downstream**: None — terminal task in the parity chain. -**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the `xrpld_*` internal metrics. +**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the internal metrics. + +> **4 of the 18 are addressed by Phase 9** — 2 fully, 2 only partially. Extend, +> do not blindly re-create: +> +> | Rule here | Addressed by (Phase 9) | Coverage | +> | ------------------ | ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +> | Unhealthy State | `NodeNotFull` (group `xrpld-node-state`) | Full | +> | High IO Latency | `NodeStoreIOLatencyHigh` (group `xrpld-jobqueue`, p95 of `ios_latency_milliseconds_bucket`) | **Partial** — Phase 9 fires at p95 **> 1000 ms for 10m**; the rule below wants **> 50 for 1m** (20× tighter) | +> | Job Queue Overflow | `JobQueueTxOverflow` (group `xrpld-jobqueue`, `jq_trans_overflow_total`) | Full | +> | Stale Ledger | `ValidatedLedgerStale` (group `xrpld-consensus`, `ledgermaster_validated_ledger_age`) | **Partial** — different metric: Phase 9 uses `ledgermaster_validated_ledger_age > 60` for 5m; the rule below uses `ledger_economy{metric="ledger_age_seconds"} > 30` for 1m | +> +> The two **Partial** rows are not closed. Either re-baseline the Phase 9 +> thresholds or ship the tighter variants here — do not skip them as duplicates. +> +> Remaining open work is **14 rules**, of which **3** (CPU High, Memory Critical, +> Disk Warning) need `node_exporter`, which is not in the stack. Nothing else is +> blocked: "Not Proposing" used to be listed as blocked on an unimplemented +> `state_tracking` gauge, but that gauge **ships** — see the Exit Criteria note +> below. +> +> **Metric-name translation.** Names carry **no** `xrpld_` prefix +> (`77f35c03db`), so as a rule of thumb read every `xrpld_` below as plain +> ``. **Two shapes do not follow that rule:** +> +> - **Multiplexed observable gauges.** Many readings are a `metric` **label +> value** on a shared instrument, not a metric name. `xrpld_txq_count` is +> `txq_metrics{metric="txq_count"}`; likewise `load_factor_metrics{…}`, +> `nodestore_state{…}`, `cache_metrics{…}`. The rows below that already use the +> `{metric="…"}` form (`state_tracking`, `validator_health`, +> `validation_agreement`, `server_info`, `peer_quality`, `load_factor_metrics`, +> `ledger_economy`) are correct; only drop the prefix on those. +> - **Unit-suffixed histograms** from `beast::insight`. `OTelCollectorImp` appends +> the unit, so `xrpld_ios_latency_bucket` is really +> `ios_latency_milliseconds_bucket` — the spelling used by +> `node-health.json:577` and `ledger-data-sync.json:1353`. **Critical Group** (8 rules, eval interval 10s): -| Rule | Condition | For | -| ------------------- | ------------------------------------------------------------- | --- | -| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | -| Not Proposing | `xrpld_state_tracking{metric="state_value"} < 6` | 10s | -| Unhealthy State | `xrpld_state_tracking{metric="state_value"} < 4` | 10s | -| Amendment Blocked | `xrpld_validator_health{metric="amendment_blocked"} == 1` | 1m | -| UNL Expiring | `xrpld_validator_health{metric="unl_expiry_days"} < 14` | 1h | -| High IO Latency | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50` | 1m | -| High Load Factor | `xrpld_load_factor_metrics{metric="load_factor"} > 1000` | 1m | -| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5` | 1m | +| Rule | Condition | For | +| ------------------- | ---------------------------------------------------------------- | --- | +| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `xrpld_state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `xrpld_state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `xrpld_validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `xrpld_validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, ios_latency_milliseconds_bucket) > 50` | 1m | +| High Load Factor | `xrpld_load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5` | 1m | **Network Group** (3 rules, eval interval 10s): @@ -481,19 +574,44 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h | TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | | Stale Ledger | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | -**Notification channel templates**: Email/SMTP, Discord, Slack, PagerDuty. +**Notification channel templates**: Slack and Email/SMTP already ship in Phase +9's `contactpoints.yaml` (`xrpld-default`, `xrpld-critical`). Discord and +PagerDuty templates remain open. -**Key files**: +**Key files** — extend the **Phase 9** provisioning tree. The +`docker/telemetry/grafana/alerting/` directory named in the original spec has +never existed in any commit; the real location is +`docker/telemetry/grafana/provisioning/alerting/`: -- New/extend: `docker/telemetry/grafana/alerting/alert-rules-parity.yaml` -- New: `docker/telemetry/grafana/alerting/contact-points.yaml` (template configs) -- New: `docker/telemetry/grafana/alerting/notification-policies.yaml` +- Extend: `docker/telemetry/grafana/provisioning/alerting/rules.yaml` (add groups + alongside the existing `xrpld-consensus`, `xrpld-validator`, `xrpld-jobqueue`, + `xrpld-node-state`, `xrpld-overlay`) +- Extend: `docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml` + (add Discord / PagerDuty receivers) +- Extend: `docker/telemetry/grafana/provisioning/alerting/policies.yaml` + (add routes; the root route and the `severity = critical` child already exist) **Exit Criteria**: -- [ ] All 18 rules evaluate without errors in Grafana alerting UI +- [ ] The 14 not-yet-shipped rules evaluate without errors in Grafana alerting UI +- [ ] The 2 rules **fully** covered by Phase 9 (Unhealthy State, Job Queue + Overflow) are not duplicated; the 2 **partially** covered ones (High IO + Latency, Stale Ledger) are either re-baselined on the Phase 9 rule or shipped + as tighter variants — decision recorded either way - [ ] Critical rules fire within expected timeframe when conditions are met - [ ] Notification channel templates are documented (not hard-coded to any service) +- [ ] `node_exporter` decision recorded for the 3 host-level rules (CPU, memory, disk) +- [x] `state_tracking` gauge implemented (Task 7.12) before adding "Not Proposing" + — **prerequisite met upstream**, not Phase 11 work. + `MetricsRegistry::registerStateTrackingGauge()` + (`src/xrpld/telemetry/MetricsRegistry.cpp:1461-1510`) creates + `CreateDoubleObservableGauge("state_tracking", "Node state and mode tracking")` + at `:1466` and observes `state_value` (`:1497`) and + `time_in_current_state_seconds` (`:1502`). Already queried by + `validator-health.json:765,971` and `ledger-data-sync.json:869`, and + documented in + [09-data-collection-reference.md](./09-data-collection-reference.md) + § State Tracking. "Not Proposing" can be written now. --- @@ -533,12 +651,14 @@ This phase addresses the cross-cutting gap identified during research: **xrpld h - [ ] Custom OTel Collector receiver builds and starts without errors - [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus - [ ] Metrics update at configured poll interval (default 30s) -- [ ] 4 new Grafana dashboards operational with data +- [ ] 4 new Grafana dashboards operational with data, none reusing a Phase 9 uid + (`validator-health`, `peer-quality`, `fee-market`, `job-queue`, `node-health`) - [ ] Prometheus alerting rules fire correctly for simulated failure conditions - [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria) - [ ] Phase 10 validation suite passes with receiver metrics included - [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries) - [ ] Documentation complete: receiver README, metric reference, alerting playbook - [ ] Go receiver has unit tests with >80% coverage -- [ ] 18 Grafana alert rules for Phase 7+ parity metrics evaluate correctly (Task 11.12) +- [ ] The 14 not-yet-shipped Grafana alert rules for Phase 7+ parity metrics + evaluate correctly (Task 11.12); the other 4 of the 18 already ship in Phase 9 - [ ] Dual-datasource architecture documented with trade-offs (Task 11.13) diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md index 74d13d54c3..dbfa4d647f 100644 --- a/OpenTelemetryPlan/Phase2_taskList.md +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -8,11 +8,11 @@ ### Related Plan Documents -| Document | Relevance | -| ------------------------------------------------------------ | ------------------------------------------------------------- | -| [04-code-samples.md](./04-code-samples.md) | TraceContextPropagator (§4.4.2), RPC instrumentation (§4.5.3) | -| [02-design-decisions.md](./02-design-decisions.md) | W3C Trace Context (§2.5), span attributes (§2.4.2) | -| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 2 tasks (§6.3), definition of done (§6.11.2) | +| Document | Relevance | +| ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| [03-implementation-strategy.md](./03-implementation-strategy.md) | Code structure and instrumentation patterns (replaces the deleted `04-code-samples.md` §4.4.2 / §4.5.3, removed by `d6450631bf`) | +| [02-design-decisions.md](./02-design-decisions.md) | W3C Trace Context (§2.5), span attributes (§2.4.2) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 2 tasks (§6.3), definition of done (§6.11.2) | --- diff --git a/OpenTelemetryPlan/Phase3_taskList.md b/OpenTelemetryPlan/Phase3_taskList.md index c5d3c95251..55e0a9ed64 100644 --- a/OpenTelemetryPlan/Phase3_taskList.md +++ b/OpenTelemetryPlan/Phase3_taskList.md @@ -8,12 +8,12 @@ ### Related Plan Documents -| Document | Relevance | -| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | -| [04-code-samples.md](./04-code-samples.md) | TraceContext protobuf (§4.4.1), PeerImp instrumentation (§4.5.1), context serialization (§4.4.2) | -| [01-architecture-analysis.md](./01-architecture-analysis.md) | Transaction flow (§1.3), key trace points (§1.6) | -| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 3 tasks (§6.4), definition of done (§6.11.3) | -| [02-design-decisions.md](./02-design-decisions.md) | Context propagation design (§2.5), attribute schema (§2.4.3) | +| Document | Relevance | +| ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) | Authoritative protocol span-flow reference — replaces the deleted `04-code-samples.md` (TraceContext protobuf §4.4.1, PeerImp instrumentation §4.5.1, context serialization §4.4.2), removed by `d6450631bf` | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Transaction flow (§1.3), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 3 tasks (§6.4), definition of done (§6.11.3) | +| [02-design-decisions.md](./02-design-decisions.md) | Context propagation design (§2.5), attribute schema (§2.4.3) | --- @@ -47,7 +47,9 @@ **Reference**: -- [04-code-samples.md §4.4.1](./04-code-samples.md) — TraceContext message definition +- `04-code-samples.md` §4.4.1 (TraceContext message definition) was deleted by + `d6450631bf`; the live definition is `include/xrpl/proto/xrpl.proto:101` + (`message TraceContext`), attached as field `1001` on the relevant messages - [02-design-decisions.md §2.5.2](./02-design-decisions.md) — Protocol buffer context propagation design --- @@ -75,7 +77,13 @@ **Reference**: -- [04-code-samples.md §4.4.2](./04-code-samples.md) — Full extract/inject implementation +- `04-code-samples.md` §4.4.2 (full extract/inject implementation) was deleted by + `d6450631bf`. As shipped there is **no** + `src/libxrpl/telemetry/TraceContextPropagator.cpp`; extract/inject live on + `SpanGuard` (`include/xrpl/telemetry/SpanGuard.h:467` extract, `:480-491` + `injectCurrentContextToProtobuf`, implemented in + `src/libxrpl/telemetry/SpanGuard.cpp`) with the protocol-layer wrappers in + `src/xrpld/telemetry/PropagationHelpers.h:52` (`injectSpanContext`) --- @@ -110,7 +118,9 @@ **Reference**: -- [04-code-samples.md §4.5.1](./04-code-samples.md) — Full PeerImp instrumentation example +- [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) + — the authoritative `tx.receive` / relay span-flow reference; replaces + `04-code-samples.md` §4.5.1, deleted by `d6450631bf` - [01-architecture-analysis.md §1.3](./01-architecture-analysis.md) — Transaction flow diagram - [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.receive trace point @@ -231,7 +241,10 @@ design. **Reference**: - [02-design-decisions.md §2.5](./02-design-decisions.md) — Context propagation design -- [04-code-samples.md §4.5.1](./04-code-samples.md) — Relay context injection pattern +- Relay context injection pattern: `04-code-samples.md` §4.5.1 was deleted by + `d6450631bf`; the live pattern is `injectSpanContext()` in + `src/xrpld/telemetry/PropagationHelpers.h:52`, with the flow documented in + [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) --- diff --git a/OpenTelemetryPlan/Phase4_taskList.md b/OpenTelemetryPlan/Phase4_taskList.md index e83a16262e..e5e380a9d6 100644 --- a/OpenTelemetryPlan/Phase4_taskList.md +++ b/OpenTelemetryPlan/Phase4_taskList.md @@ -6,21 +6,53 @@ > > **Branch**: `pratik/otel-phase4-consensus-tracing` (from `pratik/otel-phase3-tx-tracing`) -> **Note on attribute names**: the `xrpl..` keys shown below are -> written in the older dotted form for readability — it mirrors how the fully -> qualified attribute reads in a Tempo trace view. The implemented keys follow -> the convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming) -> (underscore form, e.g. `consensus_round`, `consensus_mode`); the +> **Note on attribute names**: the `xrpl..` keys that earlier +> revisions of this task list used were **never emitted**. `9e27120a15` removed +> the dotted `xrpl.*` namespace from **span** attributes repo-wide. Falsifiable +> check: `grep -rn 'seg::xrpl' src/ include/` → exactly **2** hits, both +> `include/xrpl/telemetry/SpanNames.h:117-118` (`attr::networkId`, +> `attr::networkType`), and both are **resource** attributes +> (`xrpl.network.id` / `xrpl.network.type`) set on the OTel resource at startup — +> the one place the dotted form is still reserved. No span attribute uses it. +> (Do **not** cite `grep 'makeStr("xrpl\.' src/ include/` → 0 hits as evidence: +> these keys were always composed with `join(seg::…, …)`, never that literal, so +> the grep has returned 0 for the entire history of the file and cannot fail.) +> Those spellings have been corrected in place, so every attribute key below is +> the live one. The mapping that was applied: +> `xrpl.ledger.seq` → `ledger_seq`, `xrpl.consensus.mode` → `consensus_mode`, +> `xrpl.consensus.round` → `consensus_round`, +> `xrpl.consensus.round_id` → `consensus_round_id`, +> `xrpl.consensus.ledger_id` → `consensus_ledger_id`, +> `xrpl.tx.id` → `tx_id`, +> `xrpl.validation.ledger_hash` / `xrpl.peer.validation.ledger_hash` → `ledger_hash`, +> `xrpl.validation.full` / `xrpl.peer.validation.full` → `full_validation`, +> `xrpl.peer.version` → `peer_version`. +> Separately, `19a6c2a306` split the single `trusted` key into +> `proposal_trusted` (on `consensus.proposal.receive` and `peer.proposal.receive`) +> and `validation_trusted` (on `consensus.validation.receive` and +> `peer.validation.receive`). Naming follows +> [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming); the > `*SpanNames.h` constants are the single source of truth. +> +> **Three names in this document are not span attributes at all**: +> +> - `amendment_blocked` — a **metric label value** only: +> `validator_health{metric="amendment_blocked"}` (`MetricsRegistry.cpp:1216`). +> No span carries it. +> - `server_state` — a **metric label value** only: +> `server_info{metric="server_state"}` (`MetricsRegistry.cpp:1014`). It is also +> an RPC method name. No span carries it. +> - `proposers_validated` — **never implemented** on any span. `proposersValidated` +> exists only as a C++ function/parameter name (`RCLConsensus.cpp:310`); +> `consensus.accept` carries `proposers` instead (see Task 4.8). ### Related Plan Documents -| Document | Relevance | -| ------------------------------------------------------------ | ----------------------------------------------------------- | -| [04-code-samples.md](./04-code-samples.md) | Consensus instrumentation (§4.5.2), consensus span patterns | -| [01-architecture-analysis.md](./01-architecture-analysis.md) | Consensus round flow (§1.4), key trace points (§1.6) | -| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 4 tasks (§6.5), definition of done (§6.11.4) | -| [02-design-decisions.md](./02-design-decisions.md) | Consensus attribute schema (§2.4.4) | +| Document | Relevance | +| ------------------------------------------------------------ | -------------------------------------------------------------------- | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Consensus round flow (§1.4), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 4 tasks and exit criteria (§6.5), definition of done (§6.12.4) | +| [02-design-decisions.md](./02-design-decisions.md) | Consensus attribute schema (§2.4.2 → "Consensus Attributes") | --- @@ -34,8 +66,8 @@ - `RCLConsensus::Adaptor::startRoundTracing()` creates `consensus.round` span via `SpanGuard::hashSpan()` (deterministic) or `SpanGuard::span()` (attribute strategy) -- Attributes set: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, - `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` +- Attributes set: `consensus_ledger_id`, `ledger_seq`, + `consensus_mode`, `trace_strategy`, `consensus_round_id` - Round span stored as `roundSpan_` member in `RCLConsensus::Adaptor` - `roundSpanContext_` snapshot captured for cross-thread span linking @@ -46,7 +78,9 @@ **Reference**: -- [04-code-samples.md §4.5.2](./04-code-samples.md) — startRound instrumentation example +- `RCLConsensus::Adaptor::startRoundTracing()` — the live startRound + instrumentation (the former `04-code-samples.md` §4.5.2 was deleted by + `d6450631bf`; the code is the reference now) - [01-architecture-analysis.md §1.4](./01-architecture-analysis.md) — Consensus round flow --- @@ -75,7 +109,8 @@ **Reference**: -- [04-code-samples.md §4.5.2](./04-code-samples.md) — phaseTransition instrumentation +- `Consensus.h` — the live phase-transition instrumentation (`04-code-samples.md` + was deleted by `d6450631bf`) --- @@ -89,11 +124,13 @@ - In `Adaptor::propose()`: - Creates `consensus.proposal.send` span via `SpanGuard::span()` - - Sets `xrpl.consensus.round` attribute + - Sets `consensus_round` attribute - In `PeerImp::onMessage(TMProposeSet)`: - Creates `consensus.proposal.receive` span - - Sets `trusted` attribute (bool) + - Sets `proposal_trusted` attribute (bool) — `PeerSpanNames.h:41`, + `ConsensusSpanNames.h:244`; renamed from the original `trusted` by + `19a6c2a306`, and the dotted `xrpl.peer.*` form was dropped by `9e27120a15` **Done here** (cross-node propagation, send + receive): @@ -112,8 +149,11 @@ **Reference**: -- [04-code-samples.md §4.5.2](./04-code-samples.md) — peerProposal instrumentation -- [02-design-decisions.md §2.4.4](./02-design-decisions.md) — Consensus attribute schema +- `PeerImp::onMessage(TMProposeSet)` — the live peerProposal instrumentation + (`04-code-samples.md` was deleted by `d6450631bf`) +- [02-design-decisions.md §2.4.2](./02-design-decisions.md) — Consensus attribute + schema (the "Consensus Attributes" table under "Span Attributes by Category"; + §2.4.4 is the Privacy & Sensitive Data Policy, not the schema) --- @@ -130,12 +170,14 @@ - Uses `SpanGuard::linkedSpan()` to create a follows-from link to the round span - Thread-safe: uses `roundSpanContext_` snapshot (captured on consensus thread, read on jtACCEPT thread) - - Sets `xrpl.ledger.seq` and `proposing` attributes + - Sets `ledger_seq` and `proposing` attributes - In `PeerImp::onMessage(TMValidation)`: - Creates `consensus.validation.receive` span - - Sets `trusted` attribute (bool) - - Sets `xrpl.ledger.seq` attribute + - Sets `validation_trusted` attribute (bool) — `PeerSpanNames.h:42`, + `ConsensusSpanNames.h:245`; renamed from the original `trusted` by + `19a6c2a306`, and the dotted `xrpl.peer.*` form was dropped by `9e27120a15` + - Sets `ledger_seq` attribute **Not implemented** (deferred to Phase 4b — cross-node propagation): @@ -155,9 +197,9 @@ **Implemented attributes** (across various spans): -- `xrpl.ledger.seq` — on `consensus.round`, `consensus.accept.apply` -- `xrpl.consensus.round` — on `consensus.proposal.send` -- `xrpl.consensus.mode` — on `consensus.round`, `consensus.ledger_close` +- `ledger_seq` — on `consensus.round`, `consensus.accept.apply` +- `consensus_round` — on `consensus.proposal.send` +- `consensus_mode` — on `consensus.round`, `consensus.ledger_close` - `proposers` — on `consensus.accept`, `consensus.establish`, `consensus.update_positions` - `converge_percent` — on `consensus.establish`, `consensus.update_positions`, `consensus.check` - `tx_count` — on `consensus.accept.apply` span (in `doAccept()`) @@ -185,7 +227,7 @@ - In `doAccept()` (RCLConsensus.cpp): - Records `tx.included` events on the `consensus.accept.apply` span for each transaction in the accepted set - - Each event includes `xrpl.tx.id` attribute with the transaction hash + - Each event includes `tx_id` attribute with the transaction hash - This links consensus traces to individual transactions **Key modified files**: @@ -225,54 +267,73 @@ **Objective**: Add ledger hash, validation type, and quorum data to consensus validation spans on both send and receive paths. This enables trace-level validation agreement analysis — filter by ledger hash to see which validators agreed for a given ledger. -**Status**: Not implemented. None of the enrichment attributes are set. The `consensus.validation.send` span only has `ledger.seq` and `proposing`. The `consensus.accept` span has `quorum` set to `result.proposers` (not the actual validator quorum from `app_.validators().quorum()`). No `PeerImp.cpp` changes were made. +**Status**: Implemented, except `proposers_validated`. + +- `consensus.validation.send` sets `ledger_seq`, `ledger_hash`, `proposing` and + `full_validation` (`RCLConsensus.cpp:975-981`). +- `peer.validation.receive` sets `ledger_hash` and `full_validation` + (`PeerImp.cpp:2573-2574`). +- `consensus.accept` sets `quorum` from `app_.getValidators().quorum()` + (`RCLConsensus.cpp:516`) — the earlier defect where `quorum` carried + `result.proposers` instead of the real validator quorum is **fixed**. +- Still open: `proposers_validated` on `consensus.accept` — never implemented. + `consensus.accept` already carries `proposers` (`RCLConsensus.cpp:513`), so a + second key for the same value was not added. + +All attribute keys are bare/underscore; the dotted `xrpl.*` forms in the spec +below were never emitted as **span** attributes. Check: +`grep -rn 'seg::xrpl' src/ include/` → 2 hits, both `SpanNames.h:117-118` +resource attributes (`xrpl.network.{id,type}`). See the note at the top of this +document for why the old `makeStr("xrpl\.` grep proved nothing. **What to do**: - Edit `src/xrpld/app/consensus/RCLConsensus.cpp`: - On the `consensus.validation.send` span (in `validate()` / `doAccept()`): - - Add `xrpl.validation.ledger_hash` (string) — the ledger hash being validated - - Add `xrpl.validation.full` (bool) — whether this is a full validation (not partial) + - Add `ledger_hash` (string) — the ledger hash being validated + - Add `full_validation` (bool) — whether this is a full validation (not partial) - On the `consensus.accept` span (in `onAccept()`): - - Add `validation_quorum` (int64) — from `app_.validators().quorum()` - - Add `proposers_validated` (int64) — from `result.proposers` + - Add `quorum` (int64) — from `app_.getValidators().quorum()` ✅ shipped + - Add `proposers_validated` (int64) — from `result.proposers` ❌ never + implemented; `proposers` already carries this value - Edit `src/xrpld/overlay/detail/PeerImp.cpp`: - On the `peer.validation.receive` span: - - Add `xrpl.peer.validation.ledger_hash` (string) — from deserialized `STValidation` object - - Add `xrpl.peer.validation.full` (bool) — from `STValidation` flags + - Add `ledger_hash` (string) — from deserialized `STValidation` object + - Add `full_validation` (bool) — from `STValidation` flags **New span attributes**: -| Span | Attribute | Type | Source | -| --------------------------- | ---------------------------------- | ------ | --------------------------------- | -| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Ledger hash from validate() args | -| `consensus.validation.send` | `xrpl.validation.full` | bool | Full vs partial validation | -| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | From STValidation deserialization | -| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | From STValidation flags | -| `consensus.accept` | `validation_quorum` | int64 | `app_.validators().quorum()` | -| `consensus.accept` | `proposers_validated` | int64 | `result.proposers` | +| Span | Attribute (live name) | Type | Source | Status | +| --------------------------- | --------------------- | ------ | --------------------------------- | ------------------------- | +| `consensus.validation.send` | `ledger_hash` | string | Ledger hash from validate() args | ✅ `RCLConsensus.cpp:977` | +| `consensus.validation.send` | `full_validation` | bool | Full vs partial validation | ✅ `RCLConsensus.cpp:981` | +| `peer.validation.receive` | `ledger_hash` | string | From STValidation deserialization | ✅ `PeerImp.cpp:2573` | +| `peer.validation.receive` | `full_validation` | bool | From STValidation flags | ✅ `PeerImp.cpp:2574` | +| `consensus.accept` | `quorum` | int64 | `app_.getValidators().quorum()` | ✅ `RCLConsensus.cpp:516` | +| `consensus.accept` | `proposers_validated` | int64 | `result.proposers` | ❌ never implemented | **Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Example Tempo query: ``` -{name="consensus.validation.send"} | xrpl.validation.ledger_hash = "A1B2C3..." +{name="consensus.validation.send" && span.ledger_hash = "A1B2C3..."} ``` Phase 7's `ValidationTracker` builds metric-level aggregation (1h/24h agreement %) on top of this data. -**Key modified files (not yet modified)**: +**Key modified files**: -- `src/xrpld/app/consensus/RCLConsensus.cpp` -- `src/xrpld/overlay/detail/PeerImp.cpp` +- `src/xrpld/app/consensus/RCLConsensus.cpp` (`:516`, `:975-981`) +- `src/xrpld/overlay/detail/PeerImp.cpp` (`:2573-2574`) **Exit Criteria**: - [x] `consensus.validation.send` spans carry `ledger_hash` and `full_validation` -- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` -- [ ] `consensus.accept` spans carry `validation_quorum` and `proposers_validated` +- [x] `peer.validation.receive` spans carry `ledger_hash` and `full_validation` — `PeerImp.cpp:2573-2574` +- [x] `consensus.accept` spans carry `quorum` — `RCLConsensus.cpp:516` +- [ ] `consensus.accept` spans carry `proposers_validated` — **open**, never implemented - [x] Ledger hash attributes match between send and receive for the same ledger -- [ ] No impact on consensus performance +- [ ] No impact on consensus performance — not measured --- @@ -318,13 +379,13 @@ Phase 7's `ValidationTracker` builds metric-level aggregation (1h/24h agreement ### Implemented Spans -| Span Name | Method | Key Attributes | -| --------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `consensus.proposal.send` | `Adaptor::propose` | `xrpl.consensus.round`, `is_bow_out` | -| `consensus.ledger_close` | `Adaptor::onClose` | `xrpl.ledger.seq`, `xrpl.consensus.mode` | -| `consensus.accept` | `Adaptor::onAccept` | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | -| `consensus.accept.apply` | `Adaptor::doAccept` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `xrpl.ledger.seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | -| `consensus.validation.send` | `Adaptor::onAccept` (via validate) | `proposing`, `ledger_hash`, `ledger_seq`, `full_validation`, `validation_sign_time` | +| Span Name | Method | Key Attributes | +| --------------------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.proposal.send` | `Adaptor::propose` | `consensus_round`, `is_bow_out` | +| `consensus.ledger_close` | `Adaptor::onClose` | `ledger_seq`, `consensus_mode` | +| `consensus.accept` | `Adaptor::onAccept` | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | +| `consensus.accept.apply` | `Adaptor::doAccept` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `ledger_seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | +| `consensus.validation.send` | `Adaptor::onAccept` (via validate) | `proposing`, `ledger_hash`, `ledger_seq`, `full_validation`, `validation_sign_time` | #### Close Time Attributes (consensus.accept.apply) @@ -342,13 +403,15 @@ driven by `avCT_CONSENSUS_PCT` (75% validator agreement threshold): - **`close_time_vote_bins`** — Number of distinct close-time vote bins from peer proposals. Higher values indicate less agreement among validators. - **`resolution_direction`** — Whether close-time resolution `"increased"` (coarser), `"decreased"` (finer), or stayed `"unchanged"` relative to the previous ledger. -**Exit Criteria** (from [06-implementation-phases.md §6.11.4](./06-implementation-phases.md)): +**Exit Criteria** (from [06-implementation-phases.md §6.5](./06-implementation-phases.md) +— §6.11.4 is the WALK phase, i.e. transaction tracing, and does not carry these +criteria; the Phase 4 definition of done is §6.12.4): - [x] Complete consensus round traces - [x] Phase transitions visible (open, establish, close, accept) - [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b - [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) -- [x] No impact on consensus timing +- [ ] No impact on consensus timing — **not measured** - [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept - [ ] Validation span enrichment (Task 4.8) — not implemented @@ -386,7 +449,7 @@ consensus round share the same trace_id without P2P context propagation. ### Strategy B — Attribute-Based Correlation -Use normal random trace_id but attach `xrpl.consensus.ledger_id` as an attribute +Use normal random trace_id but attach `consensus_ledger_id` as an attribute on every consensus span. Correlation happens at query time via Tempo/Grafana `by attribute` queries. @@ -423,10 +486,10 @@ In `RCLConsensus::Adaptor::startRound()`: 5. Call `startSpan("consensus.round", parentContext)` so the new span inherits the deterministic trace_id. - If `attribute`: start a normal `consensus.round` span, set - `xrpl.consensus.ledger_id = previousLedger.id()` as attribute. + `consensus_ledger_id = previousLedger.id()` as attribute. -Both strategies always set `xrpl.consensus.round_id` (round number) and -`xrpl.consensus.ledger_id` (previous ledger hash) as attributes. +Both strategies always set `consensus_round_id` (round number) and +`consensus_ledger_id` (previous ledger hash) as attributes. --- @@ -542,7 +605,7 @@ spans in `Consensus.h`. - Reads `consensus_trace_strategy` via `app_.getTelemetry().getConsensusTraceStrategy()` - **Deterministic**: uses `SpanGuard::hashSpan()` with `prevLgr.id()` data - **Attribute**: uses `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "round")` - - Sets attributes: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` + - Sets attributes: `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id` - Captures `roundSpanContext_` snapshot for cross-thread span linking - Saves `prevRoundContext_` from previous round for follows-from links @@ -811,7 +874,7 @@ and OFF, and don't affect consensus timing. | Span Name | Location | Key Attributes (actually set) | | ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | -| `consensus.round` | `RCLConsensus.cpp` | `xrpl.consensus.round_id`, `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy` | +| `consensus.round` | `RCLConsensus.cpp` | `consensus_round_id`, `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy` | | `consensus.establish` | `Consensus.h` | `converge_percent`, `establish_count`, `proposers` | | `consensus.update_positions` | `Consensus.h` | `converge_percent`, `proposers`, `have_close_time_consensus`, `close_time_threshold`, `disputes_count`, `avalanche_threshold` | | `consensus.check` | `Consensus.h` | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | @@ -819,17 +882,17 @@ and OFF, and don't affect consensus timing. ### New Events (Phase 4a) -| Event Name | Parent Span | Attributes (actually set) | -| ----------------- | ---------------------------- | ---------------------------------------------------------------- | -| `dispute.resolve` | `consensus.update_positions` | `xrpl.tx.id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | -| `tx.included` | `consensus.accept.apply` | `xrpl.tx.id` | +| Event Name | Parent Span | Attributes (actually set) | +| ----------------- | ---------------------------- | ----------------------------------------------------------- | +| `dispute.resolve` | `consensus.update_positions` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | +| `tx.included` | `consensus.accept.apply` | `tx_id` | ### New Attributes (Phase 4a) ```cpp // Round-level (on consensus.round) — ALL IMPLEMENTED -"xrpl.consensus.round_id" = int64 // Consensus round number -"xrpl.consensus.ledger_id" = string // previousLedger.id() hash +"consensus_round_id" = int64 // Consensus round number +"consensus_ledger_id" = string // previousLedger.id() hash "trace_strategy" = string // "deterministic" or "attribute" // Establish-level — IMPLEMENTED @@ -877,9 +940,12 @@ and OFF, and don't affect consensus timing. - **No `getTelemetry()` adaptor method**: `SpanGuard::span()` is a static factory that internally checks telemetry state, so `Consensus.h` doesn't need adaptor access for span creation. Only `RCLConsensus::Adaptor` accesses `app_.getTelemetry()` directly. -- **Config validation**: `consensus_trace_strategy` is validated to be either - `"deterministic"` or `"attribute"`, falling back to `"deterministic"` for - unrecognised values. +- **No config validation**: `consensus_trace_strategy` is **not** validated. + `TelemetryConfig.cpp:155-156` copies the raw string through, and the only + comparison in the code is `strategy == "attribute"` (`RCLConsensus.cpp:1296`). + Any unrecognised value — including a typo — silently takes the deterministic + branch, with no log warning. The effective fallback is correct; the absence of + a diagnostic is a known gap. - **Plan deviation**: `roundSpan_` is stored in `RCLConsensus::Adaptor` (not `Consensus.h`) because the adaptor has access to telemetry config and can implement the deterministic trace ID strategy. `establishSpan_` is correctly diff --git a/OpenTelemetryPlan/Phase7_taskList.md b/OpenTelemetryPlan/Phase7_taskList.md index e22c4b37c9..be90294ff8 100644 --- a/OpenTelemetryPlan/Phase7_taskList.md +++ b/OpenTelemetryPlan/Phase7_taskList.md @@ -75,7 +75,8 @@ - Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp` - Use RAII for MeterProvider lifecycle (shutdown on destructor) -**Reference**: [04-code-samples.md](./04-code-samples.md) — code style and patterns +**Reference**: [03-implementation-strategy.md](./03-implementation-strategy.md) — +code style and patterns (`04-code-samples.md` was deleted by `d6450631bf`) --- diff --git a/OpenTelemetryPlan/Phase9_taskList.md b/OpenTelemetryPlan/Phase9_taskList.md index 5c67ec1097..771491ea38 100644 --- a/OpenTelemetryPlan/Phase9_taskList.md +++ b/OpenTelemetryPlan/Phase9_taskList.md @@ -1,6 +1,11 @@ + + + # Phase 9: Internal Metric Instrumentation Gap Fill — Task List -> **Status**: Future Enhancement +> **Status**: Complete for Tasks 9.1-9.13. Tasks 9.14-9.17 remain open by design +> (see each task for the blocker). > > **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines. > @@ -10,6 +15,36 @@ > > **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation) +> **Note on metric names**: there is **no `xrpld_` prefix** on any emitted +> metric. `77f35c03db` removed it and lowercased names, and +> `OTelCollectorImp::formatName()` +> (`src/libxrpl/beast/insight/OTelCollector.cpp:855-874`) adds no prefix at all — +> it only lowercases the raw name and turns `.` and spaces into `_`. Earlier +> revisions of this task list spelled every metric `xrpld_`; those spellings +> have been corrected in place to the emitted names, so the names below can be +> pasted into Prometheus as written. Instruments created in +> `src/xrpld/telemetry/MetricsRegistry.cpp` (35 of them) are the single source of +> truth. `MetricsRegistry.h`'s Doxygen used to disagree on three histogram names; +> those header comments were repaired in this change set (see Tasks 9.4 and 9.5), +> so header and `.cpp` now agree. +> +> **Two shapes do not simply lose the prefix**, so `xrpld_` → `` is +> not a blanket rule: +> +> - **Multiplexed observable gauges.** Most of the value names in these task +> descriptions are a **`metric` label value** on a shared instrument, not a +> standalone metric name — queue depth is `txq_metrics{metric="txq_count"}`, not +> `txq_count`. The same applies to `nodestore_state`, `cache_metrics`, +> `load_factor_metrics`, `server_info`, `db_metrics`, `validator_health`, +> `peer_quality`, `state_tracking` and `ledger_economy`. Each task below names +> its owning instrument. +> - **Unit-suffixed histograms** coming through `beast::insight`. +> `OTelCollectorImp` appends the unit to the name, so the `ios_latency` +> histogram is `ios_latency_milliseconds_bucket` in Prometheus — not +> `ios_latency_bucket`. Instruments created directly on `MetricsRegistry` keep +> their literal name (`job_queued_us_bucket`, `rpc_method_us_bucket`) because +> the unit is already in the instrument name. + ### Related Plan Documents | Document | Relevance | @@ -40,7 +75,16 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add: +> **As shipped, this did _not_ go through `beast::insight`.** `Database.cpp` has +> no insight members. The metrics are a single `nodestore_state` +> `Int64ObservableGauge` on `MetricsRegistry` +> (`src/xrpld/telemetry/MetricsRegistry.cpp:957-965`) whose callback reads +> `Database`'s public accessors (`getFetchTotalCount()`, `getFetchHitCount()`, +> `getStoreCount()`, `getFetchDurationUs()`, `getStoreDurationUs()`, …) and +> multiplexes every value onto the `metric` label. Write-queue depth comes from +> the new `include/xrpl/nodestore/WriteStats.h`. + +- Export the following as `nodestore_state{metric="…"}` label values: - Gauge: `node_reads_total` (cumulative read operations) - Gauge: `node_reads_hit` (fetches that found an object — not a cache hit; `fetchHitCount_` increments whatever served the fetch) - Gauge: `node_writes` (cumulative write operations) @@ -50,14 +94,18 @@ These metrics serve multiple external consumer categories identified during rese - Gauge: `write_load` (current write load score) - Gauge: `read_queue` (items in read queue) -- These values are already computed in `Database::getCountsJson()` (line ~236). Wire the same counters to `beast::insight` hooks. +- These values are already computed in `Database::getCountsJson()`. The gauge + callback reads the same counters through `Database`'s public accessors. **Key modified files**: -- `src/libxrpl/nodestore/Database.cpp` -- `src/libxrpl/nodestore/Database.h` (add insight members) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (the `nodestore_state` gauge) +- `include/xrpl/nodestore/Database.h` (accessors; **not** `src/libxrpl/nodestore/Database.h`, which does not exist) +- `include/xrpl/nodestore/WriteStats.h` (new — write-queue depth snapshot) -**Derived Prometheus metrics**: `xrpld_nodestore_reads_total`, `xrpld_nodestore_reads_hit`, `xrpld_nodestore_write_load`, etc. +**Derived Prometheus metrics**: `nodestore_state{metric="node_reads_total"}`, +`nodestore_state{metric="node_reads_hit"}`, `nodestore_state{metric="write_load"}`, +etc. There is **no** `xrpld_` prefix — `OTelCollectorImp::formatName()` adds none. **Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard. @@ -77,17 +125,22 @@ These metrics serve multiple external consumer categories identified during rese - `treenode_track_size` — Tracked tree nodes - `fullbelow_size` — FullBelow cache size -- The callback should read from the same sources as `GetCounts.cpp` handler (line ~43). +- The callback reads from the same sources as the `GetCounts` handler + (`src/xrpld/rpc/handlers/admin/status/GetCounts.cpp` — **not** + `src/xrpld/rpc/handlers/GetCounts.cpp`). - Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`. **Key modified files**: - New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp` -- `src/xrpld/rpc/handlers/GetCounts.cpp` (extract shared access methods) +- New: `src/xrpld/telemetry/MetricMacros.h` (the `XRPL_METRIC_*` call-site macros) +- `src/xrpld/rpc/handlers/admin/status/GetCounts.cpp` (extract shared access methods) - `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup) -**Derived Prometheus metrics**: `xrpld_cache_SLE_hit_rate`, `xrpld_cache_ledger_hit_rate`, `xrpld_cache_treenode_size`, etc. +**Derived Prometheus metrics**: `cache_metrics{metric="SLE_hit_rate"}`, +`cache_metrics{metric="ledger_hit_rate"}`, `cache_metrics{metric="treenode_cache_size"}`, +etc. Label values are **case-sensitive** (`SLE_hit_rate`, `AL_size`, `AL_hit_rate`). --- @@ -97,7 +150,8 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- Register OTel `ObservableGauge` callbacks for TxQ state (from `TxQ.h` line ~143): +- Register OTel `ObservableGauge` callbacks for TxQ state (from + `src/xrpld/app/misc/TxQ.h` — **not** `src/xrpld/app/tx/detail/TxQ.h`): - `txq_count` — Current transactions in queue - `txq_max_size` — Maximum queue capacity - `txq_in_ledger` — Transactions in current open ledger @@ -112,9 +166,12 @@ These metrics serve multiple external consumer categories identified during rese **Key modified files**: - `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks) -- `src/xrpld/app/tx/detail/TxQ.h` (expose metrics accessor if needed) +- `src/xrpld/app/misc/TxQ.h` (expose metrics accessor if needed) -**Derived Prometheus metrics**: `xrpld_txq_count`, `xrpld_txq_max_size`, `xrpld_txq_open_ledger_fee_level`, etc. +**Derived Prometheus metrics**: `txq_metrics{metric="txq_count"}`, +`txq_metrics{metric="txq_max_size"}`, `txq_metrics{metric="txq_open_ledger_fee_level"}`, etc. +There is one instrument, `txq_metrics` (`MetricsRegistry.cpp:705`); each value above +is a `metric` label value, not a metric name of its own. **Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`fee-market`). @@ -126,13 +183,25 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp` line ~63): - - Counter: `xrpld_rpc_method_started_total{method=""}` — calls started - - Counter: `xrpld_rpc_method_finished_total{method=""}` — calls completed - - Counter: `xrpld_rpc_method_errored_total{method=""}` — calls errored - - Histogram: `xrpld_rpc_method_duration_us{method=""}` — execution time distribution +- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp`): + - Counter: `rpc_method_started_total{method=""}` — calls started + - Counter: `rpc_method_finished_total{method=""}` — calls completed + - Counter: `rpc_method_errored_total{method=""}` — calls errored + - Histogram: `rpc_method_us{method=""}` — execution time distribution -- Use OTel `Counter` and `Histogram` instruments with `method` attribute label. +- Use OTel `Counter` and `Histogram` instruments with the + `method` attribute label. The RPC instruments carry **only** `method` + (`MetricsRegistry.cpp:436-475`) — the `handler` label belongs to the job + instruments (Task 9.5), not these. + +> **Naming**: the instrument is `rpc_method_us` — declared as +> `kRpcMethodDurationUs` at `MetricsRegistry.cpp:96` and used both to register the +> explicit-bucket view and to create the instrument. `MetricsRegistry.h`'s Doxygen +> comment used to read `rpc_method_duration_us`; **that was fixed in this change** +> (`MetricsRegistry.h:789`), so header and `.cpp` now agree and there is no +> caveat left. The prefix `xrpld_` in the original spec is not emitted by anything. +> +> Same for the job histograms in Task 9.5: `job_queued_us` / `job_running_us`. - Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points. @@ -141,7 +210,7 @@ These metrics serve multiple external consumer categories identified during rese - `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters) - `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments) -**Derived Prometheus metrics**: `xrpld_rpc_method_started_total{method="server_info"}`, `xrpld_rpc_method_duration_us_bucket{method="ledger"}`, etc. +**Derived Prometheus metrics**: `rpc_method_started_total{method="server_info"}`, `rpc_method_us_bucket{method="ledger"}`, etc. **Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard. @@ -153,12 +222,24 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- Register OTel instruments for PerfLog job counters: - - Counter: `xrpld_job_queued_total{job_type=""}` — jobs queued - - Counter: `xrpld_job_started_total{job_type=""}` — jobs started - - Counter: `xrpld_job_finished_total{job_type=""}` — jobs completed - - Histogram: `xrpld_job_queued_duration_us{job_type=""}` — time spent waiting in queue - - Histogram: `xrpld_job_running_duration_us{job_type=""}` — execution time distribution +- Register OTel instruments for PerfLog job counters. All five carry **two** + labels — `job_type` and `handler` — so producers sharing a job type stay + distinguishable (`MetricsRegistry.h:794-818`, recorded at + `MetricsRegistry.cpp:498,518,527,548,553`). `handler` is the sanitised + `addJob` name; `sanitiseHandler()` folds dynamic names into a bounded domain + of exactly 44 values, so cardinality stays fixed. + - Counter: `job_queued_total{job_type="",handler=""}` — jobs queued + - Counter: `job_started_total{job_type="",handler=""}` — jobs started + - Counter: `job_finished_total{job_type="",handler=""}` — jobs completed + - Histogram: `job_queued_us{job_type="",handler=""}` — time spent waiting in queue + - Histogram: `job_running_us{job_type="",handler=""}` — execution time distribution + +> **Naming**: the instruments are `job_queued_us` / `job_running_us` +> (`kJobQueuedDurationUs` / `kJobRunningDurationUs`, `MetricsRegistry.cpp:94-95`). +> `MetricsRegistry.h`'s Doxygen comments used to read +> `job_queued_duration_us` / `job_running_duration_us`; **both were fixed in this +> change** (`MetricsRegistry.h:810,815`), so there is no header/`.cpp` divergence +> left to work around. - Hook into PerfLog's existing job tracking alongside Task 9.4. @@ -167,7 +248,7 @@ These metrics serve multiple external consumer categories identified during rese - `src/xrpld/perflog/detail/PerfLogImp.cpp` - `src/xrpld/telemetry/MetricsRegistry.cpp` -**Derived Prometheus metrics**: `xrpld_job_queued_total{job_type="ledgerData"}`, `xrpld_job_running_duration_us_bucket{job_type="transaction"}`, etc. +**Derived Prometheus metrics**: `job_queued_total{job_type="ledgerData",handler="ProcessLData"}`, `job_running_us_bucket{job_type="transaction",handler="…"}`, etc. **Grafana dashboard**: New _Job Queue Analysis_ dashboard (`job-queue`). @@ -180,15 +261,16 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: - Register OTel `ObservableGauge` callbacks for `CountedObject` instance counts: - - `xrpld_object_count{type="Transaction"}` — live Transaction objects - - `xrpld_object_count{type="Ledger"}` — live Ledger objects - - `xrpld_object_count{type="NodeObject"}` — live NodeObject instances - - `xrpld_object_count{type="STTx"}` — serialized transaction objects - - `xrpld_object_count{type="STLedgerEntry"}` — serialized ledger entries - - `xrpld_object_count{type="InboundLedger"}` — ledgers being fetched - - `xrpld_object_count{type="Pathfinder"}` — active pathfinding computations - - `xrpld_object_count{type="PathRequest"}` — active path requests - - `xrpld_object_count{type="HashRouterEntry"}` — hash router entries + - `object_count{type="xrpl::Transaction"}` — live Transaction objects + - `object_count{type="xrpl::Ledger"}` — live Ledger objects + - `object_count{type="xrpl::NodeObject"}` — live NodeObject instances + - `object_count{type="xrpl::STTx"}` — serialized transaction objects + - `object_count{type="xrpl::STLedgerEntry"}` — serialized ledger entries + - `object_count{type="xrpl::InboundLedger"}` — ledgers being fetched + - `object_count{type="xrpl::Pathfinder"}` — active pathfinding computations + - `object_count{type="xrpl::PathRequest"}` — active path requests + - `object_count{type="xrpl::HashRouter::Entry"}` — hash router entries (the type is + `HashRouter::Entry`; there is no `HashRouterEntry` type) - The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts. @@ -197,7 +279,9 @@ These metrics serve multiple external consumer categories identified during rese - `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks) - `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration) -**Derived Prometheus metrics**: `xrpld_object_count{type="Transaction"}`, `xrpld_object_count{type="NodeObject"}`, etc. +**Derived Prometheus metrics**: `object_count{type="xrpl::Transaction"}`, `object_count{type="xrpl::NodeObject"}`, etc. +The `type` label value is `beast::typeName()` — the fully-qualified +demangled C++ type name (`CountedObject.h:109`), not a short word. **Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard. @@ -225,7 +309,10 @@ These metrics serve multiple external consumer categories identified during rese - `src/xrpld/telemetry/MetricsRegistry.cpp` - `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed) -**Derived Prometheus metrics**: `xrpld_load_factor`, `xrpld_load_factor_fee_escalation`, etc. +**Derived Prometheus metrics**: `load_factor_metrics{metric="load_factor"}`, +`load_factor_metrics{metric="load_factor_fee_escalation"}`, etc. There is one +instrument, `load_factor_metrics` (`MetricsRegistry.cpp:785`); every value listed +above is a `metric` label value, not a metric name of its own. **Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard. @@ -243,7 +330,7 @@ These metrics serve multiple external consumer categories identified during rese - `read_request_bundle` (native JSON int) - `read_threads_running` (native JSON int) - `read_threads_total` (native JSON int) -- Added new `xrpld_server_info` Int64ObservableGauge with 8 metrics: +- Added new `server_info` Int64ObservableGauge with 8 metrics: - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL) - `uptime` — seconds since server start - `peers` — total peer count @@ -252,9 +339,9 @@ These metrics serve multiple external consumer categories identified during rese - `peer_disconnects_resources` — cumulative resource-related disconnects - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]` - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]` -- Added new `xrpld_build_info` Int64ObservableGauge (info-style, value=1 with `version` label) -- Added new `xrpld_complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs -- Added new `xrpld_db_metrics` Int64ObservableGauge with 4 metrics: +- Added new `build_info` Int64ObservableGauge (info-style, value=1 with `version` label) +- Added new `complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs +- Added new `db_metrics` Int64ObservableGauge with 4 metrics: - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries) - `historical_perminute` (historical ledger fetch rate) @@ -267,7 +354,7 @@ These metrics serve multiple external consumer categories identified during rese - `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`) -**Derived Prometheus metrics**: `xrpld_server_info{metric="server_state"}`, `xrpld_build_info{version="2.4.0"}`, `xrpld_complete_ledgers{bound="start",index="0"}`, `xrpld_db_metrics{metric="db_kb_total"}`, etc. +**Derived Prometheus metrics**: `server_info{metric="server_state"}`, `build_info{version="2.4.0"}`, `complete_ledgers{bound="start",index="0"}`, `db_metrics{metric="db_kb_total"}`, etc. **Grafana dashboard**: New panels added to _Node Health_ dashboard (`node-health.json`). @@ -284,15 +371,19 @@ These metrics serve multiple external consumer categories identified during rese 2. **Job Queue Analysis** (`job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth - Update 2 existing dashboards: - 1. **Node Health** (`xrpld-statsd-node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts + 1. **Node Health** (`node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts 2. **RPC Performance** (`rpc-performance`) — Add per-method RPC breakdown panels -**Key modified files**: +> Tasks 9.11-9.13 add two more new dashboards (`validator-health`, +> `peer-quality`), so Phase 9's total is **4 new + 2 updated**. -- New: `docker/telemetry/grafana/dashboards/rippled-fee-market.json` -- New: `docker/telemetry/grafana/dashboards/rippled-job-queue.json` -- `docker/telemetry/grafana/dashboards/rippled-statsd-node-health.json` -- `docker/telemetry/grafana/dashboards/rippled-rpc-perf.json` +**Key modified files** (filenames and uids after the `rippled-*` → bare rename +in `145b1469d6` and `25868f2740` — the `rippled-*.json` paths no longer exist): + +- New: `docker/telemetry/grafana/dashboards/fee-market.json` (uid `fee-market`) +- New: `docker/telemetry/grafana/dashboards/job-queue.json` (uid `job-queue`) +- `docker/telemetry/grafana/dashboards/node-health.json` (uid `node-health`) +- `docker/telemetry/grafana/dashboards/rpc-performance.json` (uid `rpc-performance`) --- @@ -302,18 +393,37 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- Update `OpenTelemetryPlan/09-data-collection-reference.md`: - - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors) - - Update Grafana dashboard reference table (add 2 new dashboards) +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: ✅ done + - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors) — §5b + "Phase 9: OTel SDK-Exported Metrics (MetricsRegistry)" + - Update Grafana dashboard reference table (add 4 new dashboards) — "New Grafana Dashboards (Phase 9)" / "Updated Grafana Dashboards (Phase 9)" - Add Prometheus query examples for new metrics - Update `docs/telemetry-runbook.md`: - - Add an Alerting section covering the provisioned rules and how to wire a receiver - - Add troubleshooting entries for new metric categories + - ✅ Alerting section covering the provisioned rules and how to wire a receiver + - ✅ Troubleshooting entries for new metric categories + - ❌ **Still open**: dashboard guides for **six** dashboards — `fee-market`, + `job-queue`, `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality` and + `validator-health`. The runbook's dashboard reference records the gap + verbatim: "Nine dashboards have a reference section below. `fee-market`, + `job-queue`, `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality`, and + `validator-health` are provisioned but not yet documented here — their panel + descriptions carry the same six-heading reference format, so open the panel + info icon in Grafana until a section is written." (15 provisioned − 6 + undocumented = 9 documented.) Also still open: the Validation Agreement + explainer (8s grace / 5m late repair) -- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`): - - 6 rules in 3 groups — consensus/ledger (`LedgerHistoryMismatch`, `LedgerCloseStalled`), validator (`ValidationsMissed`, `ValidationsNotChecked`), job queue (`JobQueueTxOverflow`, `JobQueueLatencyHigh`) - - `xrpld-default` webhook contact point + flat notification policy; auto-loaded via the existing `provisioning/` mount (no docker-compose change) +- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`) — **as shipped**: + - **13 rules in 5 groups**: `xrpld-consensus` (`LedgerHistoryMismatch`, + `LedgerCloseStalled`, `ValidatedLedgerStale`), `xrpld-validator` + (`ValidationsMissed`, `ValidationsNotChecked`), `xrpld-jobqueue` + (`JobQueueTxOverflow`, `JobQueueLatencyHigh`, `NodeStoreIOLatencyHigh`), + `xrpld-node-state` (`NodeStateFlapping`, `NodeNotFull`), `xrpld-overlay` + (`ManifestJobQueueConvoy`, `ManifestFloodInbound`, `PeerResourceDisconnects`) + - **2 contact points** — `xrpld-default` (Slack) and `xrpld-critical` + (Slack + email) — and a **nested** notification policy: root → + `xrpld-default`, child route `severity = critical` → `xrpld-critical`. + Auto-loaded via the existing `provisioning/` mount (no docker-compose change) + - 3 rules are `severity: critical`, 10 are `severity: warning` - Alerting operator docs (per-alert meaning, tuning, receiver wiring) now live in the Alerting section of `docs/telemetry-runbook.md` **Key modified files**: @@ -331,21 +441,35 @@ These metrics serve multiple external consumer categories identified during rese **What to do**: -- Extend the existing telemetry integration test: - - Start xrpld with `[telemetry] enabled=1` and `[insight] server=otel` - - Submit a batch of RPC calls and transactions - - Query Prometheus for each new metric family - - Assert non-zero values for: NodeStore reads, cache hit rates, TxQ count, PerfLog RPC counters, object counts, load factors +- ❌ **Not done on this branch**: extend the telemetry integration test to + start xrpld with `[telemetry] enabled=1` / `[insight] server=otel`, drive RPC + and transaction load, query Prometheus for each new metric family and assert + non-zero values. The end-to-end metric assertions live in the **Phase 10** + harness (`docker/telemetry/workload/expected_metrics.json`), not here. -- Add unit tests for the `MetricsRegistry` class: - - Verify callback registration and deregistration - - Verify metric values match `get_counts` JSON output - - Verify graceful behavior when telemetry is disabled +- ✅ **Done**: unit tests for the `MetricsRegistry` class — + `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` (**18** GTest cases — + `grep -cE '\bTEST(_F|_P)?\s*\(' src/tests/libxrpl/telemetry/MetricsRegistry.cpp` + = 18, and the four bullets below sum to 4 + 3 + 5 + 6 = 18): + - Callback registration / deregistration and shutdown ordering — + `async_gauges_start_after_start_is_safe`, + `async_gauges_before_start_does_not_break_start`, + `async_gauges_respect_the_compile_time_guard`, `destructor_calls_stop` + - Graceful behaviour when telemetry is disabled — `disabled_construction`, + `disabled_start_stop`, `disabled_recording_methods` + - Label sanitisation and mean scaling — `MetricsRegistrySanitiseHandler` (5 + cases, incl. `output_domain_is_exactly_44_values`) and + `MetricsRegistryScaledMean` (6 cases) + - ❌ Not covered: asserting metric values match `get_counts` JSON output — + that needs a live `Application`, so it is left to the Phase 10 harness -**Key modified files**: +**Key files**: -- `src/test/telemetry/MetricsRegistry_test.cpp` (new) -- Existing integration test script (extend assertions) +- `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` (new). The originally + planned `src/test/telemetry/MetricsRegistry_test.cpp` was **never created** — + Phase 9 tests are GTest under `src/tests/libxrpl/`, per project convention. +- `src/tests/libxrpl/telemetry/MetricMacros.cpp`, `GetMeter.cpp` (new — cover + the `XRPL_METRIC_*` macros and meter lookup) --- @@ -360,31 +484,43 @@ These metrics serve multiple external consumer categories identified during rese **Dashboard**: `validator-health.json` -| Panel | Type | PromQL | -| -------------------------- | ---------- | -------------------------------------------------------------- | -| Agreement % (1h) | stat | `xrpld_validation_agreement{metric="agreement_pct_1h"}` | -| Agreement % (24h) | stat | `xrpld_validation_agreement{metric="agreement_pct_24h"}` | -| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | -| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | -| Validation Rate | stat | `rate(xrpld_validations_sent_total[5m]) * 60` | -| Validations Checked Rate | stat | `rate(xrpld_validations_checked_total[5m]) * 60` | -| Amendment Blocked | stat | `xrpld_validator_health{metric="amendment_blocked"}` | -| UNL Expiry (days) | stat | `xrpld_validator_health{metric="unl_expiry_days"}` | -| Validation Quorum | stat | `xrpld_validator_health{metric="validation_quorum"}` | -| State Value Timeline | timeseries | `xrpld_state_tracking{metric="state_value"}` | -| Time in Current State | stat | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` | -| State Changes Rate | stat | `rate(xrpld_state_changes_total[1h])` | -| Ledgers Closed Rate | stat | `rate(xrpld_ledgers_closed_total[5m]) * 60` | +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------- | +| Agreement % (1h) | stat | `validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `state_tracking{metric="state_value"}` | +| Time in Current State | stat | `state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(ledgers_closed_total[5m]) * 60` | **Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. -**Key new files**: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` +**Key new files**: `docker/telemetry/grafana/dashboards/validator-health.json` +(uid `validator-health`). The name reached its current form in **two** renames: +`rippled-validator-health.json` → `xrpld-validator-health.json` (`145b1469d6`, +the `rippled-` → `xrpld-` pass), then `xrpld-validator-health.json` → +`validator-health.json` (`25868f2740`, which dropped the `xrpld-` prefix). **Exit Criteria**: -- [ ] All 13 panels render with non-zero data during normal operation -- [ ] `$node` filter works correctly for multi-node deployments -- [ ] Amendment blocked and UNL expiry panels use color thresholds (red=blocked/expiring) +- [x] Dashboard ships **17** panels (4 more than the 13 planned above) across 3 + rows — Validation Agreement, Validation Rates, Server State & Consensus +- [ ] All panels render with non-zero data during normal operation — needs a live + stack; the Phase 10 harness asserts the dashboard _loads_, not that panels + are non-empty +- [x] `$node` filter works correctly for multi-node deployments — `node` + template variable present (filters on `service_instance_id`), alongside + `service_name`, `deployment_environment`, `xrpl_network_type`, + `xrpl_work_item`, `xrpl_branch`, `xrpl_node_role` +- [x] Amendment blocked and UNL expiry panels use color thresholds + (red=blocked/expiring) — 11 `thresholds` blocks in the dashboard JSON --- @@ -396,22 +532,35 @@ These metrics serve multiple external consumer categories identified during rese **Dashboard**: `peer-quality.json` -| Panel | Type | PromQL | -| ---------------------- | ---------- | -------------------------------------------------------------- | -| P90 Peer Latency | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | -| Insane/Diverged Peers | stat | `xrpld_peer_quality{metric="peers_insane_count"}` | -| Higher Version Peers % | stat | `xrpld_peer_quality{metric="peers_higher_version_pct"}` | -| Upgrade Recommended | stat | `xrpld_peer_quality{metric="upgrade_recommended"}` | -| Resource Disconnects | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges` | -| Inbound vs Outbound | bargauge | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` | +| Panel | Type | PromQL | +| ---------------------- | ---------- | ----------------------------------------------------------------------- | +| P90 Peer Latency | timeseries | `peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `server_info{metric="peer_disconnects_resources"}` | +| Inbound vs Outbound | bargauge | `peer_finder_active_inbound_peers`, `peer_finder_active_outbound_peers` | -**Key new files**: `docker/telemetry/grafana/dashboards/rippled-peer-quality.json` +> `overlay_peer_disconnects_charges` (the name in the original spec) is **not a +> real instrument** — nothing registers it. The shipped panel reads +> `server_info{metric="peer_disconnects_resources"}` instead. Peer-finder gauge +> names are lowercase: `GroupImp::makeName()` + `OTelCollectorImp::formatName()` +> turn the `"Peer_Finder"` group into `peer_finder_` with no prefix. + +**Key new files**: `docker/telemetry/grafana/dashboards/peer-quality.json` +(uid `peer-quality`). Two renames, same as Task 9.11: +`rippled-peer-quality.json` → `xrpld-peer-quality.json` (`145b1469d6`), then +`xrpld-peer-quality.json` → `peer-quality.json` (`25868f2740`). **Exit Criteria**: -- [ ] All 6 panels render correctly -- [ ] P90 latency panel shows trend over time -- [ ] Upgrade recommended panel uses color threshold (red=1, green=0) +- [x] All 6 panels present — P90 Peer Latency, Insane/Diverged Peers, Higher + Version Peers %, Upgrade Recommended, Inbound vs Outbound Peers, Resource + Disconnects — across 3 rows, with the `$node` template variable +- [ ] All 6 panels render with data — needs a live stack +- [x] P90 latency panel is a `timeseries` (shows trend over time) +- [x] Upgrade recommended panel uses color threshold (red=1, green=0) — 5 + `thresholds` blocks in the dashboard JSON --- @@ -421,21 +570,22 @@ These metrics serve multiple external consumer categories identified during rese **Objective**: Add "Ledger Economy" row to the existing `node-health.json` dashboard. -| Panel | Type | PromQL | -| -------------------- | ---------- | --------------------------------------------------- | -| Base Fee (drops) | stat | `xrpld_ledger_economy{metric="base_fee_xrp"}` | -| Reserve Base (drops) | stat | `xrpld_ledger_economy{metric="reserve_base_xrp"}` | -| Reserve Inc (drops) | stat | `xrpld_ledger_economy{metric="reserve_inc_xrp"}` | -| Ledger Age | stat | `xrpld_ledger_economy{metric="ledger_age_seconds"}` | -| Transaction Rate | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}` | +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------- | +| Base Fee (drops) | stat | `ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `ledger_economy{metric="transaction_rate"}` | **Key modified files**: `docker/telemetry/grafana/dashboards/node-health.json` **Exit Criteria**: -- [ ] 5 new panels render correctly in existing dashboard -- [ ] Fee values match `server_info` RPC output -- [ ] Transaction rate shows smooth trend (not spiky) +- [x] 5 new panels present in the existing dashboard — a "Ledger Economy" row + with 5 `ledger_economy` queries is on `node-health.json` +- [ ] Fee values match `server_info` RPC output — needs a live comparison +- [ ] Transaction rate shows smooth trend (not spiky) — needs a live run --- @@ -456,9 +606,22 @@ files, so **no code fix lands on this branch**. **Why deferred**: Defect 3 requires widening the two `OverlayImpl::updateSlotAndSquelch` overloads — a public signature change on -shared overlay code. Defects 1, 2 and 4 sit in `TrafficCount.{h,cpp}`, likewise -not telemetry-owned. Routing them through the telemetry chain would hide overlay -changes from overlay reviewers and couple them to a 12-PR merge timeline. +shared overlay code. Defects 1 and 4 need `TrafficCount.cpp` and `PeerImp.cpp` +edits that are not telemetry-owned. Routing them through the telemetry chain +would hide overlay changes from overlay reviewers and couple them to a 12-PR +merge timeline. + +> **Constraint narrowed.** The blanket "no telemetry change may touch +> `TrafficCount.{h,cpp}`" no longer holds for the header: the telemetry chain +> already edits `TrafficCount.h` — Phase 6's `77f35c03db` fixed the +> `Category::GetFetchPack` label from `"getobject_Fetch Pack_get"` to +> `"getobject_Fetch_Pack_get"` at `TrafficCount.h:285`, the sole difference from +> `develop`. Defect 2 (the stale `Total` header comment, `TrafficCount.h:28-31`) +> is therefore **unblocked** and can land here. Defects **1, 3 and 4** stay +> blocked: defect 1 needs `TrafficCount.cpp`'s `kTypeLookup`, defect 3 needs the +> `OverlayImpl` signature change, and defect 4 needs `PeerImp.cpp:1079` vs `:313` +> to agree on a byte basis (compressed vs uncompressed) — a change to overlay +> accounting semantics, not telemetry. **Key modified files**: `OpenTelemetryPlan/09-data-collection-reference.md` only. @@ -466,16 +629,31 @@ changes from overlay reviewers and couple them to a 12-PR merge timeline. - [x] Each defect documented with file:line evidence in `09` §6 - [x] `overhead_cluster_*` documented as "no data", not "no cluster traffic" -- [ ] Follow-up overlay-owned branch raised for the four code fixes +- [ ] Defect 2 (stale `Total` header comment, `TrafficCount.h:28-31`) fixed on + this branch — it is **unblocked** (the chain already edits + `TrafficCount.h`) but the comment is still uncorrected +- [ ] Follow-up overlay-owned branch raised for the three still-blocked code + fixes (defects 1, 3, 4) - [ ] Re-baseline any threshold keyed on `unknown_bytes_in` when defect 1 lands --- ## Task 9.15: Peer Keepalive and Discovery Instrumentation -> **Status**: NOT IMPLEMENTED — awaiting a decision on whether `XRPL_METRIC_*` -> call sites may be added to `src/xrpld/overlay/detail/PeerImp.cpp` from this -> branch. Reference: [09 §6.3](./09-data-collection-reference.md#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) +> **Status**: NOT IMPLEMENTED. The instruments themselves are still to be +> written; the _permission_ question is settled. Reference: +> [09 §6.3](./09-data-collection-reference.md#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) +> +> **Blocker cleared.** This task used to be held "awaiting a decision on whether +> `XRPL_METRIC_*` call sites may be added to +> `src/xrpld/overlay/detail/PeerImp.cpp` from this branch". That decision is +> de facto **yes** — `PeerImp.cpp` already carries **7** such call sites on this +> branch (`:2723`, `:2741`, `:2925`, `:2928`, `:2931`, `:2947`, `:2954`, of which +> three are `XRPL_METRIC_HISTOGRAM_RECORD` — `:2925`, `:2928`, `:2931` — and four +> are labelled counters — `:2723`, `:2741`, `:2947`, `:2954`). Note that +> `grep -c XRPL_METRIC src/xrpld/overlay/detail/PeerImp.cpp` returns 8: the eighth +> hit is the `cspell:ignore` explanation comment at `PeerImp.cpp:2`, not a call +> site. What remains is the implementation work below, not an approval. **Objective**: Make peer keepalive and peer-discovery health observable. Today `mtPING`, `mtSTATUS_CHANGE` and `mtENDPOINTS` are byte counters only. @@ -494,7 +672,8 @@ changes from overlay reviewers and couple them to a 12-PR merge timeline. - `peer_id` as a label is unbounded cardinality — rejected. A bounded `peer_role`-style label is the alternative if per-peer attribution is needed. - Splitting `mtPING` out of `Category::Base` is a `TrafficCount.cpp` change and - therefore blocked with Task 9.14. + therefore still blocked with Task 9.14 defect 1. (The `.h` half of that + constraint no longer applies — see Task 9.14.) - Per the runbook's "Adding a New Metric" contract, `_total` is reserved for monotonic counters; a histogram takes no suffix. @@ -505,7 +684,8 @@ changes from overlay reviewers and couple them to a 12-PR merge timeline. **Exit Criteria**: -- [ ] Decision recorded on editing `PeerImp.cpp` from the telemetry chain +- [x] Decision recorded on editing `PeerImp.cpp` from the telemetry chain — yes; + 7 `XRPL_METRIC_*` call sites already ship in `PeerImp.cpp` - [ ] Three instruments emitting, with an explicit histogram bucket view - [ ] Rows added to `09` §5b, runbook § Metric Reference, and `expected_metrics.json` - [ ] Peer Quality dashboard panels follow the Task 9.12 conventions (`$node`, Title Case, legend dimensions) @@ -557,10 +737,11 @@ actually emits. `peer.connect`, `peer.disconnect`, `peer.message.send` and protocol message families have no spans. **Scope warning**: This is larger than Tasks 9.14-9.16 combined and changes the -"~37 spans" figure asserted in `09` §1.1 and in -`docker/telemetry/workload/expected_spans.json`. `trace_peer` is also **on by -default** and already flagged as high-volume, so adding per-message spans has a -volume cost that needs measuring before commitment. +span-family inventory asserted in `09` §1.1 (**41** emitted families) and in +`docker/telemetry/workload/expected_spans.json` (**40** catalogued — `rpc.ws_upgrade` +has no entry). `trace_peer` is also **on by default** and already flagged as +high-volume, so adding per-message spans has a volume cost that needs measuring +before commitment. **Exit Criteria**: @@ -572,17 +753,41 @@ volume cost that needs measuring before commitment. ## Exit Criteria -- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline -- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK -- [ ] Async gauge callbacks execute at 10s intervals without performance impact -- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) -- [ ] 2 existing dashboards updated with new panel groups -- [ ] Integration test validates all new metric families are non-zero -- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) -- [ ] Documentation updated with full new metric inventory -- [ ] Validator Health dashboard renders all 13 panels -- [ ] Peer Quality dashboard renders all 6 panels -- [ ] Ledger Economy panels added to node-health dashboard +- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline — every + instrument is registered in `MetricsRegistry.cpp`, but end-to-end + visibility is asserted only by the Phase 10 harness +- [x] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK — + `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` + (`async_gauges_start_after_start_is_safe`, + `async_gauges_before_start_does_not_break_start`, + `async_gauges_respect_the_compile_time_guard`, `destructor_calls_stop`) +- [x] Async gauge callbacks execute at 10s intervals — + `MetricsRegistry.cpp:289`, `readerOpts.export_interval_millis = 10000`. + (The "without performance impact" half is unmeasured — see below.) +- [x] 4 new Grafana dashboards operational (Fee Market, Job Queue, Validator + Health, Peer Quality) — all four JSONs are under + `docker/telemetry/grafana/dashboards/` +- [x] 2 existing dashboards updated with new panel groups — `node-health` + (NodeStore I/O, Caches, Server Info, Complete Ledgers & DB, Ledger + Economy, Job Queue Concurrency Limits rows) and `rpc-performance` + (per-method section) +- [ ] Integration test validates all new metric families are non-zero — not on + this branch; lives in the Phase 10 harness (`expected_metrics.json`) +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) — not + measured; needs the Phase 10 benchmark suite +- [x] Documentation updated with full new metric inventory — + `09-data-collection-reference.md` §5b + "Phase 9: OTel SDK-Exported + Metrics (MetricsRegistry)" + "Phase 7+: External Dashboard Parity Metrics" +- [x] Validator Health dashboard ships (17 panels, 4 more than the 13 planned) +- [x] Peer Quality dashboard ships (6 panels) +- [x] Ledger Economy panels added to node-health dashboard (5 panels in a + "Ledger Economy" row) +- [x] Provisioned Grafana alerting: 13 rules / 5 groups, 2 contact points, + nested notification policy +- [ ] Tasks 9.14-9.17 closed — **open by design**: 9.14 documented-not-fixed + (defects 1, 3 and 4 still blocked; defect 2 unblocked but not yet fixed), + 9.15 and 9.16 not implemented, 9.17 deferred pending approval and volume + measurement --- @@ -591,7 +796,7 @@ volume cost that needs measuring before commitment. > Design for the provisioned Grafana alert rules (Task 9.9a). Previously a standalone spec; merged here so the phase plan is self-contained. **Date:** 2026-07-06 -**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513, Jira RIPD-5187) +**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513) **Status:** Approved ### Purpose @@ -599,8 +804,8 @@ volume cost that needs measuring before commitment. Phase 9 exports ~68 internal xrpld metrics and ships Grafana dashboards for them. This adds the missing operator-facing piece: **provisioned Grafana alert rules** that fire on the health-critical metrics phase 9 introduces. The -phase-9 task list (line 311) and Jira story RIPD-5187 both already list -"alerting rules" as a phase-9 deliverable, so this closes that gap. +phase-9 task list already lists "alerting rules" as a phase-9 deliverable +(Task 9.9), so this closes that gap. Scope is deliberately narrow — the three subsystems whose failure is node-fatal: **consensus/ledger health, validator health, job queue**. RPC/API @@ -609,9 +814,11 @@ health is explicitly out of scope. ### Why phase 9 (not phase 11) Every metric these alerts fire on is _born_ in phase 9 -(`xrpld_ledger_history_mismatch_total`, `xrpld_ledgers_closed_total`, -`xrpld_validation_missed_total`, `xrpld_validations_checked_total`, -`xrpld_jq_trans_overflow_total`, `xrpld_job_queued_duration_us_bucket`). Alerts +(`ledger_history_mismatch_total`, `ledgers_closed_total`, +`validation_missed_total`, `validations_checked_total`, +`jq_trans_overflow_total`, `job_queued_us_bucket` — the histogram instrument is +`job_queued_us` (`MetricsRegistry.cpp:94`), so the Prometheus bucket series is +`job_queued_us_bucket`, not `job_queued_duration_us_bucket`). Alerts belong with the metrics they watch, and this is where the dependency lives. ### Delivery @@ -623,11 +830,11 @@ Grafana auto-loads `provisioning/alerting/*.yaml`. New files under `docker/telemetry/grafana/provisioning/alerting/`: -| File | Purpose | -| -------------------- | ------------------------------------------------------------------------------------------------------------------- | -| `contactpoints.yaml` | One contact point `xrpld-default` (webhook to a documented placeholder; comments show how to swap for Slack/email). | -| `policies.yaml` | Default notification policy: route all alerts → `xrpld-default`, grouped by `alertname` + `service_instance_id`. | -| `rules.yaml` | 6 alert rules across 3 groups (below). | +| File | Purpose | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `contactpoints.yaml` | **Two** contact points: `xrpld-default` (Slack) and `xrpld-critical` (Slack + email). | +| `policies.yaml` | **Nested** notification policy: root route → `xrpld-default`; child route matching `severity = critical` → `xrpld-critical` (`repeat_interval: 1h` vs the root's `4h`). Both grouped by `alertname` + `service_instance_id`. | +| `rules.yaml` | **13** alert rules across **5** groups (below). | Plus the Alerting section of `docs/telemetry-runbook.md` — operator runbook: what each alert means, likely causes, and how to point the contact point at a @@ -642,16 +849,44 @@ Grafana rule shape: query (A) → reduce (B, last value) → threshold (C). All Alert rules run headless, so they cannot use the dashboards' `$node` template variables — they match all series and group by `service_instance_id` instead. -| Group | Alert | Expression (5m window) | Fires | `for` | severity | -| --------- | --------------------- | ----------------------------------------------------------------------------------------------------------- | --------------------- | ----- | -------- | -| Consensus | LedgerHistoryMismatch | `sum by (service_instance_id)(rate(xrpld_ledger_history_mismatch_total[5m]))` | `> 0` | 5m | critical | -| Consensus | LedgerCloseStalled | `sum by (service_instance_id)(rate(xrpld_ledgers_closed_total[5m]))` | `< 0.001` (≈0) | 3m | critical | -| Validator | ValidationsMissed | `sum by (service_instance_id)(rate(xrpld_validation_missed_total[5m]))` | `> 0` | 5m | warning | -| Validator | ValidationsNotChecked | `sum by (service_instance_id)(rate(xrpld_validations_checked_total[5m]))` | `< 0.001` (≈0) | 5m | warning | -| Job queue | JobQueueTxOverflow | `sum by (service_instance_id)(rate(xrpld_jq_trans_overflow_total[5m]))` | `> 0` | 5m | warning | -| Job queue | JobQueueLatencyHigh | `histogram_quantile(0.99, sum by (le, service_instance_id)(rate(xrpld_job_queued_duration_us_bucket[5m])))` | `> 1000000` (µs = 1s) | 5m | warning | +All 5 groups evaluate at `interval: 1m`. Metric names carry **no** `xrpld_` +prefix — `OTelCollectorImp::formatName()` adds none. -Each rule carries labels `severity` and `category` (consensus/validator/jobqueue) +The **Threshold** column is the rule's refId `C` evaluator, read straight from +`rules.yaml` — it is the firing condition, so it is load-bearing, not decoration. + +| Group | Alert | Expression (refId A) | Threshold (refId C) | `for` | severity | +| ------------------ | ----------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | ----- | -------- | +| `xrpld-consensus` | LedgerHistoryMismatch | `sum by (service_instance_id) (increase(ledger_history_mismatch_total[15m]))` | `gt [0]` | 2m | critical | +| `xrpld-consensus` | LedgerCloseStalled | `rate(ledgers_closed_total)` decayed to ≈0 | `lt [0.001]` | 3m | critical | +| `xrpld-consensus` | ValidatedLedgerStale | `max by (service_instance_id) (ledgermaster_validated_ledger_age < 1209600)` | `gt [60]` (seconds) | 5m | critical | +| `xrpld-validator` | ValidationsMissed | miss **ratio**, gated on send activity — see the expression below the table | `gt [0.1]` | 15m | warning | +| `xrpld-validator` | ValidationsNotChecked | `rate(validations_checked_total)` ≈0 | `lt [0.001]` | 5m | warning | +| `xrpld-jobqueue` | JobQueueTxOverflow | `sum by (service_instance_id) (increase(jq_trans_overflow_total[15m]))` | `gt [0]` | 2m | warning | +| `xrpld-jobqueue` | JobQueueLatencyHigh | `histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m])))` | `gt [1000000]` (µs = 1s) | 5m | warning | +| `xrpld-jobqueue` | NodeStoreIOLatencyHigh | `histogram_quantile(0.95, sum by (le, service_instance_id) (rate(ios_latency_milliseconds_bucket[10m])))` | `gt [1000]` (ms) | 10m | warning | +| `xrpld-node-state` | NodeStateFlapping | state-transition rate over the node-state series | `gt [3]` (transitions) | 15m | warning | +| `xrpld-node-state` | NodeNotFull | operating mode below FULL | `lt [4]` (FULL = 4) | 15m | warning | +| `xrpld-overlay` | ManifestJobQueueConvoy | `sum by (service_instance_id) (jobq_manifest_waiting)` | `gt [3]` (waiting jobs) | 10m | warning | +| `xrpld-overlay` | ManifestFloodInbound | inbound manifest byte rate | `gt [524288]` (B/s = 512 **KiB**/s, not 512 kB/s) | 10m | warning | +| `xrpld-overlay` | PeerResourceDisconnects | `sum by (service_instance_id) (increase(server_info{metric="peer_disconnects_resources"}[30m]))` | `gt [5]` | 5m | warning | + +**`ValidationsMissed` is a gated ratio, not `rate(...) > 0`.** The raw-rate shape +is the pre-fix version and it fires on **every non-validating node**: +`ValidationTracker` counts a miss whenever `weValidated && networkValidated` is +not both true, and a non-validator never sets `weValidated`, so its measured +ratio is exactly **1.0**. No threshold can separate "not a validator" from +"validator disagreeing", hence the `and on (...)` activity gate. The shipped +expression is: + +- numerator: `sum by (service_instance_id) (rate(validation_missed_total[15m]))` +- denominator: `clamp_min(` that same numerator `+ sum by (service_instance_id) (rate(validation_agreements_total[15m])), 1e-9)` +- gate: `and on (service_instance_id) (sum by (service_instance_id) (rate(validations_sent_total[15m])) > 0)` +- evaluator: `gt [0.1]` — i.e. >10% disagreement among nodes that do validate + +3 rules are `severity: critical`, 10 are `severity: warning`. + +Each rule carries labels `severity` and `category` and annotations `summary` + `description` (with `{{ $labels.service_instance_id }}` and `{{ $values.B.Value }}` interpolation). @@ -660,21 +895,31 @@ and `{{ $values.B.Value }}` interpolation). - **LedgerCloseStalled `< 0.001` for 3m**: healthy nodes close a ledger every ~3-5s; a 5m rate decaying to ~0 means the node is stuck. The epsilon (not exact `0`) avoids float rate-noise suppressing the alert. -- **JobQueueLatencyHigh 1s p99**: a default starting point, easy to tune — jobs - queued >1s at p99 indicate the node is saturated. -- Others are `> 0` on error/miss counters: any sustained nonzero rate is - actionable. +- **JobQueueLatencyHigh 1s p99**: `gt [1000000]` µs = 1s. A default starting + point, easy to tune — jobs queued >1s at p99 indicate the node is saturated. +- **ValidationsMissed `> 0.1` on a gated ratio**, not `> 0` on a raw rate: the + raw rate is permanently nonzero (ratio 1.0) on non-validators, so a `> 0` rule + pages on every non-validating node in the fleet. See the note above the + rationale list. +- **ManifestFloodInbound 524288 B/s**: an earlier 50 kB/s threshold produced ~41 + sustained 5-minute samples on healthy nodes; 512 KiB/s clears normal + manifest-exchange peaks. +- Remaining `gt [0]` rules (`LedgerHistoryMismatch`, `JobQueueTxOverflow`) sit on + true error counters where any sustained nonzero rate is actionable. ### Non-goals / YAGNI - No per-alert silencing schedules, no mute timings. -- No RPC/API, overlay, or fee-market alerts (dashboards cover those visually). -- Single contact point — multi-receiver routing is left to the operator. +- No RPC/API or fee-market alerts (dashboards cover those visually). Overlay + alerts _were_ added during implementation — the `xrpld-overlay` group carries + three (manifest convoy, manifest flood, peer resource disconnects). +- Two contact points and a two-level policy tree shipped; deeper routing + (Discord, PagerDuty, per-team splits) is left to the operator. ### Verification 1. `yamllint` (or `python -c yaml.safe_load`) on all three YAML files. 2. `docker compose -f docker/telemetry/docker-compose.yml config -q` still parses. 3. Optional live check: start stack, `GET /api/v1/provisioning/alert-rules` - returns the 6 rules; Grafana logs show no provisioning errors. + returns the 13 rules; Grafana logs show no provisioning errors. 4. Code-review pass (subagent) against phase conventions before commit. diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index 54d3936c42..f85f3bdcff 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -1627,8 +1627,18 @@ validators.txt # #------------------------------------------------------------------------------- # -# Enables distributed tracing via OpenTelemetry. Requires building with -# -DXRPL_ENABLE_TELEMETRY=ON (telemetry Conan option). +# Enables distributed tracing via OpenTelemetry. This section only has an +# effect if tracing was compiled in: build with CMake -Dtelemetry=ON (or Conan +# -o telemetry=True), and build it out with -Dtelemetry=OFF (or +# -o telemetry=False), which reduces all tracing code to no-ops. The option is +# currently ON so that CI compiles the telemetry code paths; OFF is the +# intended default once this feature is merged, so pass the value you want +# rather than relying on the default. +# +# Note that -DXRPL_ENABLE_TELEMETRY=OFF does NOT work: XRPL_ENABLE_TELEMETRY is +# a compile definition added by the build, not a CMake option, so it disables +# nothing. CMake only lists it at the end of configuration under +# "Manually-specified variables were not used by the project". # # [telemetry] # @@ -1647,12 +1657,29 @@ validators.txt # OTel resource attribute `service.instance.id`. Uniquely identifies # this node. Default: the node's public key (auto-detected). # +# SET THIS EXPLICITLY IF YOU USE THE METRICS PIPELINE. The node-public-key +# fallback only reaches traces: the metrics resource is built during +# startup, before the node key is known, and cannot be changed afterwards. +# With this key unset, metrics export with an empty service.instance.id and +# the per-node filter on the Grafana dashboards has nothing to split on. +# # endpoint=http://localhost:4318/v1/traces # -# The OTLP/HTTP exporter endpoint. The server sends trace data as -# protobuf-encoded HTTP POST requests to this URL. +# The OTLP/HTTP exporter endpoint for TRACES. The server sends trace data +# as protobuf-encoded HTTP POST requests to this URL. # Default: http://localhost:4318/v1/traces. # +# beast::insight metrics ([insight] server=otel) follow this setting: a +# trailing /v1/traces is rewritten to /v1/metrics. +# +# metrics_endpoint=http://localhost:4318/v1/metrics +# +# The OTLP/HTTP exporter endpoint for the internal metrics pipeline +# (the XRPL_METRIC_* instruments). This is a separate setting from +# `endpoint` and does NOT follow it, so a node exporting to a remote +# collector must set both. +# Default: http://localhost:4318/v1/metrics. +# # --- TLS settings for the OTLP exporter connection --- # # use_tls=0 diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index 7cf2b19fa0..3ca7d17562 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -46,13 +46,21 @@ docker compose -f docker/telemetry/docker-compose.yml up -d Wait for services to be ready: ```bash -# otel-collector health -curl -sf http://localhost:13133/ && echo "collector ready" +# otel-collector readiness: any HTTP response on the OTLP/HTTP port means the +# receiver is listening. Do NOT use `curl -sf` here — a GET of / returns 404, +# which -f treats as failure even when the collector is healthy. +[ "$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/)" != "000" ] && + echo "collector ready" # Tempo readiness curl -sf http://localhost:3200/ready >/dev/null && echo "tempo ready" ``` +> The collector's `health_check` extension listens on **13133**, but +> `docker-compose.yml` publishes only 4317, 4318 and 8889 — so 13133 is not +> reachable from the host with the base stack. It is published only by the +> Phase-10 workload stack (`docker-compose.workload.yaml`). + ### Step 2: Start xrpld in standalone mode ```bash @@ -372,28 +380,62 @@ See the "Verification Queries" section below. ## Expected Span Catalog -All 16 production span names instrumented across Phases 2-5: +What follows is a **trigger** catalogue, not an attribute reference: one row per +span-name family, saying which config toggle gates it and what you have to do to +make it appear. It covers all 41 span-name families the code emits, in eight +subsystem groups — RPC (5), gRPC (1), Transaction (6), TxQ (6), Consensus (13), +Ledger (4), Peer (2), PathFind (4). -| Span Name | Source File | Phase | Key Attributes | How to Trigger | -| --------------------------- | ----------------- | ----- | ---------------------------------------------------------------------------------------- | ------------------------- | -| `rpc.http_request` | ServerHandler.cpp | 2 | -- | Any HTTP RPC call | -| `rpc.ws_upgrade` | ServerHandler.cpp | 2 | -- | WebSocket upgrade | -| `rpc.ws_message` | ServerHandler.cpp | 2 | -- | WebSocket RPC message | -| `rpc.process` | ServerHandler.cpp | 2 | -- | RPC processing | -| `rpc.command.` | RPCHandler.cpp | 2 | `command`, `version`, `rpc_role` | Any RPC command | -| `tx.process` | NetworkOPs.cpp | 3 | `xrpl.tx.hash`, `local`, `path` | Submit transaction | -| `tx.receive` | PeerImp.cpp | 3 | `xrpl.peer.id` | Peer relays transaction | -| `consensus.proposal.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.round` | Consensus proposing phase | -| `consensus.ledger_close` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.mode` | Ledger close event | -| `consensus.accept` | RCLConsensus.cpp | 4 | `xrpl.consensus.proposers`, `xrpl.consensus.round_time_ms` | Ledger accepted | -| `consensus.validation.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.proposing` | Validation sent | -| `consensus.accept.apply` | RCLConsensus.cpp | 4 | `xrpl.consensus.close_time`, `close_time_correct`, `close_resolution_ms`, `state` | Ledger apply + close time | -| `tx.apply` | BuildLedger.cpp | 5 | `xrpl.ledger.tx_count`, `xrpl.ledger.tx_failed` | Ledger close (tx set) | -| `ledger.build` | BuildLedger.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build | -| `ledger.validate` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.validations` | Ledger validated | -| `ledger.store` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq` | Ledger stored | -| `peer.proposal.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `proposal_trusted` | Peer sends proposal | -| `peer.validation.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `validation_trusted` | Peer sends validation | +For each span's **attributes** — span name, source file, full attribute set and +description, per subsystem — see +[`docs/telemetry-runbook.md`](../../docs/telemetry-runbook.md) **§ Span +Reference**; its **§ Protocol Span Flow** gives the parent/child shape of a trace +and calls out where telemetry parenting deliberately differs from the protocol +flow. Both are kept in step with the code, so they are the reference to trust. +One hole worth knowing: the runbook's Span Reference tables have no row for +`grpc.` (it appears only in Protocol Span Flow). Its attributes are +`method`, `grpc_role` and `grpc_status`, emitted from `GRPCServer.cpp` with the +key constants in `src/xrpld/app/main/GrpcSpanNames.h`. + +If you find an older inline span inventory in this file or elsewhere, do not +trust it — the copy that used to live here had drifted badly (18 rows under a +"16 spans" heading, whole families missing, and pre-rename dotted `xrpl.*` +attribute keys the code no longer emits). The code and the runbook are the source +of truth. + +### Span → How to Trigger + +"Test" is the section of this file that exercises the family. `T1` = Test 1 +(standalone), `T2` = Test 2 (6-node network). + +| Span family (count) | Config toggle | How to trigger | Test | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| **RPC** (5 total, 3 here): `rpc.http_request`, `rpc.process`, `rpc.command.` | `trace_rpc=1` | Any HTTP JSON-RPC call: `curl -s http://localhost:5005 -d '{"method":"server_info"}'`. `rpc.command.` is one family — the command name is part of the span name. | T1 | +| **RPC** (cont.): `rpc.ws_message`, `rpc.ws_upgrade` | `trace_rpc=1` | Needs a WebSocket client against `[port_ws_public]` (**6005**) or `[port_ws_admin_local]` (6006). `rpc.ws_upgrade` covers the handshake — force a failure to see its error path. `curl` alone will not do it. | — | +| **gRPC** (1): `grpc.` | `trace_rpc=1` | Call a gRPC method (`GetLedger`, `GetLedgerData`, …). **Requires a `[port_grpc]` stanza — the shipped `xrpld-telemetry*.cfg` files define none**, so add one first. | — | +| **Transaction** (6 total, 4 here): `tx.process`, `tx.preflight`, `tx.preclaim`, `tx.transactor` | `trace_transactions` | Submit any transaction (T1 Step 4). The three apply-stage spans share the tx's deterministic trace id; the `stage` attribute says where a failing tx stopped. | T1 | +| **Transaction** (cont.): `tx.receive` | `trace_transactions` | A **peer** relays a transaction. Never appears in standalone — submit on one node of the cluster and look on another. | T2 | +| **Transaction** (cont.): `tx.apply` | `trace_transactions` | Ledger close with a non-empty transaction set: submit, then `ledger_accept` (T1) or wait for consensus (T2). | T1 / T2 | +| **TxQ** (6): `txq.enqueue`, `txq.apply_direct`, `txq.batch_clear`, `txq.accept`, `txq.accept_tx`, `txq.cleanup` | `trace_transactions` | `txq.enqueue`/`apply_direct` on every submission; `txq.accept`/`accept_tx`/`cleanup` on every ledger close. To force real queueing, submit faster than ledgers close or with a fee below the required fee level. | T1 | +| **Consensus** (13): `consensus.round`, `.phase.open`, `.establish`, `.update_positions`, `.check`, `.proposal.send`, `.ledger_close`, `.accept`, `.accept.apply`, `.validation.send`, `.mode_change`, `.proposal.receive`, `.validation.receive` | `trace_consensus=1` | Requires real consensus — **standalone emits none of these**. Bring up T2 and wait for nodes to reach `proposing`; one `consensus.round` per close. `.mode_change` needs an actual mode transition (stop/start a node). | T2 | +| **Ledger** (4 total, 3 here): `ledger.build`, `ledger.validate`, `ledger.store` | `trace_ledger=1` | Any ledger close: `ledger_accept` in standalone, or consensus in T2. | T1 / T2 | +| **Ledger** (cont.): `ledger.acquire` | `trace_ledger=1` | Node fetches a **missing** ledger from peers. Start a node with no history against a running cluster, or restart one node after the others have advanced. | T2 | +| **Peer** (2): `peer.proposal.receive`, `peer.validation.receive` | `trace_peer=1` | Inbound consensus messages from peers; fresh trace roots. T2 only, and high volume. | T2 | +| **PathFind** (4): `pathfind.request`, `pathfind.compute`, `pathfind.discover`, `pathfind.update_all` | `trace_rpc=1` | `curl -s http://localhost:5005 -d '{"method":"ripple_path_find","params":[{"source_account":"…","destination_account":"…","destination_amount":"100"}]}'`. `pathfind.update_all` fires on ledger close while a request is active. | T1 | + +Notes that matter when a span you expect is missing: + +- **Toggles are per-subsystem and all default to on** (`trace_rpc`, + `trace_transactions`, `trace_consensus`, `trace_peer`, `trace_ledger`), but + `[telemetry] enabled` defaults to **0** — nothing is emitted until it is `1`. +- **`consensus.*` and `peer.*` cannot be produced in standalone mode.** If Test 1 + shows none, that is correct behaviour, not a regression — see "Expected spans + (standalone mode)" above. +- **`rpc.ws_*` and `grpc.*` need a client and a port the quick tests do not + use.** Absence in T1/T2 is expected. +- Trace ids are deterministic for transactions (`txID[0:16]`) and consensus + rounds (`prevLedgerHash[0:16]`), so you can compute the id you expect rather + than searching for it. --- @@ -456,13 +498,17 @@ curl -s "$PROM/api/v1/query?query=span_calls_total" | Open http://localhost:3000 (anonymous admin access enabled). -Pre-configured dashboards: +Pre-configured dashboards: every `.json` under +`docker/telemetry/grafana/dashboards/` is provisioned into the `xrpld` folder — +`provisioning/dashboards/dashboards.yaml` points the file provider at +`/var/lib/grafana/dashboards`, which `docker-compose.yml` bind-mounts from that +directory. Adding a file there is all that is needed; there is no per-dashboard +registration. -- **RPC Performance**: Request rates, latency percentiles by command, top commands, WebSocket rate -- **Transaction Overview**: Transaction processing rates, apply duration, peer relay, failed tx rate -- **Consensus Health**: Consensus round duration, proposer counts, mode tracking, accept heatmap -- **Ledger Operations**: Build/validate/store rates and durations, TX apply metrics -- **Peer Network**: Proposal/validation receive rates, trusted vs untrusted breakdown (requires `trace_peer=1`) +For what each dashboard covers, see +[`docs/telemetry-runbook.md`](../../docs/telemetry-runbook.md) **§ Grafana +Dashboards** — the per-dashboard reference. Listing them here would be a second +copy that rots (this section previously named 5 of the 15 provisioned). Pre-configured datasources: @@ -504,10 +550,35 @@ docker compose -f docker/telemetry/docker-compose.yml \ -f docker/telemetry/docker-compose.grafanacloud.yaml up -d ``` -The override swaps the collector onto `otel-collector-config.grafanacloud.yaml`, -which keeps the local Tempo/Prometheus/Loki exporters and adds one -OTLP/HTTP exporter to Grafana Cloud on all three pipelines. Bring the stack -up with just the base file to return to local-only. +The override swaps the collector onto `otel-collector-config.grafanacloud.yaml`. +It keeps the local Tempo/Prometheus/Loki exporters and adds an +`otlphttp/grafanacloud` exporter, but it is **not** the base config plus one +exporter — it restructures the pipelines. Bring the stack up with just the base +file to return to local-only. + +Differences that change what you will see: + +| | Base (`otel-collector-config.yaml`) | Cloud override | +| ------------------- | ----------------------------------- | --------------------------------------------------------------------------------------------------- | +| Pipelines | 3: `traces`, `metrics`, `logs` | 5: `traces/metrics`, `traces/store`, `metrics/local`, `metrics/cloud`, `logs` | +| Trace sampling | none — 100% of spans reach Tempo | `tail_sampling` keeps **0.5%** (one `probabilistic` policy, `decision_wait: 10s`) on `traces/store` | +| `debug` exporter | present on `traces` | dropped | +| `attributes/hash` | present on `traces` | **omitted** | +| Cloud metric labels | n/a | `transform/cloudlabels` on `metrics/cloud` only | + +Consequences worth knowing before you debug against the cloud stack: + +- **Traces are sampled, span metrics are not.** Sampling sits only on + `traces/store` (the pipeline feeding Tempo _and_ Grafana Cloud). The + `spanmetrics` connector is fed by the separate, unsampled `traces/metrics` + pipeline, so `span_*` rates stay exact while only ~1 trace in 200 is + retrievable by trace ID. A trace you can see in a metric may not exist in + Tempo. +- **Pathfinding account hashing does not happen on the cloud export.** The base + config's `attributes/hash` processor hashes `pathfind_source_account` and + `pathfind_dest_account`. It is absent from every cloud pipeline, so those two + attributes leave for Grafana Cloud (and, on that config, for Tempo) with their + raw account values. ### Step 4: Verify data reaches Grafana Cloud @@ -516,7 +587,7 @@ Cloud instance and confirm: - **Traces**: Explore → hosted Tempo datasource → search `{resource.service.name="xrpld"}` - **Metrics**: Explore → hosted Prometheus/Mimir → query `span_calls_total` -- **Logs**: Explore → hosted Loki → query `{job="xrpld"}` (requires `warning`+ file logging) +- **Logs**: Explore → hosted Loki → query `{service_name="xrpld"}` (requires `warning`+ file logging). **Not `{job="xrpld"}`** — see the note under Test 3 Step 3. If nothing appears, check the collector logs for auth/export errors: @@ -576,12 +647,28 @@ exports parsed entries to Loki. Verify Loki has received entries: ```bash # Query Loki for any xrpld logs curl -sG "http://localhost:3100/loki/api/v1/query" \ - --data-urlencode 'query={job="xrpld"}' \ + --data-urlencode 'query={service_name="xrpld"}' \ --data-urlencode 'limit=5' | jq '.data.result | length' ``` Expected: > 0 results. +> **Use `service_name`, not `job`.** The collector's `resource/logs` processor +> applies an `upsert` to **both** `service.name=xrpld` and `job=xrpld` +> (`otel-collector-config.yaml:57-70`), and its comment says the `job` attribute +> is there so operators can paste `{job="xrpld"}`. That does not work: on OTLP +> ingest Loki promotes only an allow-listed set of resource attributes to indexed +> stream labels (`service.name` → `service_name`, plus `service.namespace`, +> `service.instance.id`, `deployment.environment`, `k8s.*`, `cloud.*`), and `job` +> is not on the list. This repo mounts no Loki config override — the `loki` +> service runs the image's built-in `/etc/loki/local-config.yaml` +> (`docker-compose.yml:75`) — so `job` lands in **structured metadata**, which +> cannot be a stream selector. `{job="xrpld"}` therefore returns **zero results +> with no error**, which reads exactly like "logs are not being ingested". If +> this query is empty, check `{service_name="xrpld"}` before debugging the +> pipeline. All 38 Loki queries in the shipped dashboards select on +> `service_name`; none uses `job`. + ### Step 4: Verify Grafana Tempo-to-Loki correlation 1. Open Grafana at http://localhost:3000 @@ -593,7 +680,7 @@ Expected: > 0 results. ### Step 5: Verify Grafana Loki-to-Tempo correlation 1. In Grafana **Explore**, select **Loki** datasource -2. Query: `{job="xrpld"} |= "trace_id="` +2. Query: `{service_name="xrpld"} |= "trace_id="` 3. In the log results, click the **TraceID** derived field link 4. Verify it navigates to the full trace in Tempo @@ -620,9 +707,10 @@ Expected: > 0 results. docker compose -f docker/telemetry/docker-compose.yml logs otel-collector ``` 2. Verify xrpld telemetry config has `enabled=1` and correct endpoint -3. Check that otel-collector port 4318 is accessible: +3. Check that otel-collector port 4318 is accessible (`-f` would fail on the + receiver's 404 for `GET /`, so test for any HTTP status instead): ```bash - curl -sf http://localhost:4318 && echo "reachable" + curl -so /dev/null -w '%{http_code}\n' http://localhost:4318/ ``` 4. Increase `batch_delay_ms` or decrease `batch_size` in xrpld config @@ -694,14 +782,21 @@ Expected: > 0 results. ### Spanmetrics not appearing in Prometheus 1. Verify otel-collector config has `spanmetrics` connector -2. Check that the metrics pipeline is configured: +2. Check that the metrics pipeline matches `otel-collector-config.yaml` + verbatim: ```yaml service: pipelines: metrics: - receivers: [spanmetrics] + receivers: [otlp, spanmetrics] + processors: [resource/tier, resource/stripsdk, batch] exporters: [prometheus] ``` + Both receivers are required. `spanmetrics` carries the span-derived + `span_*` series; `otlp` carries the node's native `beast::insight` / + MetricsRegistry metrics, which arrive on the same OTLP port. Dropping + `otlp` silently removes every native metric while the `span_*` ones keep + working — so the dashboards only half-break. 3. Verify Prometheus can reach collector: ```bash curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets' diff --git a/docker/telemetry/grafana/dashboards/validate_dashboards.py b/docker/telemetry/grafana/dashboards/validate_dashboards.py index d1b17dbbf6..07240e6e1c 100755 --- a/docker/telemetry/grafana/dashboards/validate_dashboards.py +++ b/docker/telemetry/grafana/dashboards/validate_dashboards.py @@ -39,6 +39,7 @@ NODESTORE_CUMULATIVE = ( "node_read_bytes", "node_written_bytes", "node_reads_duration_us", + "node_writes_duration_us", ) # state_accounting_*_duration are cumulative µs. STATE_DURATION = re.compile(r"state_accounting_\w+_duration") diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml index a66e1fcceb..0a8b294fda 100644 --- a/docker/telemetry/grafana/provisioning/alerting/rules.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -2,7 +2,7 @@ # # Phase 9: Internal metric gap fill — alerting on health-critical metrics. # -# Twelve rules across five subsystems: consensus/ledger health, validator +# Thirteen rules across five subsystems: consensus/ledger health, validator # health, the job queue, node operating state, and the overlay (manifests). # # Rule shape (Grafana server-side evaluation): @@ -845,10 +845,10 @@ groups: # 24h window that an earlier revision used: # healthy p95 0.2-0.5 kB/s, p99 1.0-1.8 kB/s # observed peaks up to 2.7 MB/s during real manifest storms - # 512 kB/s sits ~280x above healthy p99 and ~5x below the peaks. An - # earlier 50 kB/s threshold produced ~41 sustained 5-min samples across - # six healthy nodes over six days (i.e. routine paging); 512 kB/s reduces - # that to 2 while still catching every genuine storm. + # 512 KiB/s (524288 B/s) sits ~280x above healthy p99 and ~5x below the + # peaks. An earlier 50 kB/s threshold produced ~41 sustained 5-min samples + # across six healthy nodes over six days (i.e. routine paging); + # 512 KiB/s reduces that to 2 while still catching every genuine storm. # # The uptime gate exists because the startup manifest burst is MEASURED # NORMAL behaviour. It does not hide real floods — the same 7-day sample @@ -869,7 +869,8 @@ groups: summary: "Inbound manifest flood on {{ $labels.service_instance_id }}" description: >- Node {{ $labels.service_instance_id }} is receiving - {{ $values.B.Value }} B/s of manifest traffic (>512 kB/s) over 10m. + {{ $values.B.Value }} B/s of manifest traffic over 10m, above the + 512 KiB/s (524288 B/s) threshold. A peer is flooding oversized TMManifests dumps. data: - refId: A diff --git a/docker/telemetry/xrpld-telemetry-mainnet.cfg b/docker/telemetry/xrpld-telemetry-mainnet.cfg index 9bd8779483..28ebce9567 100644 --- a/docker/telemetry/xrpld-telemetry-mainnet.cfg +++ b/docker/telemetry/xrpld-telemetry-mainnet.cfg @@ -9,7 +9,7 @@ # 1. Start the observability stack: # docker compose -f docker/telemetry/docker-compose.yml up -d # 2. Run xrpld: -# ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg +# ./xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg # 3. Wait for sync (server_state=full), then exercise workflows: # curl -s http://localhost:5005 -d '{"method":"server_info"}' # 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 @@ -134,7 +134,6 @@ enabled=1 service_instance_id=xrpld-mainnet endpoint=http://localhost:4318/v1/traces metrics_endpoint=http://localhost:4318/v1/metrics -exporter=otlp_http # Mainnet has high span throughput across peer/ledger/consensus. Head # sampling is fixed at 1.0 (sample everything) and not configurable; # reduce Tempo/collector load with collector-side tail sampling. diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg index bd7454e598..b362465b7a 100644 --- a/docker/telemetry/xrpld-telemetry.cfg +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -125,7 +125,6 @@ enabled=1 service_instance_id=xrpld-devnet endpoint=http://localhost:4318/v1/traces metrics_endpoint=http://localhost:4318/v1/metrics -exporter=otlp_http batch_size=512 batch_delay_ms=5000 max_queue_size=2048 diff --git a/docs/build/telemetry.md b/docs/build/telemetry.md index eca4e9f110..8735a652ca 100644 --- a/docs/build/telemetry.md +++ b/docs/build/telemetry.md @@ -28,13 +28,19 @@ When enabled, it instruments RPC requests with trace spans that are exported via OTLP/HTTP to an OpenTelemetry Collector, which forwards them to a tracing backend such as Grafana Tempo. -Telemetry is **off by default** at both compile time and runtime: +Telemetry is gated twice — once at compile time and once at runtime: -- **Compile time**: The Conan option `telemetry` and CMake option `telemetry` must be set to `True`/`ON`. - When disabled, all `SpanGuard` calls compile to inline no-ops (defined in `SpanGuard.h`) +- **Compile time**: The Conan option `telemetry` and CMake option `telemetry` decide + whether the OTel SDK is linked in and `XRPL_ENABLE_TELEMETRY` is defined. + When off, all `SpanGuard` calls compile to inline no-ops (defined in `SpanGuard.h`) with zero overhead — no OTel SDK dependency required. -- **Runtime**: The `[telemetry]` config section must set `enabled=1`. - When disabled at runtime, a no-op implementation is used. + The option is currently `True`/`ON` on the telemetry branches so that CI builds and + exercises the instrumented code; **`False`/`OFF` is the intended default once this + feature is merged.** Pass the value you want explicitly rather than relying on the + default. +- **Runtime**: Telemetry is **off by default** — the `[telemetry]` config section must + set `enabled=1`. When disabled at runtime, a no-op implementation is used even in a + build that has the SDK compiled in. ## Building with Telemetry @@ -102,12 +108,22 @@ cmake --build . --parallel $(nproc) ## Building without telemetry -Omit the `-o telemetry=True` option (or pass `-o telemetry=False`). +Pass `-o telemetry=False` to `conan install`, and `-Dtelemetry=OFF` to CMake if you +configure without the Conan-generated toolchain. Do not just omit the option — it then +resolves to whatever the current default is, and that default is `True` on the +telemetry branches. + The `opentelemetry-cpp` dependency will not be downloaded, the `XRPL_ENABLE_TELEMETRY` preprocessor define will not be set, and all tracing macros will compile to no-ops. The resulting binary is identical to one built before telemetry support was added. +> **`-DXRPL_ENABLE_TELEMETRY=OFF` disables nothing.** `XRPL_ENABLE_TELEMETRY` is not a +> CMake option — it is only a compile definition added when `telemetry` is on. Passing it +> on the command line leaves telemetry compiled in; CMake merely lists it at the end of +> configuration under `Manually-specified variables were not used by the project`. +> Use `-Dtelemetry=OFF`. + ## Troubleshooting ### Conan lockfile error diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index fa38be7a3d..10261f0cd1 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -579,9 +579,9 @@ A cluster is a set of servers run by the same operator that trust each other, ex **Scope:** cluster-wide — shared across a co-operated cluster of nodes run by one operator. -**What is observable:** cluster overhead is **not** measurable today. Cluster messages are counted under `unknown` rather than `overhead_cluster`, so the `overhead_cluster_*` series read zero on a clustered node — treat them as "no data", not "no cluster traffic". The churn guidance above cannot yet be acted on. +**What is observable:** cluster overhead is **not** measurable today, and this is a gap in the instrumentation rather than a display problem. The cluster message type is not in the overlay's message-to-category lookup table and none of the fallback branches match it, so every cluster message falls through to the `unknown` category (`src/xrpld/overlay/detail/TrafficCount.cpp`); no code path ever reports the cluster category, even though the `overhead_cluster` name is defined. Consequences: the `overhead_cluster_*` series read zero on a clustered node — treat them as "no data", not "no cluster traffic" — the churn guidance above cannot yet be acted on, and `unknown_*` is a weaker anomaly signal on a clustered node because it mixes genuinely unrecognized wire types with routine cluster traffic. If this is ever instrumented, volume moves out of `unknown_bytes_in`, so any alert threshold set against that series will need re-baselining. -**See also:** [Cluster on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/clustering) · [Data collection reference §6.0](../OpenTelemetryPlan/09-data-collection-reference.md#60-mtcluster-is-counted-as-unknown-not-implemented) +**See also:** [Cluster on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/clustering) @@ -649,9 +649,7 @@ Each peer connection is probed on a timer: the node sends a ping carrying a rand **Scope:** per node — measured on and specific to this individual server. -**What is observable:** only the p90 of the smoothed per-peer latency (`peer_quality{metric="peer_latency_p90_ms"}`) — there is no distribution, ping timeouts and wrong-cookie pongs have no counter, and ping bytes are not separable from status-change bytes because both share the `overhead` traffic category. - -**See also:** [Data collection reference §6.3](../OpenTelemetryPlan/09-data-collection-reference.md#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented) +**What is observable:** only the p90 of the smoothed per-peer latency (`peer_quality{metric="peer_latency_p90_ms"}`), and three things are not instrumented. First, the per-peer round-trip is an 8-sample moving average and is exported as that single p90 gauge, with no histogram — a bimodal peer set (a few very slow peers behind many fast ones) reads as one middling number. Second, neither failure mode is counted: a ping timeout only logs before dropping the peer, and a pong bearing the wrong cookie is discarded silently (`src/xrpld/overlay/detail/PeerImp.cpp`), so keepalive-driven drops cannot be separated from any other disconnect cause. Third, ping bytes are not separable from status-change bytes, because both message types share the `overhead` traffic category — so `overhead_*` cannot be read as keepalive volume. Peer discovery traffic is separable (it lands in `overhead_overlay_*`) but has no counters of its own for endpoints received, handed out or malformed. @@ -709,9 +707,7 @@ Squelching is a relay-control mechanism: a node tells peers to stop sending it a **Scope:** per node — measured on and specific to this individual server. -**What is observable:** read ignored directives on `squelch_ignored_messages_in/out` only. The paired `squelch_ignored_bytes_*` series are always zero because the ignored-squelch callback records no size, so bandwidth wasted by peers ignoring squelch cannot be quantified — and `squelch_ignored` is therefore not comparable on bytes against `squelch_suppressed`, which does record real sizes. - -**See also:** [Data collection reference §6.1](../OpenTelemetryPlan/09-data-collection-reference.md#61-squelch_ignored-byte-counts-not-implemented) +**What is observable:** read ignored directives on `squelch_ignored_messages_in` only. The size is not instrumented: both call sites that report an ignored squelch pass a hardcoded byte count of zero (`src/xrpld/overlay/detail/OverlayImpl.cpp`), so `squelch_ignored_bytes_in` is always zero and the bandwidth wasted by peers ignoring squelch cannot be quantified, nor can a bytes-per-message ratio be built from this category. `squelch_suppressed` does record the real wire size, so the two squelch categories are not comparable on bytes — only on message counts. The outbound side of this category is never reported at all, so `squelch_ignored_bytes_out` and `squelch_ignored_messages_out` are also permanently zero; that is expected, since "ignoring a squelch" is something a remote peer does to us and is therefore only ever observed inbound. diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 20ad2b7bcd..b208890045 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -120,25 +120,32 @@ curl -s http://localhost:5005 -d '{"method":"server_info"}' | ## Configuration Reference -| Option | Default | Description | -| -------------------------- | --------------------------------- | --------------------------------------------------------- | -| `enabled` | `0` | Master switch for telemetry | -| `endpoint` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint | -| `service_name` | `xrpld` | OpenTelemetry service name resource attribute | -| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute | -| `trace_rpc` | `1` | Enable RPC request tracing | -| `trace_transactions` | `1` | Enable transaction tracing | -| `trace_consensus` | `1` | Enable consensus tracing | -| `trace_peer` | `1` | Enable peer message tracing (high volume) | -| `trace_ledger` | `1` | Enable ledger tracing | -| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy (`deterministic` or `random`) | -| `batch_size` | `512` | Max spans per batch export | -| `batch_delay_ms` | `5000` | Delay between batch exports | -| `max_queue_size` | `2048` | Max spans queued before dropping | -| `use_tls` | `0` | Use TLS for exporter connection | -| `tls_ca_cert` | (empty) | Path to CA certificate bundle | -| `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | -| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | +| Option | Default | Description | +| -------------------------- | --------------------------------- | ------------------------------------------------------------ | +| `enabled` | `0` | Master switch for telemetry | +| `endpoint` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint | +| `service_name` | `xrpld` | OpenTelemetry service name resource attribute | +| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute | +| `trace_rpc` | `1` | Enable RPC request tracing | +| `trace_transactions` | `1` | Enable transaction tracing | +| `trace_consensus` | `1` | Enable consensus tracing | +| `trace_peer` | `1` | Enable peer message tracing (high volume) | +| `trace_ledger` | `1` | Enable ledger tracing | +| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy (`deterministic` or `attribute`) | +| `batch_size` | `512` | Max spans per batch export | +| `batch_delay_ms` | `5000` | Delay between batch exports | +| `max_queue_size` | `2048` | Max spans queued before dropping | +| `use_tls` | `0` | Use TLS for exporter connection | +| `tls_ca_cert` | (empty) | Path to CA certificate bundle | +| `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | +| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | + +> **`consensus_trace_strategy` is not validated.** The parser copies the raw +> string through (`TelemetryConfig.cpp:155-156`) and the only equality test in +> the code is `strategy == "attribute"` (`RCLConsensus.cpp:1296`). Any other +> value — including a typo such as `determinstic` — silently selects the +> deterministic branch. There is no warning in the log. The two accepted values +> are documented at `include/xrpl/telemetry/Telemetry.h:287-292`. ## Exporting to Grafana Cloud @@ -240,17 +247,28 @@ All spans instrumented in xrpld, grouped by subsystem: | -------------------- | ----------------- | ----------------------------------------------------------- | ----------------------------------------------------- | | `rpc.http_request` | ServerHandler.cpp | `request_payload_size` | Top-level HTTP RPC request | | `rpc.ws_upgrade` | ServerHandler.cpp | — | WebSocket upgrade handshake | -| `rpc.ws_message` | ServerHandler.cpp | `command` | WebSocket RPC message | +| `rpc.ws_message` | ServerHandler.cpp | `command`, `rpc_status` | WebSocket RPC message | | `rpc.process` | ServerHandler.cpp | `is_batch`, `batch_size` | RPC processing (child of rpc.http_request/ws_message) | | `rpc.command.` | RPCHandler.cpp | `command`, `version`, `rpc_role`, `rpc_status`, `load_type` | Per-command span (e.g., `rpc.command.server_info`) | +On `rpc.ws_message`, `rpc_status` is set **on four of the five error paths** +(resource threshold exceeded, bad API version / missing command, caught +exception, and an error in the command result — `ServerHandler.cpp:489`, `:522`, +`:571`, `:608`). The exception is the **invalid-JSON / oversized-request** path, +which opens its own `rpc.ws_message` span and calls only `setError()`, writing no +`rpc_status` at all (`ServerHandler.cpp:392-395`) — those rejections are visible +solely through `status_code="ERROR"`. The success path calls `setOk()` and writes +no `rpc_status` either, so there is never an `rpc_status="success"` series for +this span: count successes as total minus error, or filter on `status_code`. +`rpc.command.*` is unaffected — it sets `rpc_status` on both outcomes. + ### Transaction Spans | Span Name | Source File | Attributes | Description | | --------------- | --------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------- | | `tx.process` | NetworkOPs.cpp | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied`, `current_ledger_seq` | Transaction submission and processing | | `tx.receive` | PeerImp.cpp | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status`, `current_ledger_seq` | Transaction received from peer relay | -| `tx.apply` | BuildLedger.cpp | `ledger_seq`, `tx_count`, `tx_failed` | Transaction set applied per ledger | +| `tx.apply` | BuildLedger.cpp | `tx_count`, `tx_failed` | Transaction set applied per ledger | | `tx.preflight` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Stateless checks stage | | `tx.preclaim` | applySteps.cpp | `stage`, `tx_type`, `ter_result`, `current_ledger_seq`, `current_ledger_hash` | Ledger-aware checks stage | | `tx.transactor` | Transactor.cpp | `stage`, `tx_type`, `ter_result`, `applied`, `current_ledger_seq`, `current_ledger_hash` | Apply stage (transactor runs) | @@ -268,6 +286,11 @@ txID-keyed spans can be joined to the ledger trace it targeted `tx.transactor`) also carry `current_ledger_hash` (the current ledger's parent hash); `tx.preflight` is stateless and omits both. +`tx.apply` carries **no** `ledger_seq` of its own — the sequence is set on its +parent `ledger.build` +([BuildLedger.cpp:90](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L90)), so +read it from the parent rather than filtering `tx.apply` on it. + ### Transaction Queue Spans | Span Name | Source File | Attributes | Description | @@ -281,37 +304,56 @@ hash); `tx.preflight` is stateless and omits both. ### PathFinding Spans -| Span Name | Source File | Attributes | Description | -| --------------------- | --------------------------------- | -------------------------------------------------- | ------------------------------------------------------- | -| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | -| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | -| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths` | Graph exploration (one per RPC call in `findPaths`) | -| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | +| Span Name | Source File | Attributes | Description | +| --------------------- | --------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------- | +| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | +| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | +| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths`, `pathfind_num_source_assets` | Graph exploration (one per RPC call in `findPaths`) | +| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | ### Consensus Spans -| Span Name | Source File | Attributes | Description | -| ------------------------------ | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `consensus.round` | RCLConsensus.cpp | `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id` | Root span for a consensus round (deterministic or random trace ID) | -| `consensus.phase.open` | Consensus.h | -- | Open phase duration (child of round) | -| `consensus.proposal.send` | RCLConsensus.cpp | `consensus_round`, `is_bow_out` | Consensus proposal broadcast | -| `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | -| `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | -| `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count` | Position update and dispute resolution (see Events below) | -| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | -| `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | -| `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below) | -| `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | -| `consensus.mode_change` | RCLConsensus.cpp | `mode_old`, `mode_new` | Consensus mode transition | -| `consensus.proposal.receive` | PeerImp.cpp | `proposal_trusted`, `consensus_round` | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | -| `consensus.validation.receive` | PeerImp.cpp | `validation_trusted`, `ledger_seq` | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | +| Span Name | Source File | Attributes | Description | +| ------------------------------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | RCLConsensus.cpp | `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id` | Root span for a consensus round (deterministic or random trace ID) | +| `consensus.phase.open` | Consensus.h | `open_duration_ms`, `peer_positions_at_close` (both only if the span is still live at `closeLedger()`) | Open phase duration (child of round) | +| `consensus.proposal.send` | RCLConsensus.cpp | `consensus_round`, `is_bow_out` | Consensus proposal broadcast | +| `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | +| `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | +| `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count`, `avalanche_threshold` (only when peer positions exist), `have_close_time_consensus`, `close_time_threshold` | Position update and dispute resolution (see Events below) | +| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | +| `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | +| `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count`, `disputes_resolved_count` | Ledger application with close time details (see Events below) | +| `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | +| `consensus.mode_change` | RCLConsensus.cpp | `mode_old`, `mode_new` | Consensus mode transition | +| `consensus.proposal.receive` | PeerImp.cpp | `proposal_trusted`, `consensus_round`, `prev_ledger_prefix`, `position_hash_prefix` | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | +| `consensus.validation.receive` | PeerImp.cpp | `validation_trusted`, `ledger_seq` (only when the validation carries `sfLedgerSequence`), `full_validation`, `validation_sign_time` | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | #### Consensus Span Events -| Parent Span | Event Name | Event Attributes | Description | -| ---------------------------- | ----------------- | ----------------------------------------------------------- | ------------------------------------------------------- | -| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied | -| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction included in the accepted ledger | +| Parent Span | Event Name | Event Attributes | Description | +| ---------------------------- | ------------------ | ----------------------------------------------------------- | -------------------------------------------------------- | +| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied | +| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction included in the accepted ledger | +| `consensus.round` | `phase.open` | -- | Round entered the open phase (also re-fired on recovery) | +| `consensus.round` | `phase.recovery` | -- | Round started with `StartRoundReason::Recovered` | +| `consensus.round` | `phase.establish` | -- | Round entered the establish phase on close | +| `consensus.round` | `phase.accepted` | -- | Round reached the accepted phase | +| `consensus.round` | `outcome.yes` | -- | Round settled with consensus reached | +| `consensus.round` | `outcome.moved_on` | -- | Round abandoned; the network moved on without us | +| `consensus.round` | `outcome.expired` | -- | Round expired without settling | + +The nine events above are the complete set. The seven on `consensus.round` +carry **no event attributes** — they are timestamps marking phase entry and the +terminal outcome, so a round's whole life reads off one span's event list. +Phase entry additionally rewrites the round's span-level `consensus_phase` +attribute, which is why `phase.recovery` is the one phase event that leaves +`consensus_phase` unchanged (it fires with an empty label). Evidence: +[RCLConsensus.cpp:1344](../src/xrpld/app/consensus/RCLConsensus.cpp#L1344), +[1386](../src/xrpld/app/consensus/RCLConsensus.cpp#L1386), +[1400](../src/xrpld/app/consensus/RCLConsensus.cpp#L1400); outcomes are chosen +from `result_->state` at +[Consensus.h:1517-1525](../include/xrpl/consensus/Consensus.h#L1517). #### Close Time Queries (Tempo TraceQL) @@ -339,18 +381,39 @@ Span attributes are filtered with `span.` inside `{}`. Combine conditions ### Ledger Spans -| Span Name | Source File | Attributes | Description | -| ----------------- | -------------------- | ------------------------------------- | ----------------------------- | -| `ledger.build` | BuildLedger.cpp:31 | `ledger_seq`, `tx_count`, `tx_failed` | Ledger build during consensus | -| `ledger.validate` | LedgerMaster.cpp:915 | `ledger_seq`, `validations` | Ledger promoted to validated | -| `ledger.store` | LedgerMaster.cpp:409 | `ledger_seq` | Ledger stored in history | +| Span Name | Source File | Attributes | Description | +| ----------------- | ----------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `ledger.build` | BuildLedger.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build during consensus | +| `ledger.validate` | LedgerMaster.cpp | `ledger_seq`, `validations` | Ledger promoted to validated | +| `ledger.store` | LedgerMaster.cpp | `ledger_seq` | Ledger stored in history | +| `ledger.acquire` | InboundLedger.cpp | `ledger_seq`, `acquire_reason`, `timeouts`, `peer_count`, `outcome` | Fetch a missing ledger from peers (parent varies — see [known issues](#where-telemetry-parenting-differs-from-protocol-flow)) | + +`ledger.acquire` sets only `ledger_seq` and `acquire_reason` when the span opens +in `init()`; `outcome`, `timeouts` and `peer_count` are written on the `done()` +path. All three are therefore **absent** when `init()` satisfies the ledger +straight from the local store and the acquire never runs — treat a missing +`outcome` as "never went to the network", not as a lost span. + +`ledger.build` does **not** carry `tx_count` / `tx_failed`. Those two live on its +child `tx.apply` span, which is where the set is actually applied +([BuildLedger.cpp:191](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L191)) — +join on the trace, not on one span. + +> **Gap: no `ledger.*` span carries a ledger hash.** The attribute constant +> `ledger_span::attr::ledgerHash` is declared +> ([LedgerSpanNames.h:41](../src/xrpld/app/ledger/detail/LedgerSpanNames.h#L41)) +> but is never set by any call site, so `ledger.build` / `ledger.store` / +> `ledger.validate` / `ledger.acquire` are identifiable by `ledger_seq` only. A +> query filtering on `span.ledger_hash` over a `ledger.*` span returns nothing. +> `ledger_hash` **is** set on `consensus.validation.send` and on the peer spans, +> so use those when a hash is required. ### Peer Spans -| Span Name | Source File | Attributes | Description | -| ------------------------- | ---------------- | ------------------------------- | ----------------------------- | -| `peer.proposal.receive` | PeerImp.cpp:1667 | `peer_id`, `proposal_trusted` | Proposal received from peer | -| `peer.validation.receive` | PeerImp.cpp:2264 | `peer_id`, `validation_trusted` | Validation received from peer | +| Span Name | Source File | Attributes | Description | +| ------------------------- | ----------- | ------------------------------- | ----------------------------- | +| `peer.proposal.receive` | PeerImp.cpp | `peer_id`, `proposal_trusted` | Proposal received from peer | +| `peer.validation.receive` | PeerImp.cpp | `peer_id`, `validation_trusted` | Validation received from peer | Both peer receive spans are `kConsumer` inbound entry points started as fresh trace roots. They never inherit an ambient span left active on the peer thread, @@ -692,11 +755,11 @@ in `Establish` across many heartbeats until the outcome is decided. > 1. **Avalanche rounds inside one Establish phase** — each `timerEntry` runs > `phaseEstablish` again (`establishCounter_++`) and raises the inclusion > threshold **50% → 65% → 70% → 95%** as the round ages -> ([ConsensusParms.h:145](../src/xrpld/consensus/ConsensusParms.h#L145)). +> ([ConsensusParms.h:145](../include/xrpl/consensus/ConsensusParms.h#L145)). > `checkConsensus` returning `No` keeps the node in `Establish` and loops; a > round cannot even `Expire` before a minimum of > `avalancheCutoffs.size() × avMinRounds = 4 × 2 = 8` passes -> ([Consensus.h:1938](../src/xrpld/consensus/Consensus.h#L1938)). +> ([Consensus.h:1937](../include/xrpl/consensus/Consensus.h#L1937)). > 2. **Retry across consensus rounds** — a round can end `MovedOn` / `Expired`, > meaning the network settled a _different_ ledger. The node still builds a > ledger, but the **next** round's `checkLedger` detects the wrong prior, @@ -758,44 +821,44 @@ Consensus loops and branches (evidence): - **`consensus.establish` is the parent of `update_positions` and `check`**: `phaseEstablish` creates the establish span (`startEstablishTracing`), and both child spans parent to its captured context - ([Consensus.h:2100](../src/xrpld/consensus/Consensus.h#L2100), - [1629](../src/xrpld/consensus/Consensus.h#L1629), - [1838](../src/xrpld/consensus/Consensus.h#L1838)). + ([Consensus.h:2099](../include/xrpl/consensus/Consensus.h#L2099), + [1628](../include/xrpl/consensus/Consensus.h#L1628), + [1837](../include/xrpl/consensus/Consensus.h#L1837)). - **Avalanche-convergence loop (rounds within one ledger)**: repeated `heartbeat → timerEntry → phaseEstablish` bumps `establishCounter_` and raises the inclusion threshold each pass; `checkConsensus` = `No` stays in `Establish` ([NetworkOPs.cpp:1214](../src/xrpld/app/misc/NetworkOPs.cpp#L1214); - [Consensus.h:1468](../src/xrpld/consensus/Consensus.h#L1468); - thresholds [ConsensusParms.h:145](../src/xrpld/consensus/ConsensusParms.h#L145)). + [Consensus.h:1467](../include/xrpl/consensus/Consensus.h#L1467); + thresholds [ConsensusParms.h:145](../include/xrpl/consensus/ConsensusParms.h#L145)). - **Retry-across-rounds loop (many rounds per settled ledger)**: `MovedOn` / `Expired` accepts a non-preferred ledger; the next round's `checkLedger` finds the wrong prior and recovers before re-deliberating - ([Consensus.h:1194](../src/xrpld/consensus/Consensus.h#L1194)); round-to-round + ([Consensus.h:1193](../include/xrpl/consensus/Consensus.h#L1193)); round-to-round via `endConsensus → beginConsensus` ([NetworkOPs.cpp:2315](../src/xrpld/app/misc/NetworkOPs.cpp#L2315)). - **Two extra establish loop-backs before accept**: `shouldPause` (laggard backpressure) and `!haveCloseTimeConsensus_` (TX consensus but not close-time) each `return` and re-loop, distinct from `checkConsensus == No` - ([Consensus.h:1497](../src/xrpld/consensus/Consensus.h#L1497), - [1500](../src/xrpld/consensus/Consensus.h#L1500)); close time can + ([Consensus.h:1496](../include/xrpl/consensus/Consensus.h#L1496), + [1499](../include/xrpl/consensus/Consensus.h#L1499)); close time can "agree to disagree" at prior close + 1s ([docs/consensus.md:163](consensus.md)). - **acquireTxSet / gotTxSet loop**: a disagreeing peer position triggers an async `acquireTxSet`; the later `gotTxSet` regenerates disputes and can extend the - establish phase ([Consensus.h:932](../src/xrpld/consensus/Consensus.h#L932)). + establish phase ([Consensus.h:931](../include/xrpl/consensus/Consensus.h#L931)). - **Bow-out / mode change**: `handleWrongLedger → leaveConsensus` sends a bow-out proposal and demotes Proposing → Observing for the rest of the round - ([Consensus.h:1977](../src/xrpld/consensus/Consensus.h#L1977)); `startRound` + ([Consensus.h:1976](../include/xrpl/consensus/Consensus.h#L1976)); `startRound` begins in Proposing **or** Observing ([docs/consensus.md:176](consensus.md)). - **Buffered Open-phase inputs**: `peerProposal` / `gotTxSet` arriving during Open are stored, then seeded as disputes at `closeLedger` (`createDisputes`); `playbackProposals` replays them at `startRound` / `handleWrongLedger` ([docs/consensus.md:244](consensus.md); - [Consensus.h:817](../src/xrpld/consensus/Consensus.h#L817)). + [Consensus.h:816](../include/xrpl/consensus/Consensus.h#L816)). - **Outcome fork** after `checkConsensus`: `No` (loop) / `Yes` (onAccept) / - `MovedOn` / `Expired` ([Consensus.h:1516](../src/xrpld/consensus/Consensus.h#L1516)). + `MovedOn` / `Expired` ([Consensus.h:1515](../include/xrpl/consensus/Consensus.h#L1515)). - **Expired guard**: a round cannot leave on `Expired` before `avalancheCutoffs.size() × avMinRounds` (= 8) passes — below that, `Expired` - loops like `No` ([Consensus.h:1938](../src/xrpld/consensus/Consensus.h#L1938)). + loops like `No` ([Consensus.h:1937](../include/xrpl/consensus/Consensus.h#L1937)). - The **deterministic-vs-random trace-strategy** branch at round start ([RCLConsensus.cpp:1291](../src/xrpld/app/consensus/RCLConsensus.cpp#L1291)) sets only the trace ID — it has **zero protocol effect**. @@ -918,10 +981,11 @@ flowchart TB UALL -.->|dead / aborted| DEAD ``` -**Ledger acquire** — a **separate trace root** (not part of the close flow) that -fetches a missing or correct-prior ledger from peers, retries per peer/timer, and -finishes with a reason-dependent store; `checkAccept` + `tryAdvance` run on **any** -completed acquire: +**Ledger acquire** — a flow **outside the close flow** that fetches a missing or +correct-prior ledger from peers, retries per peer/timer, and finishes with a +reason-dependent store; `checkAccept` + `tryAdvance` run on **any** completed +acquire. `ledger.acquire` is usually a trace root, but not reliably so — see the +[parenting known issues](#where-telemetry-parenting-differs-from-protocol-flow): ```mermaid flowchart TB @@ -963,13 +1027,13 @@ Side-flow evidence: [181](../src/xrpld/rpc/detail/PathRequestManager.cpp#L181)). - **Acquire outcome fork**: `timeouts_ > kLedgerTimeoutRetriesMax` (= 6) sets `failed_` → terminal `logFailure`, no store/checkAccept - ([InboundLedger.cpp:387](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L387)). + ([InboundLedger.cpp:402](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L402)). - **done() reason branch (store side only)**: `HISTORY` → `onLedgerFetched`, **no** `storeLedger`; else → `storeLedger`. But `checkAccept` + `tryAdvance` run for **any** `complete_ && !failed_` acquire regardless of reason - ([InboundLedger.cpp:495](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L495) - store switch; [507](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L507) - reason-independent checkAccept/tryAdvance). + ([InboundLedger.cpp:537](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L537) + store switch; [552](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L552) + reason-independent checkAccept/tryAdvance on the `AcqDone` job). - **tryAdvance multi-ledger loop**: `doAdvance` runs `do { … } while (advanceWork_)`, publishing a range of ledgers and recursively triggering further HISTORY acquire ([LedgerMaster.cpp:1905](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1905)). @@ -980,22 +1044,88 @@ The graph above is protocol control flow. The OpenTelemetry span **parent links* are built differently and, in several places, do **not** represent a real call edge. Read a trace with these in mind: -| Telemetry does this | Real protocol flow | -| ------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `tx.process` is a `hashSpan` root from `txID` — an independent trace root ([TxTracing.h:63](../src/xrpld/telemetry/TxTracing.h#L63)). | The real edge is the synchronous `doSubmit → processTransaction` call; it is **not** a child of `rpc.command.submit`. | -| `tx.preflight` / `tx.preclaim` / `tx.transactor` share one `txID`-derived trace ID. | That shared ID is a correlation trick, not a call edge. The real order is the composed `apply()` at [apply.cpp:118](../src/libxrpl/tx/apply.cpp#L118). They are **not** children of `tx.process` or `tx.apply`. | -| `consensus.round` uses a deterministic trace ID from the previous ledger hash. | This makes **all validators share one trace ID** (a cross-node shared root), not a per-node parent. The real round-to-round edge is `endConsensus → beginConsensus`. | -| `consensus.accept` (main thread) and `consensus.accept.apply` (JtAccept worker) are wired via a captured context. | The real edge is the queued `JtAccept` job, a thread hand-off ([RCLConsensus.cpp:483](../src/xrpld/app/consensus/RCLConsensus.cpp#L483)). | -| `pathfind.update_all` parents nothing from the original `pathfind.request`. | The causal link is the ledger-close job on `JtUpdatePf`, not span nesting. | -| `ledger.acquire` and its downstream `ledger.store` / `ledger.validate`. | Reached via the `AcqDone` job, not parent inheritance; `ledger.acquire` is its own root. | -| `peer.*.receive` (fresh `kConsumer` root) and `consensus.*.receive` on the same message. | Two **sequential stages of one synchronous handler**, not parent/child; on a duplicate/untrusted drop the `consensus.*.receive` is never created. | -| Receive spans adopt the sender's `trace_id` + `span_id` as a genuine cross-node parent. | Deliberate: the receive span becomes a child of a **different node's** span (a cross-node context marker, not an in-process edge). `tx.receive` is asymmetric — it borrows only the sender's `span_id` and re-derives its own `trace_id` from `txID`. | +| Telemetry does this | Real protocol flow | +| ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `tx.process` is a `hashSpan` root from `txID` — an independent trace root ([TxTracing.h:63](../src/xrpld/telemetry/TxTracing.h#L63)). | The real edge is the synchronous `doSubmit → processTransaction` call; it is **not** a child of `rpc.command.submit`. | +| `tx.preflight` / `tx.preclaim` / `tx.transactor` share one `txID`-derived trace ID. | That shared ID is a correlation trick, not a call edge. The real order is the composed `apply()` at [apply.cpp:118](../src/libxrpl/tx/apply.cpp#L118). They are **not** children of `tx.process` or `tx.apply`. Because nothing else nests under it either, `tx.apply` is **always a leaf** — the stage spans for the transactions it applied sit in the txID-keyed trace, not beneath it. | +| `consensus.round` uses a deterministic trace ID from the previous ledger hash. | This makes **all validators share one trace ID** (a cross-node shared root), not a per-node parent. The real round-to-round edge is `endConsensus → beginConsensus`. | +| `consensus.accept` (main thread) and `consensus.accept.apply` (JtAccept worker) are wired via a captured context. | The real edge is the queued `JtAccept` job, a thread hand-off ([RCLConsensus.cpp:483](../src/xrpld/app/consensus/RCLConsensus.cpp#L483)). | +| `pathfind.update_all` parents nothing from the original `pathfind.request`. | The causal link is the ledger-close job on `JtUpdatePf`, not span nesting. | +| `ledger.acquire` and its downstream `ledger.store` / `ledger.validate`. | Reached via the `AcqDone` job, not parent inheritance. All three are non-scoped `SpanGuard::span` spans, so none of them parents the others; each takes whatever ambient span its own caller happens to have active. See the `ledger.*` known issue below. | +| `peer.*.receive` (fresh `kConsumer` root) and `consensus.*.receive` on the same message. | Two **sequential stages of one synchronous handler**, not parent/child; on a duplicate/untrusted drop the `consensus.*.receive` is never created. | +| Receive spans adopt the sender's `trace_id` + `span_id` as a genuine cross-node parent. | Deliberate: the receive span becomes a child of a **different node's** span (a cross-node context marker, not an in-process edge). `tx.receive` is asymmetric — it borrows only the sender's `span_id` and re-derives its own `trace_id` from `txID`. | > **Known telemetry artifacts** (from live audits, memory `otel-span-hierarchy-audit`): > an RPC entry span's scope can leak across a reused coroutine worker, and the -> `hashSpan` roots (`tx.*`) — along with plain roots like `ledger.acquire` — can -> surface in Tempo as dangling "root span not yet received". These are -> exporter/parenting artifacts, not real control-flow parents. +> `hashSpan` roots (`tx.*`) — along with `ledger.acquire` / `ledger.store` / +> `ledger.validate` whenever they do come out parentless — can surface in Tempo +> as dangling "root span not yet received". These are exporter/parenting +> artifacts, not real control-flow parents. + +Three further divergences are **known issues in the code**, not deliberate design. +Unlike the rows above, these produce a parent that is simply wrong, and all three +are pending a code fix: + +- **`grpc.*` and `pathfind.update_all` do not open a fresh root.** All four RPC + entry points create their span with `freshRoot`, so a reused coroutine worker + cannot leak a stale ambient parent into them + ([ServerHandler.cpp:473](../src/xrpld/rpc/detail/ServerHandler.cpp#L473), + [640](../src/xrpld/rpc/detail/ServerHandler.cpp#L640)). `grpc.` + ([GRPCServer.cpp:173](../src/xrpld/app/main/GRPCServer.cpp#L173)) and + `pathfind.update_all` + ([PathRequestManager.cpp:91](../src/xrpld/rpc/detail/PathRequestManager.cpp#L91)) + use the plain constructor instead, so either can be adopted by whatever span + happened to be active on the worker that picked the job up. A gRPC call + appearing beneath an unrelated transaction's trace is this bug, not a real + call edge. +- **`ledger.acquire` / `ledger.store` / `ledger.validate` are not reliably roots + either.** All three use `SpanGuard::span` + ([InboundLedger.cpp:113](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L113), + [LedgerMaster.cpp:463](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L463), + [987](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L987)), which inherits the + ambient span ([SpanGuard.cpp:233](../src/libxrpl/telemetry/SpanGuard.cpp#L233)) + rather than `freshRoot` + ([245](../src/libxrpl/telemetry/SpanGuard.cpp#L245)) — the same defect as + `grpc.*` above. Whether they come out as roots depends purely on the caller: + - **Root, as documented.** On the `JtAdvance` / `AcqDone` job path + (`LedgerMaster::doAdvance`, `RCLConsensus::Adaptor::acquireLedger` → + [RCLConsensus.cpp:171](../src/xrpld/app/consensus/RCLConsensus.cpp#L171)) no + span is active on the worker, so nothing is inherited. `acquireSpan_` itself is + a non-scoped `SpanGuard`, so it never becomes the ambient parent of the + `ledger.store` / `ledger.validate` that follow it. + - **Mis-parented.** `InboundLedgers::acquire` is also called **synchronously from + an RPC handler** — `ledger_request` → `rpc::getOrAcquireLedger` + ([RPCLedgerHelpers.cpp:483](../src/xrpld/rpc/detail/RPCLedgerHelpers.cpp#L483)) + — which runs inside the scoped `rpc.command.` span + ([RPCHandler.cpp:168](../src/xrpld/rpc/detail/RPCHandler.cpp#L168)). There + `ledger.acquire` becomes a child of that RPC command, and when `init()` is + satisfied from the local store the `ledger.store` / `ledger.validate` it calls + ([InboundLedger.cpp:164](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L164), + [168](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L168)) land there as + siblings. A ledger acquisition nested under an `rpc.command.*` trace is this + bug, not a real call edge. + + **`ledger.build` and `tx.apply` use the same ambient-parent construct but are + safe.** `ledger.build` is a plain `ScopedSpanGuard` + ([BuildLedger.cpp:55](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L55)): its + only callers are `RCLConsensus::doAccept` + ([RCLConsensus.cpp:935-937](../src/xrpld/app/consensus/RCLConsensus.cpp#L935)) + on the `JtAccept` worker and the replay path + ([LedgerDeltaAcquire.cpp:208](../src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp#L208)), + and every consensus accept span is a non-scoped `SpanGuard` + ([RCLConsensus.cpp:598-599](../src/xrpld/app/consensus/RCLConsensus.cpp#L598)), + so no ambient span exists to be inherited there. `tx.apply` + ([BuildLedger.cpp:123](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L123)) is + reached only synchronously from `buildLedgerImpl` while `ledger.build`'s scope is + live, so its ambient parent is always `ledger.build` — which is exactly the + intended edge. + +- **`consensus.round` is not always a root.** The `consensus_trace_strategy=attribute` + path has two creation branches; the fallback branch — taken on the first traced + round of a run, and whenever consensus tracing is off — sets no parent at all + ([RCLConsensus.cpp:1310](../src/xrpld/app/consensus/RCLConsensus.cpp#L1310)), + so that round inherits the ambient context instead of starting a trace. Rounds + under the default `deterministic` strategy are unaffected. --- @@ -1120,10 +1250,20 @@ sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.tr > a rising `tx.transactor` failure rate points to apply-time problems. Alert per > stage rather than on a single aggregate so the failing stage is obvious. -> **Sampling caveat**: these stage metrics are span-derived and inherit the -> **tracer head-sampling** ratio (`sampling_ratio`). At `sampling_ratio < 1.0` -> they undercount proportionally — treat them as relative trends, not absolute -> transaction counts. Native StatsD metrics are unsampled. +> **Sampling caveat**: these stage metrics are span-derived, but head sampling +> is **fixed at 100% and is not configurable** — the ratio is a compile-time +> constant ([Telemetry.h:234](../include/xrpl/telemetry/Telemetry.h#L234) +> `static constexpr double samplingRatio = 1.0;`) and there is no +> `sampling_ratio` config key to set +> ([TelemetryConfig.cpp:139](../src/libxrpl/telemetry/TelemetryConfig.cpp#L139) +> — "nothing to parse"). So locally these counts are **exact**, not a sample. +> Volume reduction is a collector-side **tail** sampling decision instead, and +> the only policy shipped is a single 0.5% probabilistic one that lives **only** +> in `otel-collector-config.grafanacloud.yaml` — the base +> `otel-collector-config.yaml` has no tail sampling at all, so a stock local +> stack retains every trace. Where that Cloud policy is in force it applies to +> the trace-storage branch only; spanmetrics run on a separate branch and still +> see 100% of spans, so the derived RED metrics stay exact either way. ### Transaction Queue Health @@ -1448,7 +1588,7 @@ Note that `job_count` is exported as `jobq_job_count`: the JobQueue is constructed with `collectorManager_->group("jobq")` (Application.cpp:386), `GroupImp::makeName()` joins prefix and name with a `.` (Groups.cpp:42), and `OTelCollectorImp::formatName()` then turns the `.` into `_` and lowercases the -whole string (OTelCollector.cpp:860). The same mechanism produces the +whole string (OTelCollector.cpp:855-874). The same mechanism produces the `jobq_{jobtype}_*` names above and the pre-existing `jobq_{jobtype}_milliseconds` timing family. @@ -1595,9 +1735,9 @@ ledger acquisition deferring". Use `acquire_ledger_deferrals` and These five come from the `PerfLog` job hooks, not from beast::insight, so they are exported by the `MetricsRegistry` meter. `job_queued_us` and `job_running_us` have explicit microsecond bucket views registered -(`addMicrosecondHistogramView()`, MetricsRegistry.cpp:253-254) spanning 100 µs to -60 s; without those the SDK default buckets stop at 10 ms and every quantile -saturates. +(`addMicrosecondHistogramView()` calls at MetricsRegistry.cpp:310-311; the helper +itself is at `:197`) spanning 100 µs to 60 s; without those the SDK default +buckets stop at 10 ms and every quantile saturates. | Prometheus Metric | Kind | Labels | Description | | -------------------- | --------- | --------------------- | ------------------------------------ | @@ -1854,11 +1994,16 @@ Fifteen dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/` Fourteen are Prometheus-backed; `log-derived-insights` is the only Loki/LogQL board and is documented last, together with the LogQL-specific traps it exposed. -> Nine dashboards have a reference section below. `fee-market`, `job-queue`, -> `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality`, and -> `validator-health` are provisioned but not yet documented here — their panel -> descriptions carry the same six-heading reference format, so open the panel -> info icon in Grafana until a section is written. +> **Nine of the fifteen have a reference section.** Eight are in this chapter +> (`rpc-performance`, `transaction-overview`, `consensus-health`, +> `ledger-operations`, `peer-network`, `node-health`, `network-traffic`, +> `rpc-pathfinding`); the ninth, `log-derived-insights`, is documented under +> [Log-Trace Correlation](#log-derived-insights-log-derived-insights). The +> remaining **six** — `fee-market`, `job-queue`, `ledger-data-sync`, +> `overlay-traffic-detail`, `peer-quality`, and `validator-health` — are +> provisioned but not yet documented here. Their panel descriptions carry the same +> six-heading reference format, so open the panel info icon in Grafana until a +> section is written. ### RPC Performance (`rpc-performance`) @@ -1868,7 +2013,7 @@ board and is documented last, together with the LogQL-specific traps it exposed. | RPC Latency p95 by Command | timeseries | `histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])))` | `command` | | RPC Error Rate | bargauge | Error spans / total spans × 100, grouped by `command` | `command`, `status_code` | | RPC Latency Heatmap | heatmap | `sum(increase(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])) by (le)` | `le` (bucket boundaries) | -| Overall RPC Throughput | timeseries | `rpc.request` + `rpc.process` rate | — | +| Overall RPC Throughput | timeseries | `rpc.http_request` + `rpc.process` rate | — | | RPC Success vs Error | timeseries | by `status_code` (UNSET vs ERROR) | `status_code` | | Top Commands by Volume | bargauge | `topk(10, ...)` by `command` | `command` | | WebSocket Message Rate | stat | `rpc.ws_message` rate | — | @@ -1935,18 +2080,18 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | | Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | | Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_milliseconds_bucket)` | — | | Job Queue Depth | timeseries | `jobq_job_count` | — | | Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[5m])` | — | +| Key Jobs Execution Time | timeseries | `histogram_quantile($quantile, sum by (le) (rate(job_running_us_bucket{job_type="acceptLedger"}[$__rate_interval])))` (+ 10 more key jobs) | `job_type` | +| Key Jobs Dequeue Wait Time | timeseries | `histogram_quantile($quantile, sum by (le) (rate(job_queued_us_bucket{job_type="acceptLedger"}[$__rate_interval])))` (+ 10 more) | `job_type` | | FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | | FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | | Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | | State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, sum by (le, job_type) (rate(job_running_us_bucket[$__rate_interval])))` | `job_type` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, sum by (le, job_type) (rate(job_queued_us_bucket[$__rate_interval])))` | `job_type` | | Server State | stat | `server_info{metric="server_state"}` | `metric` | | Uptime | stat | `server_info{metric="uptime"}` | `metric` | | Peer Count | stat | `server_info{metric="peers"}` | `metric` | @@ -1957,6 +2102,24 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | | Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | +> **The four job panels read the `MetricsRegistry` histograms fed by the PerfLog +> job hooks, not the `jobq_*` ones.** +> `$quantile` is a dashboard template variable holding a fraction (`0.95`), fed +> straight into `histogram_quantile()`. There is **no `quantile` label** on any +> xrpld series — that was a StatsD-era summary convention, and a selector like +> `{quantile="$quantile"}` matches nothing and reports no error. The job queue +> exposes two parallel families: `job_running_us` / `job_queued_us` +> (`MetricsRegistry` instruments, labelled by `job_type` and `handler`, +> microseconds — what these panels use; +> [MetricsRegistry.cpp:94-95](../src/xrpld/telemetry/MetricsRegistry.cpp#L94), +> [363-366](../src/xrpld/telemetry/MetricsRegistry.cpp#L363), recorded from the +> `PerfLog` job hooks at +> [PerfLogImp.cpp:432](../src/xrpld/perflog/detail/PerfLogImp.cpp#L432)) and +> `jobq_[_q]_milliseconds` +> (beast::insight, one instrument per job type, milliseconds — +> [JobTypeData.h:97](../include/xrpl/core/JobTypeData.h#L97)). Both are live; +> prefer the labelled `job_*_us` pair so one query covers every job type. + ### Network Traffic -- System Metrics (`network-traffic`) | Panel | Type | PromQL | Labels Used | @@ -1984,16 +2147,26 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### RPC & Pathfinding -- System Metrics (`rpc-pathfinding`) -| Panel | Type | PromQL | Labels Used | -| ------------------------- | ---------- | ------------------------------------------------ | ----------- | -| RPC Request Rate | stat | `rate(rpc_requests[5m])` | — | -| RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_bucket)` | — | -| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_bucket)` | — | -| RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | -| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | -| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | -| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | -| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | +| Panel | Type | PromQL | Labels Used | +| ------------------------- | ---------- | ------------------------------------------------------------- | ----------- | +| RPC Request Rate | stat | `rate(rpc_requests[5m])` | — | +| RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_milliseconds_bucket)` | — | +| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_milliseconds_bucket)` | — | +| RPC Response Time Heatmap | heatmap | `rpc_time_milliseconds_bucket` | — | +| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_milliseconds_bucket)` | — | +| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_milliseconds_bucket)` | — | +| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | +| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | + +> **The `_milliseconds` suffix comes from the exporter, not from xrpld.** These +> histograms are created with unit `"ms"` +> ([OTelCollector.cpp:615](../src/libxrpl/beast/insight/OTelCollector.cpp#L615)), +> so the Prometheus exporter appends the unit to the family name — `rpc_time` +> becomes `rpc_time_milliseconds_bucket`. Querying the bare `rpc_time_bucket`, +> `ios_latency_bucket` or `pathfind_fast_bucket` returns no data and no error. +> **Known issue**: `rpc_size` counts bytes but shares the same `"ms"` histogram +> constructor, so it is exported as `rpc_size_milliseconds_bucket` — the suffix +> is wrong, the name is nonetheless the one to query. ### Span → Metric → Dashboard Summary @@ -2029,6 +2202,7 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | `ledger.build` | `{span_name="ledger.build"}` | Ledger Ops (Build Rate, Duration, Heatmap) | | `ledger.validate` | `{span_name="ledger.validate"}` | Ledger Ops (Validation Rate) | | `ledger.store` | `{span_name="ledger.store"}` | Ledger Ops (Store Rate) | +| `ledger.acquire` | `{span_name="ledger.acquire"}` | -- (available but not paneled) | | `peer.proposal.receive` | `{span_name="peer.proposal.receive"}` | Peer Network (Rate, Trusted/Untrusted) | | `peer.validation.receive` | `{span_name="peer.validation.receive"}` | Peer Network (Rate, Trusted/Untrusted) | @@ -2069,7 +2243,7 @@ Alerts fire only after the condition holds for the `for` dwell time. | `NodeStateFlapping` | warning | > 3 re-entries into FULL per hour | 15m | | `NodeNotFull` | warning | `server_state` < 4 (FULL) | 15m | | `ManifestJobQueueConvoy` | warning | `jobq_manifest_waiting` > 3 | 10m | -| `ManifestFloodInbound` | warning | `rate(overhead_manifest_bytes_in)` > 512 kB/s | 10m | +| `ManifestFloodInbound` | warning | `rate(overhead_manifest_bytes_in)` > 512 KiB/s | 10m | | `PeerResourceDisconnects` | warning | > 5 resource-driven peer disconnects per 30m | 5m | Two expression idioms recur and are load-bearing — do not "simplify" them away: @@ -2090,6 +2264,19 @@ from the validated network chain. Likely causes: corrupted local state, a bug, or a node that fell out of sync and rebuilt incorrectly. Investigate the node's ledger acquisition logs; a healthy node never mismatches. +> **Query trap — `sum(ledger_history_mismatch_total)` double-counts.** One +> mismatch increments **two** instruments inside the same `handleMismatch()` +> call: the legacy beast::insight counter, which carries no `reason` label +> ([LedgerHistory.cpp:323](../src/xrpld/app/ledger/LedgerHistory.cpp#L323)), and +> the `MetricsRegistry` counter, which does +> ([LedgerHistory.cpp:331](../src/xrpld/app/ledger/LedgerHistory.cpp#L331)). +> Both normalise to the same Prometheus family, so an unfiltered `sum()` or +> `increase()` reports exactly **twice** the real mismatch count. Aggregate over +> the labelled series only — `sum by (reason) (...)`, or +> `sum(ledger_history_mismatch_total{reason!=""})` — and halve any historical +> figure taken from the unfiltered form. The alert rule is unaffected: it only +> tests `> 0`. This is a known issue; the duplicate producer awaits a code fix. + **LedgerCloseStalled** — No ledgers closed for 3 minutes. A healthy node closes one every ~3-5s. Likely causes: lost peer connectivity, consensus stall, or the process is hung. This rule also fires on _NoData_ — if the series disappears the @@ -2189,7 +2376,8 @@ This is the most reliable manifest-flood signal because `jobq_manifest_waiting` is `0` at the 99.9th percentile on every node over 24h — any sustained backlog is a genuine outlier rather than normal variance. -**ManifestFloodInbound** — Inbound manifest byte-rate exceeds 512 kB/s. Catches the +**ManifestFloodInbound** — Inbound manifest byte-rate exceeds 512 KiB/s (524288 +B/s — the rule's literal `params: [524288]`). Catches the wire-level cause (a peer shipping oversized dumps) even when the job pool absorbs it without a visible backlog. Measured over 7 days: healthy p95 0.2-0.5 kB/s and p99 1.0-1.8 kB/s, against peaks up to 2.7 MB/s during real storms — so the @@ -2491,8 +2679,8 @@ Filters: `$service_name`, `$deployment_environment`, `$node`, #### LogQL traps this dashboard exposed -Ten mistakes that fail **silently** — each cost a debugging cycle, so check them -before adding any LogQL panel. +Eleven mistakes that fail **silently** — each cost a debugging cycle, so check +them before adding any LogQL panel. 1. **`partition` is structured metadata, not a stream label.** `{service_name="xrpld", partition="ManifestCache"}` returns **zero rows with @@ -3124,10 +3312,20 @@ not a sign the cache is working. ## Disabling Telemetry -Set `enabled=0` in config (runtime disable) or build without the flag: +Set `enabled=0` in the `[telemetry]` config section (runtime disable, no rebuild), or +compile telemetry out: ```bash +conan install . --build=missing -o telemetry=False cmake --preset default -Dtelemetry=OFF ``` +Pass the flag explicitly rather than omitting it — an omitted flag resolves to whatever +the build's current default is. That default is `ON` on the telemetry branches so CI +compiles the instrumented paths, and `OFF` once the feature is merged; `-Dtelemetry=OFF` +is correct either way. `-DXRPL_ENABLE_TELEMETRY=OFF` does **not** work: that name is only +a compile definition added when `telemetry` is ON, not a CMake option, so telemetry stays +compiled in and CMake only lists it under `Manually-specified variables were not used by +the project`. + When telemetry is compiled out, all trace macros expand to no-ops with zero overhead. diff --git a/include/xrpl/beast/insight/OTelCollector.h b/include/xrpl/beast/insight/OTelCollector.h index e043531f99..46d103dc90 100644 --- a/include/xrpl/beast/insight/OTelCollector.h +++ b/include/xrpl/beast/insight/OTelCollector.h @@ -58,7 +58,7 @@ namespace beast::insight { * @code * auto collector = beast::insight::OTelCollector::New( * "http://localhost:4318/v1/metrics", // OTLP/HTTP endpoint - * "xrpld", // metric name prefix + * "xrpld", // logging label only * "node-1", // service.instance.id * "xrpld", // service.name * "mainnet", // xrpl.network.type @@ -105,8 +105,12 @@ public: * * @param endpoint OTLP/HTTP metrics endpoint URL * (e.g. "http://localhost:4318/v1/metrics"). - * @param prefix Prefix prepended to all metric names - * (e.g. "xrpld"). + * @param prefix Label for the collector's startup log line + * (e.g. "xrpld"). Exported metric names are produced + * by formatName(), which lowercases the raw name and + * maps dots and spaces to underscores. The service is + * identified by the `service.name` OTel resource + * attribute. * @param instanceId Unique identifier for this node instance, * emitted as the `service.instance.id` OTel * resource attribute. Defaults to empty string diff --git a/include/xrpl/proto/xrpl.proto b/include/xrpl/proto/xrpl.proto index 468034ad3a..f38c426278 100644 --- a/include/xrpl/proto/xrpl.proto +++ b/include/xrpl/proto/xrpl.proto @@ -95,9 +95,16 @@ message TMPublicKey { // Older peers that do not understand field 1001 will simply ignore it // per protobuf wire-format rules, preserving backwards compatibility. // -// trace_state is reserved for future use (secure tracing pipeline, -// OpenTelemetryPlan/secure-OTel.md). It is currently neither populated -// on inject nor read on extract; consumers must not rely on it. +// trace_state (field 4) is reserved and inert: it is neither populated on +// inject nor read on extract, so consumers must not rely on it. Beyond the +// W3C tracestate use noted on the field below, it is the intended home for +// an authenticated token a receiver could verify before adopting a peer's +// trace context as its parent. Today this message is unauthenticated peer +// input: the receiver only checks that the ids are well formed (16-byte +// trace_id, 8-byte span_id, neither all-zero) and otherwise starts a fresh +// trace, so the ids are a hint, not trusted provenance. An authenticated +// scheme would need a shared verification key, a canonical form to sign, +// and a defined policy for peers that send no token. message TraceContext { optional bytes trace_id = 1; // 16-byte trace identifier optional bytes span_id = 2; // 8-byte parent span identifier diff --git a/include/xrpl/telemetry/TraceContextPropagator.h b/include/xrpl/telemetry/TraceContextPropagator.h index e8d851e53e..d2282665a3 100644 --- a/include/xrpl/telemetry/TraceContextPropagator.h +++ b/include/xrpl/telemetry/TraceContextPropagator.h @@ -101,13 +101,23 @@ injectToProtobuf(opentelemetry::context::Context const& ctx, protocol::TraceCont // Serialize flags proto.set_trace_flags(spanCtx.trace_flags().flags()); - // TODO(observability/secure-OTel): the protobuf TraceContext message - // also carries `trace_state` (field 4), which is currently neither - // populated here nor read by extractFromProtobuf above. The field is - // reserved for the secure tracing pipeline outlined in - // OpenTelemetryPlan/secure-OTel.md, where an authenticated token in - // tracestate will let receivers reject spoofed/poisoned trace context. - // Wire trace_state through inject/extract once the consumer lands. + /** + * TODO: wire `trace_state` (protobuf TraceContext field 4) through + * inject and extract. It is neither written here nor read by + * extractFromProtobuf above, so the field is inert on the wire. + * + * Two uses are intended. One is W3C tracestate vendor-specific + * key-value pairs, for cross-vendor propagation. The other is an + * authenticated token. Today a peer's trace context is + * unauthenticated input: extractFromProtobuf only checks that the ids + * are well formed (16-byte trace_id, 8-byte span_id, neither + * all-zero) before using them as a parent, so the ids are a hint + * rather than trusted provenance. A token the receiver could verify + * would let it decide whether to adopt a peer's context at all. That + * needs a shared verification key, a canonical form to sign, and a + * defined policy for peers that send no token. None of that exists + * yet, which is why the field stays unpopulated. + */ } } // namespace xrpl::telemetry diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp index ab17159272..e28dc60ad2 100644 --- a/src/libxrpl/beast/insight/OTelCollector.cpp +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -5,7 +5,7 @@ * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake * telemetry=ON). Maps beast::insight instruments to OTel SDK instruments * created on the GLOBAL Meter published by the telemetry module. This class - * is a legacy shim: it no longer owns an export pipeline. The MeterProvider, + * is an adapter only: it owns no export pipeline. The MeterProvider, * PeriodicExportingMetricReader, OTLP exporter and histogram view all live in * xrpl::telemetry::Telemetry. * @@ -134,8 +134,8 @@ class OTelCounterImpl : public CounterImpl public: /** * @param name Export-ready metric name, already run through - * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "rpc_size"). + * formatName() by the collector: lowercase, with `.` and + * ` ` mapped to `_` (e.g. "rpc_size"). * @param meter OTel Meter used to create the counter instrument. */ OTelCounterImpl( @@ -178,8 +178,8 @@ class OTelEventImpl : public EventImpl public: /** * @param name Export-ready metric name, already run through - * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "rpc_size"). + * formatName() by the collector: lowercase, with `.` and + * ` ` mapped to `_` (e.g. "rpc_size"). * @param meter OTel Meter used to create the histogram instrument. */ OTelEventImpl( @@ -227,8 +227,8 @@ class OTelGaugeImpl : public GaugeImpl public: /** * @param name Export-ready metric name, already run through - * formatName() by the collector: prefix prepended - * and dots replaced with underscores. + * formatName() by the collector: lowercase, with `.` + * and ` ` mapped to `_`. * @param meter OTel Meter used to create the observable gauge. * @param collector Owning collector, used to invoke hooks before reads. */ @@ -310,8 +310,8 @@ class OTelMeterImpl : public MeterImpl public: /** * @param name Export-ready metric name, already run through - * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "rpc_size"). + * formatName() by the collector: lowercase, with `.` and + * ` ` mapped to `_` (e.g. "rpc_size"). * @param meter OTel Meter used to create the counter instrument. */ OTelMeterImpl( @@ -340,7 +340,7 @@ private: //------------------------------------------------------------------------------ /** - * @brief Main OTel Collector implementation (legacy shim). + * @brief Main OTel Collector implementation (adapter over the global Meter). * * Obtains its Meter from the GLOBAL MeterProvider owned and published by the * telemetry module (xrpl::telemetry::Telemetry), rather than building its own @@ -380,8 +380,11 @@ private: * Caveats: * - Observable gauge callbacks run on the SDK's internal thread. Hook * handlers must be thread-safe. - * - Metric names are formed as "prefix_name" with dots replaced by - * underscores to match StatsD->Prometheus naming conventions. + * - Metric names carry NO prefix. formatName() only lowercases the raw + * name and turns dots and spaces into underscores, to match + * StatsD->Prometheus naming conventions. The service is identified by + * the OTel resource (service.name), so prefix_ is kept for logging + * only and never affects an exported name. * - The OTel Prometheus exporter appends "_total" to counters. The * metric names we register do NOT include this suffix — Prometheus * adds it automatically. @@ -402,11 +405,14 @@ public: /** * @brief Construct the OTel collector over the global MeterProvider. * - * @param endpoint OTLP/HTTP metrics endpoint URL. Informational only: - * the global telemetry pipeline is authoritative for - * the actual export endpoint. Retained for logging and - * back-compat with the New() signature. - * @param prefix Prefix for all metric names. + * @param endpoint OTLP/HTTP metrics endpoint URL, recorded in the + * collector's startup log line. Export uses the + * endpoint configured on the global telemetry + * pipeline. + * @param prefix Label for the collector's startup log line + * (e.g. "xrpld"). Exported metric names come from + * formatName(); the service is identified by the + * service.name resource attribute. * @param instanceId Value for the service.instance.id resource attribute. * When empty, the attribute is omitted. * @param serviceName Value for the service.name resource attribute. @@ -498,10 +504,12 @@ public: /** @} */ /** - * @brief Format a metric name with the configured prefix. + * @brief Format a raw metric name for export. * - * Replaces dots with underscores to match StatsD->Prometheus naming. - * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age" + * Lowercases the name and replaces dots and spaces with underscores to + * match StatsD->Prometheus naming. Adds NO prefix: the service is + * identified by the OTel resource (service.name). + * Example: name="LedgerMaster.Validated_Ledger_Age" * -> "ledgermaster_validated_ledger_age" * * @param name Raw metric name from beast::insight callers. @@ -517,7 +525,8 @@ private: Journal journal_; /** - * Prefix for all metric names (e.g., "xrpld"). + * Configured metric-name prefix (e.g., "xrpld"). Log-only: it is + * echoed in the startup log line and never applied to a metric name. */ std::string prefix_; @@ -708,17 +717,17 @@ OTelCollectorImp::OTelCollectorImp( Journal journal) : journal_(journal), prefix_(std::move(prefix)) { - // instanceId/serviceName/networkType are retained on the New() signature - // for back-compat but no longer used here: the telemetry module owns the - // resource attributes for the shared metrics pipeline. + // instanceId/serviceName/networkType are accepted but unused here: the + // telemetry module owns the resource attributes for the shared metrics + // pipeline, so setting them from this collector would have no effect. (void)instanceId; (void)serviceName; (void)networkType; if (journal_.info()) { - // endpoint is informational: the global telemetry pipeline owns the - // real exporter. It is logged here for back-compat and diagnostics. + // endpoint is logged for diagnostics only: the global telemetry + // pipeline owns the exporter that actually sends the metrics. journal_.info() << "OTelCollector starting: endpoint=" << endpoint << " prefix=" << prefix_; } @@ -846,9 +855,9 @@ OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge) std::string OTelCollectorImp::formatName(std::string const& name) { - // Produce a clean, lowercase, Prometheus-compatible metric name. - // No prefix — the OTel resource (service.name) identifies the service. - // Dots and spaces become underscores; everything lowercased. + // Produce a lowercase, Prometheus-compatible metric name: dots and + // spaces become underscores. Service identity travels in the + // service.name resource attribute, not in the metric name. std::string result; result.reserve(name.size()); for (char const c : name) diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index ff72f74b41..017e2c7bae 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -786,7 +786,7 @@ private: */ opentelemetry::nostd::unique_ptr> rpcErroredCounter_; /** - * Histogram: rpc_method_duration_us{method=""} + * Histogram: rpc_method_us{method=""} */ opentelemetry::nostd::unique_ptr> rpcDurationHistogram_; @@ -807,12 +807,12 @@ private: */ opentelemetry::nostd::unique_ptr> jobFinishedCounter_; /** - * Histogram: job_queued_duration_us{job_type="",handler=""} + * Histogram: job_queued_us{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobQueuedDurationHistogram_; /** - * Histogram: job_running_duration_us{job_type="",handler=""} + * Histogram: job_running_us{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobRunningDurationHistogram_;