diff --git a/.cspell.config.yaml b/.cspell.config.yaml index 466769add8..cd0fc8b346 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -264,6 +264,7 @@ words: - qalloc - qbsprofile - queuable + - ransitioned - Raphson - rcflags - reparent @@ -335,6 +336,7 @@ words: - summands - superpeer - superpeers + - synthesise - takergets - takerpays - ters @@ -365,6 +367,7 @@ words: - unfindable - unflatten - unfund + - unheld - unimpair - unroutable - unscalable @@ -382,6 +385,7 @@ words: - unvetoed - upvotes - USDB + - utilisation - variadics - venv - vfalco diff --git a/docker/telemetry/.env.alerting.example b/docker/telemetry/.env.alerting.example index 9822c635f1..594c55c064 100644 --- a/docker/telemetry/.env.alerting.example +++ b/docker/telemetry/.env.alerting.example @@ -1,24 +1,31 @@ # rippled OTel alerting delivery — copy to `.env.alerting` and fill in. # # `.env.alerting` is gitignored; never commit a real Slack webhook or address. -# Grafana reads these when the stack starts and expands the ${VARS} referenced -# in grafana/provisioning/alerting/contactpoints.yaml. See the Alerting -# section of docs/telemetry-runbook.md. +# See the Alerting section of docs/telemetry-runbook.md. # -# Any var left blank simply disables that delivery path — the stack still runs. +# IMPORTANT — these variables do NOT feed contactpoints.yaml. +# That file carries literal placeholder values which you edit in place, because +# Grafana expands ${VAR} but does NOT support ${VAR:-default}: an unset variable +# expands to empty, fails provisioning validation, and Grafana EXITS 1 — taking +# the whole telemetry stack down, not just alerting. So a var left blank here +# does not "disable a delivery path"; referencing a blank one breaks startup. +# +# What these are actually for: +# GF_SMTP_* consumed by the Grafana container (compose `env_file`) to +# turn on mail delivery. Without these, an email contact point +# provisions fine and then silently sends nothing. +# ALERT_EMAIL_TO read by upload_alerts_to_grafana.py to build the Grafana +# CLOUD email contact point over the REST API (Cloud has no +# provisioning filesystem). Not used by the local stack. -# --- Slack --- +# --- Slack (local stack: paste the webhook into contactpoints.yaml instead) --- # Incoming-webhook URL from the Slack app (Incoming Webhooks feature). -# Used by both the warning (xrpld-default) and critical (xrpld-critical) tiers. +# Kept here as a convenient place to record it, NOT as a ${VAR} source. SLACK_WEBHOOK_URL= -# Channel label. With an incoming webhook the target channel is fixed by the -# webhook itself; this only satisfies Grafana's Slack validator. Defaults to -# #xrpld-alerts if unset. -SLACK_CHANNEL=#xrpld-alerts - -# --- Email (critical tier only) --- -# Comma-separated recipient list for critical alerts. +# --- Email --- +# Recipient for Grafana Cloud alerts (comma- or semicolon-separated). +# Consumed by upload_alerts_to_grafana.py. ALERT_EMAIL_TO= # SMTP relay Grafana sends through. Email only delivers when SMTP is enabled diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 230a487e96..640b5b464c 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1545,11 +1545,11 @@ "h": 1, "w": 24, "x": 0, - "y": 115 + "y": 121 }, "collapsed": false, "panels": [], - "id": 27 + "id": 38 }, { "title": "Job Queue Backlog and Deferred by Type", @@ -1559,7 +1559,7 @@ "h": 8, "w": 12, "x": 0, - "y": 116 + "y": 122 }, "options": { "tooltip": { @@ -1608,7 +1608,7 @@ "h": 8, "w": 12, "x": 12, - "y": 116 + "y": 122 }, "options": { "tooltip": { @@ -1649,7 +1649,7 @@ "h": 1, "w": 24, "x": 0, - "y": 124 + "y": 130 }, "collapsed": false, "panels": [], @@ -1663,7 +1663,7 @@ "h": 8, "w": 12, "x": 0, - "y": 125 + "y": 131 }, "options": { "tooltip": { @@ -1735,7 +1735,7 @@ "h": 8, "w": 12, "x": 12, - "y": 125 + "y": 131 }, "options": { "tooltip": { @@ -1784,7 +1784,7 @@ "h": 8, "w": 12, "x": 0, - "y": 133 + "y": 139 }, "options": { "tooltip": { @@ -1844,7 +1844,7 @@ "h": 8, "w": 12, "x": 12, - "y": 133 + "y": 139 }, "options": { "tooltip": { @@ -1893,7 +1893,7 @@ "h": 8, "w": 12, "x": 0, - "y": 141 + "y": 147 }, "options": { "tooltip": { @@ -1949,7 +1949,7 @@ "h": 8, "w": 12, "x": 12, - "y": 141 + "y": 147 }, "options": { "tooltip": { @@ -1998,7 +1998,7 @@ "h": 8, "w": 12, "x": 0, - "y": 149 + "y": 155 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json new file mode 100644 index 0000000000..87028f1ec0 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -0,0 +1,2471 @@ +{ + "description": "What this shows: Signals derived from xrpld's debug.log via Loki (LogQL), covering detail that no metric or span records — manifest dispositions per master key, resource fee charges per IP and public key, ledger-fetch duplicate ratios, peer disconnect reasons, and consensus phase transitions. — Use it to: Investigate behaviour that the Prometheus dashboards cannot express, and to attribute load or abuse to a specific peer, key, or job. — REQUIRES DEBUG LOGS: most panels here read from log lines emitted at DBG severity, which xrpld suppresses by default (default threshold is Info, see Main.cpp). On a default-configured node those panels are EMPTY, and an empty panel here means 'not collecting', NOT 'no problem'. Enable per partition with `log_level debug` — for example `log_level ManifestCache debug`. Rows are marked [DBG] when they require debug logs and [DEFAULT OK] when they work at the default Info level.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "schemaVersion": 39, + "tags": ["node", "logs", "logql"], + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Log-Derived Insights", + "uid": "log-derived-insights", + "refresh": "30s", + "annotations": { + "list": [] + }, + "templating": { + "list": [ + { + "name": "DS_LOKI", + "label": "Loki Data Source", + "description": "Loki data source holding xrpld debug.log lines", + "type": "datasource", + "query": "loki", + "current": { + "text": "Loki", + "value": "loki" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "refresh": 1, + "options": [], + "skipUrlSync": false + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by emitting service", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Deployment Environment", + "description": "Filter by deployment tier (local, ci, test, prod)", + "type": "query", + "query": "label_values({service_name=\"xrpld\"}, deployment_environment)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values({service_name=\"xrpld\"}, service_instance_id)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network Type", + "description": "Filter by XRPL network. Structured metadata, so values are enumerated rather than discovered.", + "type": "custom", + "query": "mainnet,testnet,devnet", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "severity", + "label": "Log Severity", + "description": "Filter by xrpld log severity (DBG, NFO, WRN, ERR, FTL). Structured metadata, not a stream label.", + "type": "custom", + "query": "DBG,NFO,WRN,ERR,FTL", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "consensus_phase", + "label": "Consensus Phase", + "description": "Derived from log text: ConsensusPhase transitions (Open, Establish, Accepted)", + "type": "custom", + "query": "Open,Establish,Accepted", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "consensus_mode", + "label": "Consensus Mode", + "description": "Derived from log text: ConsensusLogger operating mode (full, syncing, observing, tracking)", + "type": "custom", + "query": "full,syncing,observing,tracking", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "manifest_action", + "label": "Manifest Action", + "description": "Derived from log text: ManifestCache disposition. UntrustedCapacity exists only on rc5+ builds.", + "type": "custom", + "query": "AcceptedNew,AcceptedUpdate,Stale,Revoked,Invalid,UntrustedCapacity", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "charge_reason", + "label": "Fee Charge Reason", + "description": "Derived from log text: Resource fee charge reason", + "type": "custom", + "query": "useless data,moderate peer request,heavy peer request,light peer request,trivial peer request,unwanted data,init drop", + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "options": [], + "refresh": 0, + "sort": 0 + }, + { + "name": "topn", + "label": "Top N", + "description": "Row limit for top-N tables. Loki caps a query at 2000 series, so unbounded per-key aggregation fails.", + "type": "custom", + "query": "5,10,15,20,25", + "includeAll": false, + "current": { + "text": "10", + "value": "10" + }, + "multi": false, + "options": [], + "refresh": 0, + "sort": 0 + } + ] + }, + "panels": [ + { + "type": "text", + "title": "Read This First — Debug Log Requirement", + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "markdown", + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### ⚠️ Most panels on this dashboard require debug logs\n\nxrpld's default log threshold is **Info** (`Severity thresh = Severity::Info` in `Main.cpp`). Every panel in a row marked **[DBG]** reads log lines emitted at `DBG` severity, which a default-configured node **does not write**.\n\n**An empty [DBG] panel means \"not collecting\", NOT \"no problem\".** A manifest dashboard reading zero because `ManifestCache` sits at Info is indistinguishable from a healthy one.\n\nEnable per partition rather than globally — global debug is a firehose (`Resource` alone emits ~329k lines / 6h):\n\n```\nlog_level ManifestCache debug\nlog_level Resource debug\nlog_level InboundLedger debug\nlog_level Peer debug\nlog_level LedgerConsensus debug\n```\n\nRows marked **[DEFAULT OK]** work at the default Info level and need no configuration change." + }, + "id": 1 + }, + { + "type": "row", + "title": "Worst Offenders — Node Ranking [MIXED]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "panels": [], + "id": 2 + }, + { + "type": "stat", + "title": "Nodes By Error Volume", + "description": "###### What this is:\n*Which nodes are logging the most errors.*\n\n###### How it's computed:\n*Count of ERR and FTL log lines per node over the dashboard window, ranked highest first.*\n\n###### Reading it:\n*The top entry is the node in the most trouble. Compare nodes rather than reading an absolute value.*\n\n###### Healthy range:\n*Zero, or a small flat count. ERR is not routine.*\n\n###### Watch for:\n*Any node pulling far ahead of its peers, which usually means a fault local to that node rather than a network condition.*\n\n###### Keywords:\n- **ERR / FTL** *(per line)* — the two most severe xrpld log levels; both survive the default Info threshold.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: errors", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 3 + }, + { + "type": "stat", + "title": "Nodes By Attack-Like Input", + "description": "###### What this is:\n*Which nodes are receiving the most malformed or unwanted peer payloads.*\n\n###### How it's computed:\n*Count of Resource fee charges whose reason is useless data, unwanted data, or init drop, per node.*\n\n###### Reading it:\n*These three reasons indicate a peer sent something the node could not use, so a high count is the closest log-derived proxy for abusive input.*\n\n###### Healthy range:\n*Low and flat. Some useless data is normal on a busy overlay.*\n\n###### Watch for:\n*A sharp rise on one node, especially paired with a single dominant IP in the charged-peers table below.*\n\n###### Keywords:\n- **Useless data** *(per charge)* — payload the node could not use, e.g. an empty or malformed message.\n- **Init drop** *(per charge)* — a connection dropped during initialisation.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: charges", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 4 + }, + { + "type": "stat", + "title": "Nodes By Total Fee Charged", + "description": "###### What this is:\n*Which nodes are absorbing the most peer-imposed load, weighted by fee amount.*\n\n###### How it's computed:\n*Sum of every resource fee amount parsed from the Resource partition, per node.*\n\n###### Reading it:\n*Weighted by cost rather than event count, so one heavy request at 2000 outranks eight moderate ones at 250.*\n\n###### Healthy range:\n*Proportional to peer count and request volume; compare nodes of similar role.*\n\n###### Watch for:\n*One node far above peers of the same role, which means it is carrying disproportionate peer load.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount: 2000 heavy, 250 moderate, 150 useless data.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: fee", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000000 + }, + { + "color": "red", + "value": 10000000 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P[0-9]+)\\)` | unwrap fee [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 5 + }, + { + "type": "stat", + "title": "Nodes By Manifest Rejection", + "description": "###### What this is:\n*Which nodes are rejecting the most inbound manifests.*\n\n###### How it's computed:\n*Count of ManifestCache lines with a Stale, Invalid, Revoked, or UntrustedCapacity outcome, per node.*\n\n###### Reading it:\n*A manifest flood shows up here first. Stale dominates normally because peers re-gossip manifests the node already holds.*\n\n###### Healthy range:\n*Workload-dependent; nodes on the same network should sit close together.*\n\n###### Watch for:\n*A single node far ahead, or any Invalid at all, which means a signature failed verification.*\n\n###### Keywords:\n- **Rejection** *(per manifest)* — any non-accepted disposition.\n- **UntrustedCapacity** *(per manifest)* — rate-limit rejection for an unlisted key; rc5+ builds only.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest`\n\n###### Note:\n*Requires debug logs on the ManifestCache partition.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: rejections", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "red", + "value": 10000 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 6 + }, + { + "type": "stat", + "title": "Nodes By Consensus Problem", + "description": "###### What this is:\n*Which nodes are logging the most consensus warnings and errors.*\n\n###### How it's computed:\n*Count of LedgerConsensus lines at WRN severity or above, per node.*\n\n###### Reading it:\n*Consensus is a network-wide process, so a single node standing out points at that node rather than the network.*\n\n###### Healthy range:\n*Low. Some warnings occur during normal round churn.*\n\n###### Watch for:\n*One node far above its peers, or a step change after a deploy.*\n\n###### Keywords:\n- **Consensus round** *(per ledger)* — one Open to Establish to Accepted cycle.\n- **Dispute** *(per transaction)* — a transaction peers disagree about including.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::timerEntry / checkConsensus`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: events", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 7 + }, + { + "type": "stat", + "title": "Nodes By Job Latency Breach", + "description": "###### What this is:\n*Which nodes are missing job latency targets most often.*\n\n###### How it's computed:\n*Count of LoadMonitor slow-job lines per node; the emitter only fires above a 500ms threshold.*\n\n###### Reading it:\n*A direct read on which node is most overloaded. Works at the default log level.*\n\n###### Healthy range:\n*Low. A busy node breaches occasionally.*\n\n###### Watch for:\n*A node far ahead of its peers, which usually means disk or CPU pressure local to it.*\n\n###### Keywords:\n- **Breach** *(per job)* — one job execution over the 500ms LoadMonitor threshold.\n\n###### Computation boundary:\n*Result: Per node — a count of BREACHES, never of total job executions.*\n*Derived in the Grafana query. Use the native `job_*` metrics for totals.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts breaches above 500ms only, so it is an exception count and not a latency measure.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: breaches", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 8 + }, + { + "type": "stat", + "title": "Nodes By Sync Instability", + "description": "###### What this is:\n*Which nodes are spending the most time out of the full state.*\n\n###### How it's computed:\n*Count of STATE-> transitions into any non-full state, per node.*\n\n###### Reading it:\n*A stable node holds full and emits nothing here, so any non-zero value means it left full that many times.*\n\n###### Healthy range:\n*Zero on a synced node.*\n\n###### Watch for:\n*A repeating count, which means the node is flapping rather than having had one bad moment.*\n\n###### Keywords:\n- **Operating mode** *(per node)* — Disconnected, Connected, Syncing, Tracking, Full.\n- **Flapping** *(per node)* — repeated departures from full.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "suffix: transitions", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))", + "instant": true, + "queryType": "instant" + } + ], + "id": 9 + }, + { + "type": "stat", + "title": "Nodes By Ledger Fetch Waste", + "description": "###### What this is:\n*Which nodes waste the most bandwidth fetching ledger data they already hold.*\n\n###### How it's computed:\n*Duplicate ledger nodes divided by total fetched, per node, from the InboundLedger stats lines.*\n\n###### Reading it:\n*A ratio of 0.8 means four in five fetched nodes were already present locally.*\n\n###### Healthy range:\n*Below roughly 0.3. Some duplication is unavoidable when fetching from several peers.*\n\n###### Watch for:\n*Sustained values above 0.8, which waste both bandwidth and peer resource credit.*\n\n###### Keywords:\n- **Duplicate** *(per fetch)* — a ledger node already held locally.\n\n###### Computation boundary:\n*Result: Per node — a RATIO of two summed counters, not a count.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### Note:\n*Requires debug logs on the InboundLedger partition.*\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_instance_id}", + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 0.8 + } + ] + }, + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap dupe [$__range])) / (sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap dupe [$__range])) + sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap good [$__range]))))", + "instant": true, + "queryType": "instant" + } + ], + "id": 10 + }, + { + "type": "row", + "title": "Node Operating State Transitions [DEFAULT OK]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "panels": [], + "id": 11 + }, + { + "type": "timeseries", + "title": "Node State Transition Rate", + "description": "###### What this is:\n*Rate of node operating-state transitions: disconnected, connected, syncing, tracking, and full.*\n\n###### How it's computed:\n*Per-second count of `STATE->` log lines, with the target state parsed from the line.*\n\n###### Reading it:\n*Each line is one transition INTO that state. A stable node sits in full and emits nothing.*\n\n###### Healthy range:\n*Flat at zero once synced. Any non-zero value means the node is changing state.*\n\n###### Watch for:\n*A repeating full to syncing to tracking to full cycle, which means the node cannot hold sync. Live observation shows 110 transitions per state per day on a flapping node.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n- **Transition** *(per node)* — one `STATE->` log line, emitted only when the state actually changes.\n\n###### Computation boundary:\n*Result: Per node — each series counts one server's own transitions.*\n*NOT recorded as a metric anywhere. Derived entirely in the Grafana query by regex over raw log text; xrpld only writes the line.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.state} [${__field.labels.service_instance_id}]", + "unit": "suffix: transitions/s", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Transitions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (state, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `STATE->(?P\\w+)` | state != `` | label_format state=`{{if eq .state \"full\"}}Full{{else if eq .state \"syncing\"}}Syncing{{else if eq .state \"tracking\"}}Tracking{{else if eq .state \"connected\"}}Connected{{else if eq .state \"disconnected\"}}Disconnected{{else}}{{.state}}{{end}}` [$__auto]))" + } + ], + "id": 12 + }, + { + "type": "state-timeline", + "title": "Node State Timeline", + "description": "###### What this is:\n*The node's operating state over time, as a discrete timeline.*\n\n###### How it's computed:\n*The most recent `STATE->` transition in each interval, rendered as a state band.*\n\n###### Reading it:\n*Shows at a glance how long the node spent in each state and exactly when it left full.*\n\n###### Healthy range:\n*One unbroken full band across the window.*\n\n###### Watch for:\n*Any band that is not full, and repeated narrow bands, which indicate state flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — one band per server per state.*\n*NOT recorded as a metric anywhere. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### Note:\n*Only transitions are logged, so a node that never changes state produces no data here. Read with the transition-rate panel beside it.*\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.state} [${__field.labels.service_instance_id}]", + "custom": { + "fillOpacity": 80, + "lineWidth": 0, + "spanNulls": true + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "showValue": "auto", + "rowHeight": 0.9, + "mergeValues": true, + "alignValue": "center", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (state, service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `STATE->(?P\\w+)` | state != `` | label_format state=`{{if eq .state \"full\"}}Full{{else if eq .state \"syncing\"}}Syncing{{else if eq .state \"tracking\"}}Tracking{{else if eq .state \"connected\"}}Connected{{else if eq .state \"disconnected\"}}Disconnected{{else}}{{.state}}{{end}}` [$__auto]))" + } + ], + "id": 13 + }, + { + "type": "row", + "title": "Log Volume & Severity Mix [DEFAULT OK]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "panels": [], + "id": 14 + }, + { + "type": "timeseries", + "title": "Log Line Rate By Severity", + "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's filelog receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.severity} [${__field.labels.service_instance_id}]", + "unit": "suffix: lines/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Log Lines / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (severity, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | label_format severity=`{{if eq .severity \"DBG\"}}Debug{{else if eq .severity \"NFO\"}}Info{{else if eq .severity \"WRN\"}}Warning{{else if eq .severity \"ERR\"}}Error{{else if eq .severity \"FTL\"}}Fatal{{else}}{{.severity}}{{end}}` [$__auto]))" + } + ], + "id": 15, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Log Line Rate By Partition (Top $topn)", + "description": "###### What this is:\n*The busiest xrpld log partitions by line rate.*\n\n###### How it's computed:\n*Per-second count of log lines grouped by the partition field, limited to the top N series.*\n\n###### Reading it:\n*Shows which subsystem dominates log volume, which is the main cost driver for Loki ingest.*\n\n###### Healthy range:\n*Workload-dependent. Resource, JobQueue, and LedgerConsensus are normally the loudest.*\n\n###### Watch for:\n*A partition suddenly dominating, which usually means a subsystem entered a retry or error loop.*\n\n###### Keywords:\n- **Partition** *(per line)* — the xrpld subsystem that emitted the line, e.g. ManifestCache, Resource, LedgerConsensus.\n\n###### Computation boundary:\n*Result: Per node per partition — a count of log lines.*\n*Derived in the Grafana query; truncated to the top N by Loki's series limit.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.partition} [${__field.labels.service_instance_id}]", + "unit": "suffix: lines/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Log Lines / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (partition, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" [$__auto])))" + } + ], + "id": 16, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Manifests — Disposition & Producers [DBG]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "panels": [], + "id": 17 + }, + { + "type": "timeseries", + "title": "Manifest Disposition Rate", + "description": "###### What this is:\n*Rate of manifest apply outcomes: accepted, stale, revoked, invalid, or rate-limited.*\n\n###### How it's computed:\n*Per-second count of ManifestCache log lines, with the action parsed out of the `Manifest: ;Pk: ...` text.*\n\n###### Reading it:\n*AcceptedNew and AcceptedUpdate are useful work; Stale and UntrustedCapacity are rejections and normally dominate.*\n\n###### Healthy range:\n*Accepted rates are low and bursty. A high sustained Stale rate is normal — peers gossip manifests this node already holds.*\n\n###### Watch for:\n*Any Invalid, which means a signature failed verification, and a sustained UntrustedCapacity climb, which indicates a manifest flood from unlisted keys.*\n\n###### Keywords:\n- **Manifest** *(per validator)* — a signed record binding a validator's master key to its current signing key.\n- **Disposition** *(per manifest)* — the apply outcome: AcceptedNew, AcceptedUpdate, Stale, Revoked, Invalid, UntrustedCapacity.\n- **Stale** *(per manifest)* — sequence number not greater than the one already held; the common benign rejection.\n\n###### Computation boundary:\n*Result: Per node per action — counts log EVENTS, not distinct manifests.*\n*NOT recorded as a metric. `applyManifest` has no instrumentation; this is regex over `logMftAct` output in the Grafana query.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.action} [${__field.labels.service_instance_id}]", + "unit": "suffix: manifests/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Manifests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (action, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" | regexp `Manifest: (?P[A-Za-z]+);` | action =~ \"$manifest_action\" [$__auto]))" + } + ], + "id": 18, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Manifest Accept Vs Reject Rate", + "description": "###### What this is:\n*Manifest outcomes collapsed into two series: accepted versus rejected.*\n\n###### How it's computed:\n*Per-second counts of ManifestCache lines, with AcceptedNew and AcceptedUpdate summed as accepted and all other actions summed as rejected.*\n\n###### Reading it:\n*Gives the useful-work fraction of inbound manifest processing without per-action detail.*\n\n###### Healthy range:\n*Rejected normally far exceeds accepted; peers routinely re-gossip known manifests.*\n\n###### Watch for:\n*A rejected rate in the hundreds per second, which indicates a manifest flood consuming the JtManifest job queue.*\n\n###### Keywords:\n- **Accepted** *(per manifest)* — AcceptedNew or AcceptedUpdate; the manifest changed cache state.\n- **Rejected** *(per manifest)* — Stale, Invalid, Revoked, or UntrustedCapacity.\n\n###### Computation boundary:\n*Result: Per node — two summed series over the same log lines.*\n*Derived in the Grafana query. Accepted may legitimately be absent: measured 1,360 AcceptedNew per 7d, so short windows often contain none. Both series carry `or vector(0)` so a zero-accept window renders a flat zero line rather than disappearing.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "suffix: manifests/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Manifests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "legendFormat": "Accepted [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "B", + "legendFormat": "Rejected [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)" + } + ], + "id": 19, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "table", + "title": "Top $topn Manifest Producers By Master Key", + "description": "###### What this is:\n*The master keys responsible for the most manifest log events in the selected window.*\n\n###### How it's computed:\n*Count of ManifestCache lines grouped by the base58 master key parsed from the log text, limited to the top N.\nCounts are per log EVENT, not per distinct manifest.*\n\n###### Reading it:\n*Identifies which validator keys generate the most manifest churn on this node.*\n\n###### Healthy range:\n*A small number of keys with modest repeat counts. Live observation shows roughly 19 to 24 events per key per hour.*\n\n###### Watch for:\n*A single unlisted key with a very high count, which is the signature of a targeted manifest flood.*\n\n###### Keywords:\n- **Master key** *(per validator)* — the long-lived base58 key a manifest is signed under.\n- **Log event** *(per line)* — one manifest apply attempt; the same key recurs as peers re-gossip.\n\n###### Computation boundary:\n*Result: Per master key per node — counts log EVENTS, NOT distinct manifests and NOT distinct keys.*\n*Derived in the Grafana query. Truncated to top N: Loki rejects queries returning over 2000 series, so a true distinct-key count is impossible here.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`logMftAct`\n\n###### Note:\n*Truncated to the top N. Loki rejects a query returning over 2000 series, so a full distinct-key count is not possible here.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 74 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Last *" + }, + "properties": [ + { + "id": "displayName", + "value": "Log Events" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pk" + }, + "properties": [ + { + "id": "displayName", + "value": "Master Key" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service_instance_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "displayName": "Log Events", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": ["sum"], + "countRows": false, + "fields": "" + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": ["lastNotNull"], + "labelsToFields": true, + "includeTimeField": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Field": true, + "Time": true + } + } + } + ], + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "instant": true, + "queryType": "instant", + "expr": "topk($topn, sum by (pk, service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" | regexp `Manifest: (?P[A-Za-z]+);Pk: (?P[A-Za-z0-9]+);` | pk != `` | action =~ \"$manifest_action\" [$__range])))" + } + ], + "id": 20, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Resource Fee Charges — Load Attribution [DBG]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 88 + }, + "panels": [], + "id": 21 + }, + { + "type": "timeseries", + "title": "Fee Charge Rate By Reason", + "description": "###### What this is:\n*Rate at which this node charges peers a resource fee, split by the reason for the charge.*\n\n###### How it's computed:\n*Per-second count of Resource log lines, with the reason parsed from the `for ($)` text.*\n\n###### Reading it:\n*Shows what kind of peer behaviour is costing this node the most. Moderate peer request normally dominates.*\n\n###### Healthy range:\n*Workload-dependent and roughly proportional to peer count and request volume.*\n\n###### Watch for:\n*A rising useless data rate, which means peers are sending malformed or unwanted payloads.*\n\n###### Keywords:\n- **Resource fee** *(per peer)* — an internal load credit charged against a peer, unrelated to XRP transaction fees.\n- **Reason** *(per charge)* — why the charge was applied, e.g. moderate peer request, useless data.\n\n###### Computation boundary:\n*Result: Per node per reason — a count of charge events.*\n*NOT recorded as a metric. Derived in the Grafana query from `Logic::charge` log output.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 89 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.reason} [${__field.labels.service_instance_id}]", + "unit": "suffix: charges/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Charges / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (reason, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | severity =~ \"$severity\" | regexp `for (?P[a-z ]+) \\(\\$(?P[0-9]+)\\)` | reason =~ \"$charge_reason\" | reason != `` | label_format reason=`{{if eq .reason \"useless data\"}}Useless Data{{else if eq .reason \"moderate peer request\"}}Moderate Peer Request{{else if eq .reason \"heavy peer request\"}}Heavy Peer Request{{else if eq .reason \"light peer request\"}}Light Peer Request{{else if eq .reason \"trivial peer request\"}}Trivial Peer Request{{else if eq .reason \"unwanted data\"}}Unwanted Data{{else if eq .reason \"init drop\"}}Init Drop{{else}}{{.reason}}{{end}}` [$__auto]))" + } + ], + "id": 22, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Fee-Weighted Charge Load", + "description": "###### What this is:\n*Resource fee charges weighted by the fee amount, rather than counted equally.*\n\n###### How it's computed:\n*Per-second sum of the fee value parsed from each Resource charge line, using LogQL unwrap.*\n\n###### Reading it:\n*A single heavy peer request at 2000 outweighs eight moderate ones at 250, so this ranks real cost rather than event count.*\n\n###### Healthy range:\n*Workload-dependent; should track peer request volume smoothly.*\n\n###### Watch for:\n*Spikes that do not appear in the charge-count panel, which mean a shift toward expensive request types.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount, e.g. 250 moderate, 2000 heavy; higher means costlier.\n\n###### Computation boundary:\n*Result: Per node per reason — a SUM of fee amounts, not a count of events.*\n*Derived in the Grafana query via LogQL `unwrap` over the parsed fee value.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 89 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.reason} [${__field.labels.service_instance_id}]", + "unit": "suffix: fee/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Fee Units / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (reason, service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | severity =~ \"$severity\" | regexp `for (?P[a-z ]+) \\(\\$(?P[0-9]+)\\)` | reason =~ \"$charge_reason\" | reason != `` | label_format reason=`{{if eq .reason \"useless data\"}}Useless Data{{else if eq .reason \"moderate peer request\"}}Moderate Peer Request{{else if eq .reason \"heavy peer request\"}}Heavy Peer Request{{else if eq .reason \"light peer request\"}}Light Peer Request{{else if eq .reason \"trivial peer request\"}}Trivial Peer Request{{else if eq .reason \"unwanted data\"}}Unwanted Data{{else if eq .reason \"init drop\"}}Init Drop{{else}}{{.reason}}{{end}}` | unwrap fee [$__auto]))" + } + ], + "id": 23, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "table", + "title": "Top $topn Charged Peers By IP And Public Key", + "description": "###### What this is:\n*The peers accumulating the most resource fee charges, identified by IP address and node public key.*\n\n###### How it's computed:\n*Sum of fee amounts parsed from Resource charge lines, grouped by IP and public key, limited to the top N.*\n\n###### Reading it:\n*This is the direct answer to which peer is costing this node the most, and is the primary abuse-attribution view.*\n\n###### Healthy range:\n*Charges spread across many peers with no single dominant entry.*\n\n###### Watch for:\n*One IP far above the rest, especially paired with the useless data reason, which indicates an abusive or misbehaving peer.*\n\n###### Keywords:\n- **Node public key** *(per peer)* — the peer's base58 identity, stable across reconnects.\n- **IP address** *(per peer)* — source address; the optional `:port` suffix is stripped so one peer is one row.\n\n###### Computation boundary:\n*Result: Per peer per node — a SUM of fee amounts over the dashboard window.*\n*Derived in the Grafana query. Truncated to top N by Loki's 2000-series limit, so this is the head of the distribution, never a total.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Truncated to the top N because of Loki's 2000-series query limit.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 99 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Last *" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Fee Charged" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ip" + }, + "properties": [ + { + "id": "displayName", + "value": "IP Address" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pubkey" + }, + "properties": [ + { + "id": "displayName", + "value": "Public Key" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service_instance_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "displayName": "Total Fee Charged", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": ["sum"], + "countRows": false, + "fields": "" + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": ["lastNotNull"], + "labelsToFields": true, + "includeTimeField": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Field": true, + "Time": true + } + } + } + ], + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "instant": true, + "queryType": "instant", + "expr": "topk($topn, sum by (ip, pubkey, service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | severity =~ \"$severity\" | regexp `Charging IP Address: (?P[0-9a-fA-F:.]+?)(?::[0-9]+)?, Public Key: (?P[A-Za-z0-9]+) for (?P[a-z ]+) \\(\\$(?P[0-9]+)\\)` | reason =~ \"$charge_reason\" | unwrap fee [$__range])))" + } + ], + "id": 24, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Ledger Acquisition Efficiency [DBG]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 113 + }, + "panels": [], + "id": 25 + }, + { + "type": "timeseries", + "title": "Ledger Node Fetch Duplicate Ratio", + "description": "###### What this is:\n*The fraction of fetched ledger nodes that were duplicates this node already held.*\n\n###### How it's computed:\n*Sum of the dupe counter divided by the sum of good plus dupe, parsed from the `Ledger AS/TX node stats` log lines.*\n\n###### Reading it:\n*This is wasted ledger-fetch bandwidth. A ratio of 0.5 means half of everything fetched was already present.*\n\n###### Healthy range:\n*Below roughly 0.3. Some duplication is unavoidable when fetching from several peers at once.*\n\n###### Watch for:\n*Sustained values above 0.8. Live observation has recorded good:142 dupe:891, an 86 percent duplicate rate, which wastes both bandwidth and peer resource credit.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a ratio of two summed counters, dupe / (good + dupe).*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over the counters in the InboundLedger stats line.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer / takeAsRootNode`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 114 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Duplicate Ratio", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 0.8 + } + ] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "legendFormat": "Duplicate Ratio [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap good [$__auto])))" + } + ], + "id": 26, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Ledger Node Fetch Rate — Good Vs Duplicate Vs Timeout", + "description": "###### What this is:\n*Absolute rate of useful ledger nodes fetched, duplicates received, and acquire timeouts.*\n\n###### How it's computed:\n*Per-second sums of the good, dupe, and timeouts counters parsed from InboundLedger acquire log lines.*\n\n###### Reading it:\n*Gives the absolute volumes behind the duplicate-ratio panel, so a high ratio at trivial volume can be told apart from a high ratio at scale.*\n\n###### Healthy range:\n*Good should exceed duplicate during active sync. Timeouts should stay near zero.*\n\n###### Watch for:\n*A rising timeout series, which means peers are not answering ledger requests and sync will stall.*\n\n###### Keywords:\n- **Good** *(per fetch)* — a useful, previously unheld ledger node.\n- **Timeout** *(per acquire)* — a ledger request a peer never answered.\n\n###### Computation boundary:\n*Result: Per node — SUMS of the parsed counters, giving absolute volumes behind the ratio panel.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 114 + }, + "fieldConfig": { + "defaults": { + "unit": "suffix: nodes/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Ledger Nodes / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "legendFormat": "Good [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `node stats: good:(?P[0-9]+)` | unwrap good [$__auto]))" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "B", + "legendFormat": "Duplicate [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `node stats: good:(?P[0-9]+) dupe:(?P[0-9]+)` | unwrap dupe [$__auto]))" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "C", + "legendFormat": "Acquire Timeouts [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P[0-9]+)` | unwrap timeouts [$__auto]))" + } + ], + "id": 27, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Peer Lifecycle & Disconnects [DBG]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 124 + }, + "panels": [], + "id": 28 + }, + { + "type": "timeseries", + "title": "Peer Disconnect Rate By Reason", + "description": "###### What this is:\n*Rate of peer connection endings, split by the reason recorded in the log.*\n\n###### How it's computed:\n*Per-second count of Peer log lines matching Timeout, Closed, or a refused connection attempt.*\n\n###### Reading it:\n*Distinguishes clean teardown from failure. Closed is a normal ending; Timeout and Connection refused are not.*\n\n###### Healthy range:\n*Closed dominant with a low, steady background of the others.*\n\n###### Watch for:\n*A Timeout rate approaching the Closed rate, which points at network trouble or unresponsive peers.*\n\n###### Keywords:\n- **Closed** *(per peer)* — a clean connection teardown; the normal ending.\n- **Timeout** *(per peer)* — the peer stopped responding.\n- **Connection refused** *(per attempt)* — an outbound attempt the remote rejected.\n\n###### Computation boundary:\n*Result: Per node per outcome — a count of peer lifecycle events.*\n*Derived in the Grafana query. Note `overlay_peer_disconnects` exists as a metric but carries no reason breakdown, which is what this panel adds.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close / onTimer`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 125 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.outcome} [${__field.labels.service_instance_id}]", + "unit": "suffix: events/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Peer Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (outcome, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Peer` | severity =~ \"$severity\" | regexp `\\] (?PTimeout|Closed|onConnect: Connection refused)` | outcome != `` | label_format outcome=`{{if eq .outcome \"Closed\"}}Closed{{else if eq .outcome \"Timeout\"}}Timeout{{else if eq .outcome \"onConnect: Connection refused\"}}Connection Refused{{else}}{{.outcome}}{{end}}` [$__auto]))" + } + ], + "id": 29, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Peer Handshake And Accept Rate", + "description": "###### What this is:\n*Rate of completed peer handshakes and accepted inbound connections.*\n\n###### How it's computed:\n*Per-second count of PeerFinder log lines matching handshake and accept events.*\n\n###### Reading it:\n*Read together with the disconnect panel: healthy churn has handshakes roughly balancing disconnects.*\n\n###### Healthy range:\n*Low and steady once the node has a full peer slate.*\n\n###### Watch for:\n*A high handshake rate alongside a high disconnect rate, which means peers connect and immediately drop.*\n\n###### Keywords:\n- **Handshake** *(per peer)* — protocol negotiation completed with a peer.\n- **Accept** *(per peer)* — an inbound connection admitted to a peer slot.\n\n###### Computation boundary:\n*Result: Per node — counts of PeerFinder events.*\n*Derived in the Grafana query.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/Logic.h)\n\n###### Function:\n`Logic::on_handshake / on_accept`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 125 + }, + "fieldConfig": { + "defaults": { + "unit": "suffix: events/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Peer Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "legendFormat": "Handshake [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "B", + "legendFormat": "Accept [{{service_instance_id}}]", + "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))" + } + ], + "id": 30, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Consensus Phase & Mode [DEFAULT OK]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 135 + }, + "panels": [], + "id": 31 + }, + { + "type": "timeseries", + "title": "Consensus Phase Transition Rate", + "description": "###### What this is:\n*Rate of consensus phase transitions into Open, Establish, and Accepted.*\n\n###### How it's computed:\n*Per-second count of log lines announcing a ConsensusPhase transition, grouped by the target phase.*\n\n###### Reading it:\n*A healthy node cycles Open to Establish to Accepted once per ledger, so all three series should track together.*\n\n###### Healthy range:\n*Roughly one transition per phase per ledger interval, near 0.25 per second on a 4-second close.*\n\n###### Watch for:\n*Establish transitions outnumbering Accepted, which means rounds start but fail to converge.*\n\n###### Keywords:\n- **Consensus phase** *(per round)* — Open collects transactions, Establish converges on a set, Accepted applies it.\n- **Transition** *(per round)* — one phase change; a healthy node cycles all three once per ledger.\n\n###### Computation boundary:\n*Result: Per node per phase — a count of transitions.*\n*Derived in the Grafana query by regex over the phase-change log line.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::phase transition logging`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 136 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.phase} [${__field.labels.service_instance_id}]", + "unit": "suffix: transitions/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Transitions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (phase, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `[Tt]ransitioned to ConsensusPhase::(?P\\w+)` | phase =~ \"$consensus_phase\" [$__auto]))" + } + ], + "id": 32, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Consensus Operating Mode Rate", + "description": "###### What this is:\n*Rate of consensus heartbeat log lines, split by the operating mode reported in each.*\n\n###### How it's computed:\n*Per-second count of ConsensusLogger heartbeat lines, with the mode parsed from the `mode: ` text.*\n\n###### Reading it:\n*This acts as a mode-occupancy proxy: the heartbeat fires about once per second, so the dominant series is the node's current mode.*\n\n###### Healthy range:\n*Almost entirely the full series on a synced node.*\n\n###### Watch for:\n*Any sustained syncing, observing, or tracking share, which means the node is not participating normally.*\n\n###### Keywords:\n- **Consensus mode** *(per node)* — the node's participation level: full proposing, observing, syncing, tracking.\n- **Heartbeat** *(per second)* — the ConsensusLogger timer line, emitted roughly once per second.\n\n###### Computation boundary:\n*Result: Per node per mode — a count of heartbeat lines, used as a mode-occupancy PROXY rather than a true duration.*\n*Derived in the Grafana query. For exact durations use the state_accounting metrics on the Node Health dashboard.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`ConsensusLogger heartbeat`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 136 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.mode} [${__field.labels.service_instance_id}]", + "unit": "suffix: beats/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Heartbeats / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "sum by (mode, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `NetworkOPs` | severity =~ \"$severity\" | regexp `ConsensusLogger.*?mode: (?P\\w+)\\.` | mode =~ \"$consensus_mode\" | mode != `` | label_format mode=`{{if eq .mode \"full\"}}Full{{else if eq .mode \"syncing\"}}Syncing{{else if eq .mode \"observing\"}}Observing{{else if eq .mode \"tracking\"}}Tracking{{else}}{{.mode}}{{end}}` [$__auto]))" + } + ], + "id": 33, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Validator List Quorum And Trusted Set Size", + "description": "###### What this is:\n*The quorum threshold and trusted validator count this node computed, as recorded in the log.*\n\n###### How it's computed:\n*Quorum and total are parsed from the `Using quorum of N for new set of M trusted validators` line and plotted as values.*\n\n###### Reading it:\n*Confirms the node agrees with the network on how many validators it trusts and how many must agree.*\n\n###### Healthy range:\n*Stable, with quorum at roughly 80 percent of the trusted total.*\n\n###### Watch for:\n*A drop in the trusted total, which shrinks the quorum and weakens the node's safety margin.*\n\n###### Keywords:\n- **Quorum** *(per node)* — how many trusted validators must agree; normally about 80 percent of the trusted set.\n- **Trusted set** *(per node)* — validators from the UNL this node currently trusts.\n\n###### Computation boundary:\n*Result: Per node — the last VALUE parsed from the log line, not a rate or count.*\n*Derived in the Grafana query via `unwrap`. Overlaps the `unl_quorum` metric; prefer that metric for the value and this panel for churn events.*\n\n###### Source:\n[ValidatorList.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorList.cpp)\n\n###### Function:\n`ValidatorList::updateTrusted`\n\n###### References:\n[Unique Node List](https://xrpl.org/docs/concepts/consensus-protocol/unique-node-list)", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 146 + }, + "fieldConfig": { + "defaults": { + "unit": "suffix: validators", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "axisLabel": "Validators", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "min", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "legendFormat": "Quorum [{{service_instance_id}}]", + "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap quorum [$__auto]))" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "B", + "legendFormat": "Trusted Validators [{{service_instance_id}}]", + "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap total [$__auto]))" + } + ], + "id": 34, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Slow Job Latency Breaches [DEFAULT OK]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 156 + }, + "panels": [], + "id": 35 + }, + { + "type": "timeseries", + "title": "Slow Job Run Time (p99)", + "description": "###### What this is:\n*The 99th percentile run time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the run millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Identifies which job types are the slowest when they do run long.*\n\n###### Healthy range:\n*Only a handful of job names should appear at all. Values in the tens of seconds indicate real stalls.*\n\n###### Watch for:\n*ProcessLData or InboundLedger dominating, which is the signature of the cold-read sync bottleneck.*\n\n###### Keywords:\n- **Job** *(per task)* — a unit of work on xrpld's job queue, e.g. ProcessLData, InboundLedger.\n- **Run time** *(per job)* — time spent executing, excluding queue wait.\n- **Breach** *(per job)* — an execution exceeding the 500ms LoadMonitor threshold; only these are logged.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY, never over all executions.*\n*Derived in the Grafana query via `unwrap`. For total job counts and latencies use the native `job_*` metrics on the Job Queue Analysis dashboard.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*These lines are only emitted above a 500ms latency threshold, so this panel counts BREACHES, never total job executions. Use the Job Queue Analysis dashboard for totals.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 157 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.jobname} [${__field.labels.service_instance_id}]", + "unit": "ms", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Run Time (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "quantile_over_time(0.99, {service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` | severity =~ \"$severity\" | regexp `Job: (?P\\S+) run: (?P[0-9]+)ms wait: (?P[0-9]+)ms` | unwrap run [$__auto]) by (jobname, service_instance_id)" + } + ], + "id": 36, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Slow Job Breach Rate By Job Name (Top $topn)", + "description": "###### What this is:\n*How often each job type breaches its latency target.*\n\n###### How it's computed:\n*Per-second count of LoadMonitor job lines grouped by job name, limited to the top N.*\n\n###### Reading it:\n*Read with the p99 panel: a job can breach rarely but severely, or often but mildly.*\n\n###### Healthy range:\n*Near zero. Any sustained non-zero rate means a job type is routinely missing its target.*\n\n###### Watch for:\n*A step change after a deploy, which points at a regression in that job's path.*\n\n###### Keywords:\n- **Breach** *(per job)* — an execution over the 500ms threshold.\n- **Job type** *(per task)* — the named job class, e.g. sweep, ProcessLData.\n\n###### Computation boundary:\n*Result: Per node per job type — a count of BREACHES, not of executions.*\n*Derived in the Grafana query; truncated to top N.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 157 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.jobname} [${__field.labels.service_instance_id}]", + "unit": "suffix: breaches/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Breaches / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (jobname, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` | severity =~ \"$severity\" | regexp `Job: (?P\\S+) run: (?P[0-9]+)ms wait: (?P[0-9]+)ms` | jobname != `` [$__auto])))" + } + ], + "id": 37, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "timeseries", + "title": "Slow Job Queue Wait Time (p99)", + "description": "###### What this is:\n*The 99th percentile queue wait time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the wait millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Separates queueing delay from execution cost. High wait with low run means the job queue is saturated rather than the work being slow.*\n\n###### Healthy range:\n*Near zero. Live observation shows wait at 0ms for most breaches, meaning run time is the cause.*\n\n###### Watch for:\n*Wait time rising above run time, which shifts the diagnosis from slow work to a saturated job queue.*\n\n###### Keywords:\n- **Wait time** *(per job)* — time queued before execution began.\n- **Run versus wait** *(per job)* — high wait means a saturated queue; high run means slow work.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 167 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.jobname} [${__field.labels.service_instance_id}]", + "unit": "ms", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Queue Wait Time (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "quantile_over_time(0.99, {service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` | severity =~ \"$severity\" | regexp `Job: (?P\\S+) run: (?P[0-9]+)ms wait: (?P[0-9]+)ms` | unwrap wait [$__auto]) by (jobname, service_instance_id)" + } + ], + "id": 38, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "row", + "title": "Error & Warning Stream [DEFAULT OK]", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 177 + }, + "panels": [], + "id": 39 + }, + { + "type": "timeseries", + "title": "Warning And Error Rate By Partition (Top $topn)", + "description": "###### What this is:\n*Rate of WRN, ERR, and FTL log lines by partition.*\n\n###### How it's computed:\n*Per-second count of log lines at warning severity or above, grouped by partition and limited to the top N.*\n\n###### Reading it:\n*This is the one row that works fully at default log level, so it is the first place to look on an unmodified node.*\n\n###### Healthy range:\n*Low and flat. LoadMonitor warnings are expected on a busy node.*\n\n###### Watch for:\n*Any ERR or FTL series appearing, and step changes in a partition that is normally quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — WRN, ERR, or FTL; the levels a default-configured node still writes.\n- **Partition** *(per line)* — the emitting xrpld subsystem.\n\n###### Computation boundary:\n*Result: Per node per partition per severity — a count of log lines.*\n*Derived in the Grafana query. This row is the only one that works fully at default log level.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 178 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.partition} ${__field.labels.severity} [${__field.labels.service_instance_id}]", + "unit": "suffix: lines/s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Log Lines / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "value", + "options": { + "WRN": { + "text": "Warning", + "index": 0 + }, + "ERR": { + "text": "Error", + "index": 1 + }, + "FTL": { + "text": "Fatal", + "index": 2 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["lastNotNull", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "topk($topn, sum by (partition, severity, service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `WRN|ERR|FTL` [$__auto])))" + } + ], + "id": 40, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + }, + { + "type": "logs", + "title": "Recent Warnings And Errors", + "description": "###### What this is:\n*The most recent log lines at warning severity or above.*\n\n###### How it's computed:\n*Raw log stream filtered to WRN, ERR, and FTL, newest first.*\n\n###### Reading it:\n*Use this to read the actual text behind a spike in the rate panel beside it.*\n\n###### Healthy range:\n*Sparse. LoadMonitor slow-job warnings are the common benign entry.*\n\n###### Watch for:\n*Repeating identical errors, which indicate a stuck retry loop rather than a transient fault.*\n\n###### Keywords:\n- **Log stream** *(per node)* — the raw lines behind the aggregate rates, newest first.\n\n###### Computation boundary:\n*Result: Raw log lines, no aggregation.*\n*Rendered directly from Loki; no computation applied.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 178 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "expr": "{service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `WRN|ERR|FTL` | line_format `{{.partition}}:{{.severity}} {{.message}}`" + } + ], + "id": 41, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + } + } + ] +} diff --git a/docker/telemetry/grafana/dashboards/validate_dashboards.py b/docker/telemetry/grafana/dashboards/validate_dashboards.py index e921d02c4f..d1b17dbbf6 100755 --- a/docker/telemetry/grafana/dashboards/validate_dashboards.py +++ b/docker/telemetry/grafana/dashboards/validate_dashboards.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -"""Dashboard lint: cumulative metrics must be rate()-wrapped; tier filters present.""" +"""Dashboard lint: cumulative metrics rate()-wrapped; tier filters; sane panel grid.""" import json, re, sys +GRID_COLUMNS = 24 + # Prometheus gauges that hold a CUMULATIVE total -> must be rate()/increase()-wrapped. CUMULATIVE_PREFIXES = ( "total_bytes_", @@ -55,6 +57,55 @@ def iter_panels(dash): yield sub +def check_layout(path, dash): + """Duplicate panel ids and grid collisions -- both break Grafana's loader. + + Grafana keys panels by id when it builds the dashboard model, so two panels + sharing an id make the load non-deterministic. Overlapping gridPos rectangles + have no valid layout. Only top-level panels are checked: panels nested inside + a collapsed row do not occupy the outer grid. + """ + errs = [] + top = dash.get("panels", []) or [] + + seen_ids = {} + for p in top: + pid = p.get("id") + if pid is None: + continue + if pid in seen_ids: + errs.append( + f"{path}: duplicate panel id {pid}: " + f"[{seen_ids[pid]}] and [{p.get('title', '')}]" + ) + else: + seen_ids[pid] = p.get("title", "") + + # Mark every grid cell each panel covers; a second claim on a cell is a collision. + owner = {} + for p in top: + g = p.get("gridPos") or {} + x, y = g.get("x", 0), g.get("y", 0) + w, h = g.get("w", 0), g.get("h", 0) + title = p.get("title", "") + if x + w > GRID_COLUMNS: + errs.append( + f"{path} [{title}]: spans past the {GRID_COLUMNS}-column grid (x={x}, w={w})" + ) + clashed = set() + for cy in range(y, y + h): + for cx in range(x, min(x + w, GRID_COLUMNS)): + prev = owner.get((cy, cx)) + if prev is None: + owner[(cy, cx)] = title + elif prev not in clashed: + clashed.add(prev) + errs.append( + f"{path} [{title}]: grid overlap with [{prev}] at y={cy} x={cx}" + ) + return errs + + def expr_is_wrapped(expr): return ( "rate(" in expr @@ -70,6 +121,7 @@ def check(path, forbid_5m): dash = json.load(open(path)) except Exception as e: return [f"{path}: INVALID JSON: {e}"] + errs += check_layout(path, dash) for p in iter_panels(dash): title = p.get("title", "") for tg in p.get("targets", []) or []: diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index 922caed081..03fa2344b4 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -7,23 +7,44 @@ # xrpld-critical — Slack + email; receives critical-severity alerts. # The severity split is wired in policies.yaml. # -# Secrets and personal addresses are NOT hard-coded. To enable delivery, -# replace the disabled placeholder values below with a real Slack webhook -# and alert email — e.g. by copying docker/telemetry/.env.alerting.example -# to .env.alerting (gitignored) and swapping the placeholder lines to -# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of -# docs/telemetry-runbook.md for setup. -# -# The defaults are deliberately non-empty, invalid-but-valid-shaped +# --------------------------------------------------------------------------- +# Supplying a real destination +# --------------------------------------------------------------------------- +# The values below are deliberately non-empty, invalid-but-valid-shaped # placeholders (an unroutable webhook host and a .invalid email). Grafana's -# alerting provisioning validator REQUIRES a non-empty Slack url and email -# addresses, and it does NOT support ${VAR:-default} expansion — an unset -# ${VAR} expands to empty and crashes Grafana on startup. The placeholders -# keep the whole stack booting with zero configuration; alerts simply route -# nowhere until a real destination is supplied. +# provisioning validator REQUIRES a non-empty Slack url and a non-empty email +# `addresses`, so the placeholders keep the whole stack booting with zero +# configuration; alerts simply route nowhere until a destination is supplied. # -# Email delivery additionally requires SMTP configured on the Grafana -# service (GF_SMTP_* in docker-compose.yml). +# To enable delivery, edit the two placeholder values IN PLACE with a real +# webhook / address. Do NOT commit the result — and do not substitute +# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO} here expecting a fallback: +# Grafana expands ${VAR} but does NOT support ${VAR:-default}, so an unset +# variable expands to empty, fails validation, and Grafana exits 1 — taking +# the whole telemetry stack down, not just alerting. +# +# Two further traps when editing this file: +# +# * Never leave a `receivers:` list empty to "disable" a tier. A contact +# point with no receivers ceases to exist, the policy tree then references +# a missing receiver, and Grafana refuses to boot. +# +# * File provisioning is upsert-only. Deleting a receiver here does NOT +# remove it from an instance that already booted with it — the old +# receiver keeps delivering. Removal requires an explicit +# `deleteContactPoints:` block (see the commented example at the bottom). +# +# Email delivery additionally requires SMTP configured on the Grafana service. +# The GF_SMTP_* variables live in docker/telemetry/.env.alerting (gitignored, +# see .env.alerting.example) and reach Grafana via the compose `env_file`; +# they are NOT set in docker-compose.yml. Email stays off until +# GF_SMTP_ENABLED=true and the relay settings point somewhere real. +# +# Grafana Cloud does NOT use this file — Cloud has no provisioning filesystem. +# Cloud delivery is created over the REST API by upload_alerts_to_grafana.py, +# which builds a single email-only contact point and attaches it to each rule +# via per-rule notification_settings. See that script's header for why the +# notification policy tree must not be touched on a shared Cloud instance. apiVersion: 1 @@ -37,10 +58,14 @@ contactPoints: settings: # Disabled placeholder: an unroutable webhook host. A non-empty url # selects Slack webhook mode (no recipient/token required) and keeps - # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to - # enable delivery. + # provisioning valid. Replace with a real webhook to enable delivery. url: https://hooks.slack.invalid/disabled - title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + # `rulename` is used rather than CommonLabels.service_instance_id + # because on NoData/Error evaluations Grafana replaces the query's + # label set with only {datasource_uid, ref_id}, so the node label is + # absent and the title would render blank — and several rules are + # deliberately configured to fire that way. + title: "{{ .CommonLabels.rulename }}" disableResolveMessage: false # --- Critical tier: Slack + email --- @@ -52,14 +77,25 @@ contactPoints: settings: # Disabled placeholder — see xrpld-slack-default above. url: https://hooks.slack.invalid/disabled - title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + title: "[CRITICAL] {{ .CommonLabels.rulename }}" disableResolveMessage: false - uid: xrpld-email-critical type: email settings: # Disabled placeholder: a .invalid address keeps the required # `addresses` field non-empty so provisioning validates. Replace - # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver. + # with a real address (and enable SMTP) to deliver. Multiple + # recipients are semicolon- or comma-separated. addresses: alerts-disabled@xrpld.invalid + # One message listing all recipients, rather than one message each. singleEmail: true disableResolveMessage: false + +# To retire a receiver that a running Grafana has already stored, uncomment +# and list its uid here — deleting the block above is not sufficient: +# +# deleteContactPoints: +# - orgId: 1 +# uid: xrpld-slack-default +# - orgId: 1 +# uid: xrpld-slack-critical diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml index cf396da0c5..d8a2803c9f 100644 --- a/docker/telemetry/grafana/provisioning/alerting/rules.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -2,20 +2,42 @@ # # Phase 9: Internal metric gap fill — alerting on health-critical metrics. # -# Six rules across three node-fatal subsystems: consensus/ledger health, -# validator health, and the job queue. Every metric referenced here is -# introduced by phase 9's MetricsRegistry. +# Twelve rules across five subsystems: consensus/ledger health, validator +# health, the job queue, node operating state, and the overlay (manifests). # # Rule shape (Grafana server-side evaluation): -# A Prometheus query — a 5-minute rate / histogram_quantile, aggregated +# A Prometheus query — a rate / increase / histogram_quantile, aggregated # `by (service_instance_id)` so each node evaluates # independently. Alert rules run headless and cannot # use the dashboards' `$node` template variable. # B reduce (last) — collapse A's series to its most recent value. # C threshold — the firing condition; `condition: C`. # +# Conventions that are load-bearing — do not "simplify" these away: +# +# * Every expr selects {service_name="xrpld"}. The same Prometheus also +# hosts a legacy statsd fleet under job="integrations/unix" which exports +# some of these names (state_accounting_* in particular) with no xrpld +# resource attributes. Without the selector those series get summed in. +# +# * `isPaused: true` on every rule. The key is camelCase; `is_paused` is +# SILENTLY IGNORED by the provisioning loader (no error, no warning) and +# leaves the rule live. Note the inconsistency: the sibling field +# `notification_settings` IS snake_case. +# +# * Sparse counters use increase(...[15m]) with a short `for`, not +# rate(...[5m]) with for: 5m. A single increment keeps rate[5m] nonzero +# for only ~4 minutes of dwell, so `for: 5m` can never be satisfied and +# the rule silently never fires for one-off events. +# +# * Rules whose intent is "this node stopped doing X" synthesise an explicit +# zero via `or (0 * max_over_time(...))`, because `sum by()` returns rows +# only for still-reporting nodes: a single dead node's row just disappears +# and noDataState never triggers. +# # Thresholds are documented in docs/telemetry-runbook.md (Alerting section) -# and are intended to be tuned. +# and are derived from measured values across a 7-node dev/devnet population. +# Production nodes (higher peer counts, real traffic) need a re-tune. apiVersion: 1 @@ -29,12 +51,14 @@ groups: interval: 1m rules: # A closed ledger that later fails validation against the network — - # any sustained nonzero rate means this node built history the rest - # of the network rejects. + # any mismatch means this node built history the rest of the network + # rejects. A healthy node never mismatches, so a single event matters: + # hence increase() over a wide window rather than a decaying rate(). - uid: xrpld-ledger-history-mismatch title: LedgerHistoryMismatch condition: C - for: 5m + for: 2m + isPaused: true noDataState: NoData execErrState: Error labels: @@ -43,25 +67,25 @@ groups: annotations: summary: "Ledger history mismatch on {{ $labels.service_instance_id }}" description: >- - Node {{ $labels.service_instance_id }} is recording ledger history - mismatches ({{ $values.B.Value }}/s over 5m). The node's built ledger - diverges from the validated network chain. + Node {{ $labels.service_instance_id }} recorded + {{ $values.B.Value }} ledger history mismatch(es) in the last 15m. + The node's built ledger diverges from the validated network chain. data: - refId: A relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(ledger_history_mismatch_total[5m])) + expr: sum by (service_instance_id) (increase(ledger_history_mismatch_total{service_name="xrpld"}[15m])) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - refId: B relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: __expr__ model: @@ -74,7 +98,7 @@ groups: uid: __expr__ - refId: C relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: __expr__ model: @@ -90,12 +114,15 @@ groups: uid: __expr__ # Healthy nodes close a ledger every ~3-5s. Zero closes for 3 minutes - # means consensus/ledger advancement is stuck. NoData (metric absent) - # also fires — a vanished series here means the node is down. + # means consensus/ledger advancement is stuck. The `or 0 * + # max_over_time` term synthesises a zero row for a node that was + # reporting within the last hour but has now gone silent, so a single + # dead node trips the threshold instead of vanishing from the result. - uid: xrpld-ledger-close-stalled title: LedgerCloseStalled condition: C for: 3m + isPaused: true noDataState: Alerting execErrState: Error labels: @@ -106,7 +133,7 @@ groups: description: >- Node {{ $labels.service_instance_id }} has closed no ledgers for several minutes (5m rate has decayed to zero). Consensus or ledger - advancement is stuck. + advancement is stuck, or the process is gone. data: - refId: A relativeTimeRange: @@ -115,7 +142,9 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(ledgers_closed_total[5m])) + expr: |- + sum by (service_instance_id) (rate(ledgers_closed_total{service_name="xrpld"}[5m])) + or (0 * max by (service_instance_id) (max_over_time(ledgers_closed_total{service_name="xrpld"}[1h]))) instant: true range: false intervalMs: 1000 @@ -150,32 +179,26 @@ groups: type: __expr__ uid: __expr__ - # ------------------------------------------------------------------ # - # Validator health # - # ------------------------------------------------------------------ # - - orgId: 1 - name: xrpld-validator - folder: xrpld - interval: 1m - rules: - # This validator's own validations are not reaching / agreeing with - # the network. A sustained nonzero miss rate risks the validator being - # dropped from UNLs. - - uid: xrpld-validations-missed - title: ValidationsMissed + # The validated ledger falling behind wall-clock is the single clearest + # "this node is unhealthy" signal on XRPL: it is the symptom every other + # consensus/sync failure eventually produces. Measured p95 is 4s on every + # node over 24h, so 60s carries ~15x headroom. + - uid: xrpld-validated-ledger-stale + title: ValidatedLedgerStale condition: C for: 5m + isPaused: true noDataState: NoData execErrState: Error labels: - severity: warning - category: validator + severity: critical + category: consensus annotations: - summary: "Validations missed on {{ $labels.service_instance_id }}" + summary: "Validated ledger stale on {{ $labels.service_instance_id }}" description: >- - Node {{ $labels.service_instance_id }} is missing validations - ({{ $values.B.Value }}/s over 5m). Its validations are not agreeing with - the validated ledger. + Node {{ $labels.service_instance_id }} has a validated ledger age of + {{ $values.B.Value }}s (>60s). The node is not keeping up with the + validated network chain. data: - refId: A relativeTimeRange: @@ -184,7 +207,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(validation_missed_total[5m])) + expr: max by (service_instance_id) (ledgermaster_validated_ledger_age{service_name="xrpld"}) instant: true range: false intervalMs: 1000 @@ -214,18 +237,109 @@ groups: conditions: - evaluator: type: gt - params: [0] + params: [60] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Validator health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-validator + folder: xrpld + interval: 1m + rules: + # This validator's own validations are not agreeing with the network. + # + # IMPORTANT — why this is a ratio gated on validations_sent_total, and + # not `rate(validation_missed_total) > 0`: + # ValidationTracker classifies a ledger as a miss whenever + # (weValidated && networkValidated) is not both true. A node that does + # not validate never sets weValidated, so EVERY reconciled ledger counts + # as a miss and the raw rate is permanently nonzero — measured ratio is + # exactly 1.0 on non-validating nodes. No threshold can separate "not a + # validator" from "validator disagreeing"; the `and on(...)` gate + # excludes non-validators entirely, and the ratio then measures real + # disagreement among nodes that do validate. + - uid: xrpld-validations-missed + title: ValidationsMissed + condition: C + for: 15m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: validator + annotations: + summary: "Validations missed on {{ $labels.service_instance_id }}" + description: >- + Validator {{ $labels.service_instance_id }} is missing + {{ $values.B.Value }} (fraction) of its validations over 15m. Its + validations are not agreeing with the validated ledger, which risks + removal from UNLs. + data: + - refId: A + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: |- + ( + sum by (service_instance_id) (rate(validation_missed_total{service_name="xrpld"}[15m])) + / clamp_min( + sum by (service_instance_id) (rate(validation_missed_total{service_name="xrpld"}[15m])) + + sum by (service_instance_id) (rate(validation_agreements_total{service_name="xrpld"}[15m])), + 1e-9) + ) + and on (service_instance_id) + (sum by (service_instance_id) (rate(validations_sent_total{service_name="xrpld"}[15m])) > 0) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0.1] datasource: type: __expr__ uid: __expr__ # The node has stopped checking incoming validations. Zero checked # validations means it is no longer processing the validation stream - # from peers. + # from peers. Synthesises a zero for a silent-but-recently-seen node + # (see the LedgerCloseStalled comment). - uid: xrpld-validations-not-checked title: ValidationsNotChecked condition: C for: 5m + isPaused: true noDataState: Alerting execErrState: Error labels: @@ -245,7 +359,9 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(validations_checked_total[5m])) + expr: |- + sum by (service_instance_id) (rate(validations_checked_total{service_name="xrpld"}[5m])) + or (0 * max by (service_instance_id) (max_over_time(validations_checked_total{service_name="xrpld"}[1h]))) instant: true range: false intervalMs: 1000 @@ -289,11 +405,14 @@ groups: interval: 1m rules: # Transactions are being dropped because the job queue is full — the - # node is shedding load it cannot process. + # node is shedding load it cannot process. Overflow arrives in bursts, + # so this uses increase() over a wide window (see the header note on + # sparse counters). - uid: xrpld-jobqueue-tx-overflow title: JobQueueTxOverflow condition: C - for: 5m + for: 2m + isPaused: true noDataState: NoData execErrState: Error labels: @@ -302,25 +421,25 @@ groups: annotations: summary: "Job queue transaction overflow on {{ $labels.service_instance_id }}" description: >- - Node {{ $labels.service_instance_id }} is overflowing its transaction - job queue ({{ $values.B.Value }}/s over 5m). Transactions are being - dropped under load. + Node {{ $labels.service_instance_id }} overflowed its transaction + job queue {{ $values.B.Value }} time(s) in the last 15m. + Transactions are being dropped under load. data: - refId: A relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(jq_trans_overflow_total[5m])) + expr: sum by (service_instance_id) (increase(jq_trans_overflow_total{service_name="xrpld"}[15m])) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - refId: B relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: __expr__ model: @@ -333,7 +452,7 @@ groups: uid: __expr__ - refId: C relativeTimeRange: - from: 600 + from: 1200 to: 0 datasourceUid: __expr__ model: @@ -349,12 +468,13 @@ groups: uid: __expr__ # p99 time a job waits in the queue before running. A sustained p99 - # above 1s means the node is saturated and work is backing up. Tune the - # threshold (in microseconds) to the deployment. + # above 1s means the node is saturated and work is backing up. `le` must + # stay inside the inner sum or histogram_quantile cannot interpolate. - uid: xrpld-jobqueue-latency-high title: JobQueueLatencyHigh condition: C for: 5m + isPaused: true noDataState: NoData execErrState: Error labels: @@ -374,7 +494,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m]))) + expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket{service_name="xrpld"}[5m]))) instant: true range: false intervalMs: 1000 @@ -408,3 +528,439 @@ groups: datasource: type: __expr__ uid: __expr__ + + # Node-store read/write latency. Sustained high IO latency is the usual + # upstream cause of state flapping and sync stalls, so this often fires + # first and explains the others. Measured p99-of-p95 is 37-49ms on + # healthy nodes and 488-566ms on nodes that are actively flapping, so + # 1000ms flags genuine degradation rather than the current baseline. + - uid: xrpld-nodestore-io-latency-high + title: NodeStoreIOLatencyHigh + condition: C + for: 10m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: jobqueue + annotations: + summary: "Node store IO latency high on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has a p95 node-store IO + latency of {{ $values.B.Value }}ms (>1s) over 10m. Check disk + utilisation and whether the store is on a slow volume. + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: histogram_quantile(0.95, sum by (le, service_instance_id) (rate(ios_latency_milliseconds_bucket{service_name="xrpld"}[10m]))) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [1000] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Node operating state # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-node-state + folder: xrpld + interval: 1m + rules: + # Node state flapping: full -> syncing/tracking -> full, repeatedly. + # + # state_accounting_full_transitions counts transitions INTO full + # (NetworkOPs.cpp StateAccounting::mode) and is exported as a cumulative + # gauge, so increase() is correct — and its counter-reset correction + # turns a process restart into a small positive delta rather than a + # false spike. + # + # state_changes_total cannot be used here: it carries no from/to labels, + # so it cannot distinguish a flap from a normal startup walk. + # + # The uptime gate is load-bearing. Every node walks + # disconnected -> connected -> syncing -> tracking -> full once at boot; + # without the gate every restart pages. Measured: flapping nodes re-enter + # full 4-6 times per hour sustained, healthy nodes 0-1, so >3 separates + # the populations with a 3x margin. + - uid: xrpld-node-state-flapping + title: NodeStateFlapping + condition: C + for: 15m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: node_state + annotations: + summary: "Node state flapping on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} re-entered the FULL state + {{ $values.B.Value }} times in the last hour (>3). It is oscillating + between full and syncing/connected rather than holding sync. Check + node-store IO latency, peer connectivity, and clock sync. + data: + - refId: A + relativeTimeRange: + from: 3900 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: |- + sum by (service_instance_id) (increase(state_accounting_full_transitions{service_name="xrpld"}[1h])) + and on (service_instance_id) + (sum by (service_instance_id) (server_info{service_name="xrpld", metric="uptime"}) > 3600) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 3900 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 3900 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [3] + datasource: + type: __expr__ + uid: __expr__ + + # A node stuck OUT of full. Distinct from flapping: a node that drops to + # syncing and stays there produces no further full-transitions, so the + # flapping rule by definition cannot catch it. + # server_state enum (NetworkOPs.h): DISCONNECTED=0, CONNECTED=1, + # SYNCING=2, TRACKING=3, FULL=4. + - uid: xrpld-node-not-full + title: NodeNotFull + condition: C + for: 15m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: node_state + annotations: + summary: "Node not in FULL state on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has been below FULL + (state={{ $values.B.Value }}; 0=disconnected 1=connected 2=syncing + 3=tracking 4=full) for 15m. It is not fully synced with the network. + data: + - refId: A + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: |- + max by (service_instance_id) (server_info{service_name="xrpld", metric="server_state"}) + and on (service_instance_id) + (sum by (service_instance_id) (server_info{service_name="xrpld", metric="uptime"}) > 3600) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: lt + params: [4] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Overlay / manifests # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-overlay + folder: xrpld + interval: 1m + rules: + # Manifest job convoy — the primary manifest-flooding signal. + # + # Peers send TMManifests dumps up to ~57MB (just under + # kMaximumMessageSize, overlay/Message.h). JtManifest is registered with + # maxLimit (core/JobTypes.h), so every peer's dump runs concurrently and + # they convoy on ManifestCache::mutex_; OverlayImpl::onManifests also + # re-verifies the blob a second time on Accept. Measured effect: each + # RcvManifests job took 16-18s and the whole 8-worker pool was occupied. + # + # jobq_manifest_waiting is 0 at the 99.9th percentile on every node over + # 24h, so any sustained backlog is a genuine outlier rather than normal + # variance. Threshold >3 with a 10m dwell keeps the measured startup + # burst (peaks of 5 and 11, lasting well under 10m) from paging. + - uid: xrpld-manifest-job-convoy + title: ManifestJobQueueConvoy + condition: C + for: 10m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: overlay + annotations: + summary: "Manifest job convoy on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has {{ $values.B.Value }} + manifest jobs waiting (>3) for 10m. Peer manifest dumps are + saturating the job pool and convoying on the manifest cache lock. + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (jobq_manifest_waiting{service_name="xrpld"}) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [3] + datasource: + type: __expr__ + uid: __expr__ + + # Inbound manifest byte-rate flood. Complements the convoy rule: this + # catches the wire-level cause (a peer shipping huge dumps) even when the + # job pool absorbs it without a visible backlog. + # + # Measured: steady state 0.7-2.3 kB/s; p99 during the startup flood + # 267-420 kB/s. 50 kB/s sits ~20x above steady state and well below the + # flood. The uptime gate suppresses the measured startup storm, which is + # normal behaviour — the trade-off is that a flood confined to the first + # 30 minutes after boot is deliberately not alerted. + - uid: xrpld-manifest-flood-inbound + title: ManifestFloodInbound + condition: C + for: 10m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: overlay + annotations: + summary: "Inbound manifest flood on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is receiving + {{ $values.B.Value }} B/s of manifest traffic (>50 kB/s) over 10m. + A peer is flooding oversized TMManifests dumps. + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: |- + sum by (service_instance_id) (rate(overhead_manifest_bytes_in{service_name="xrpld"}[10m])) + and on (service_instance_id) + (sum by (service_instance_id) (server_info{service_name="xrpld", metric="uptime"}) > 1800) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [51200] + datasource: + type: __expr__ + uid: __expr__ + + # Resource-driven peer disconnects. The node is dropping peers for + # exceeding resource budgets, which precedes peer starvation and sync + # loss. Sparse and bursty, so increase() over a wide window. + # + # Measured p95 of the 30m increase: 0 on every healthy node, 4.0 and 11.7 + # on the two nodes that are independently known to be degraded (the same + # two that flap). So >5 sits above the healthy baseline entirely and only + # trips on a node already in trouble. + - uid: xrpld-peer-resource-disconnects + title: PeerResourceDisconnects + condition: C + for: 5m + isPaused: true + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: overlay + annotations: + summary: "Resource-driven peer disconnects on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} disconnected + {{ $values.B.Value }} peer(s) for resource-budget violations in the + last 30m. Sustained disconnects can starve the node of peers. + data: + - refId: A + relativeTimeRange: + from: 2100 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (increase(server_info{service_name="xrpld", metric="peer_disconnects_resources"}[30m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 2100 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 2100 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [5] + datasource: + type: __expr__ + uid: __expr__ diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 939cba641b..4aaff6444c 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1877,7 +1877,15 @@ three signals' attributes over OTLP directly. ## Grafana Dashboards -Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: +Fifteen dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`. +Fourteen are Prometheus-backed; `log-derived-insights` is the only Loki/LogQL +board and is documented last, together with the LogQL-specific traps it exposed. + +> Nine dashboards have a reference section below. `fee-market`, `job-queue`, +> `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality`, and +> `validator-health` are provisioned but not yet documented here — their panel +> descriptions carry the same six-heading reference format, so open the panel +> info icon in Grafana until a section is written. ### RPC Performance (`rpc-performance`) @@ -2052,26 +2060,54 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ## Alerting -xrpld provisions six Grafana alert rules on the health-critical metrics, so a -stock stack alerts out of the box with no UI setup. Rules are provisioned from +xrpld provisions thirteen Grafana alert rules on the health-critical metrics, so +a stock stack alerts out of the box with no UI setup. Rules are provisioned from `docker/telemetry/grafana/provisioning/alerting/` and load automatically when the Grafana container starts. They appear under **Alerting → Alert rules**, folder **xrpld**. +> **All rules ship `isPaused: true`.** Thresholds are tuned against a small +> dev/devnet population, so every rule is deactivated on arrival — compare it +> against your own baseline, then unpause. The key is camelCase: `is_paused` is +> **silently ignored** by the provisioning loader (no error, no warning) and +> leaves the rule live. Note the sibling field `notification_settings` _is_ +> snake_case. + ### Alert catalogue -All rules evaluate every minute against the Prometheus datasource, over a -5-minute window, and group by `service_instance_id` so each node alerts on its -own. Alerts fire only after the condition holds for the `for` dwell time. +All rules evaluate every minute against the Prometheus datasource and aggregate +`by (service_instance_id)` so each node alerts on its own. Every expr selects +`{service_name="xrpld"}` — the same Prometheus may also host a legacy statsd +fleet exporting some of these names (`state_accounting_*` in particular) with no +xrpld resource attributes, and without the selector those series get summed in. +Alerts fire only after the condition holds for the `for` dwell time. -| Alert | Severity | Fires when | For | -| ----------------------- | -------- | ----------------------------------------- | --- | -| `LedgerHistoryMismatch` | critical | `rate(ledger_history_mismatch_total)` > 0 | 5m | -| `LedgerCloseStalled` | critical | `rate(ledgers_closed_total)` ≈ 0 | 3m | -| `ValidationsMissed` | warning | `rate(validation_missed_total)` > 0 | 5m | -| `ValidationsNotChecked` | warning | `rate(validations_checked_total)` ≈ 0 | 5m | -| `JobQueueTxOverflow` | warning | `rate(jq_trans_overflow_total)` > 0 | 5m | -| `JobQueueLatencyHigh` | warning | p99 `job_queued_us` > 1s | 5m | +| Alert | Severity | Fires when | For | +| ------------------------- | -------- | -------------------------------------------------- | --- | +| `LedgerHistoryMismatch` | critical | `increase(ledger_history_mismatch_total[15m])` > 0 | 2m | +| `LedgerCloseStalled` | critical | `rate(ledgers_closed_total)` ≈ 0 | 3m | +| `ValidatedLedgerStale` | critical | `ledgermaster_validated_ledger_age` > 60s | 5m | +| `ValidationsMissed` | warning | validator miss _ratio_ > 0.1 | 15m | +| `ValidationsNotChecked` | warning | `rate(validations_checked_total)` ≈ 0 | 5m | +| `JobQueueTxOverflow` | warning | `increase(jq_trans_overflow_total[15m])` > 0 | 2m | +| `JobQueueLatencyHigh` | warning | p99 `job_queued_us` > 1s | 5m | +| `NodeStoreIOLatencyHigh` | warning | p95 `ios_latency_milliseconds` > 1s | 10m | +| `NodeStateFlapping` | warning | > 3 re-entries into FULL per hour | 15m | +| `NodeNotFull` | warning | `server_state` < 4 (FULL) | 15m | +| `ManifestJobQueueConvoy` | warning | `jobq_manifest_waiting` > 3 | 10m | +| `ManifestFloodInbound` | warning | `rate(overhead_manifest_bytes_in)` > 50 kB/s | 10m | +| `PeerResourceDisconnects` | warning | > 5 resource-driven peer disconnects per 30m | 5m | + +Two expression idioms recur and are load-bearing — do not "simplify" them away: + +- **Sparse counters use `increase(...[15m])` with a short `for`,** not + `rate(...[5m])` with `for: 5m`. A single increment keeps `rate[5m]` nonzero for + only ~4 minutes of dwell, so a 5-minute `for` can never be satisfied and the + rule silently never fires for the one-off events it exists to catch. +- **"Node stopped doing X" rules synthesise an explicit zero** via + `or (0 * max_over_time(...[1h]))`, because `sum by()` returns rows only for + still-reporting nodes: one dead node's row simply disappears from the result, + so `noDataState` never triggers unless _every_ node vanishes at once. #### Consensus / ledger health @@ -2085,12 +2121,28 @@ one every ~3-5s. Likely causes: lost peer connectivity, consensus stall, or the process is hung. This rule also fires on _NoData_ — if the series disappears the node is likely down. Check peer count and process health first. +**ValidatedLedgerStale** — The validated ledger has fallen more than 60s behind. +This is the clearest single "is this node healthy" signal on XRPL: it is the +symptom nearly every consensus or sync failure eventually produces, so it is +often the first thing to check and the last thing to clear. Measured p95 is ~4s +on a healthy node. + #### Validator health **ValidationsMissed** — This validator's validations are not agreeing with the validated ledger. Sustained misses risk removal from UNLs. Check clock sync, peer connectivity, and whether the node is keeping up with ledger close. +> **Why this is a ratio gated on `validations_sent_total`, not +> `rate(validation_missed_total) > 0`:** `ValidationTracker` classifies a ledger +> as a miss whenever `weValidated && networkValidated` is not _both_ true. A node +> that does not validate never sets `weValidated`, so **every** reconciled ledger +> counts as a miss and the raw rate is permanently nonzero — the measured miss +> ratio is exactly `1.0` on non-validating nodes. No threshold can separate "not +> a validator" from "validator disagreeing", so the rule gates on +> `validations_sent_total > 0` to exclude non-validators entirely, and then +> measures the ratio among nodes that genuinely do validate. + **ValidationsNotChecked** — The node has stopped checking incoming validations from peers. Likely causes: overlay/peer disconnection or a stalled validation pipeline. Fires on NoData as well. @@ -2105,6 +2157,71 @@ being dropped. The node is shedding load it cannot process. Check CPU, the before running. The node is saturated. Correlate with CPU and the Job Queue dashboard. +**NodeStoreIOLatencyHigh** — p95 node-store IO latency exceeds 1s. Sustained +store latency is the usual _upstream cause_ of state flapping and sync stalls, so +this often fires alongside `NodeStateFlapping` and explains it. Check disk +utilisation and whether the node store sits on a slow volume — moving it to a +local NVMe has previously cut time-to-`full` by more than 3x. Measured p99-of-p95 +is 37-49ms on healthy nodes and 488-566ms on nodes that are actively flapping. + +#### Node operating state + +**NodeStateFlapping** — The node is oscillating `full → syncing/connected → full` +instead of holding sync. Measured: a flapping node re-enters `full` 4-6 times per +hour sustained, while a healthy node manages 0-1, so the `> 3` threshold sits +between the two populations with roughly a 3x margin. + +The rule counts `state_accounting_full_transitions`, which counts transitions +_into_ `full` and is exported as a cumulative gauge — `increase()` is therefore +correct, and its counter-reset correction turns a process restart into a small +positive delta rather than a false spike. `state_changes_total` cannot be used +here: it carries no from/to labels, so it cannot tell a flap from a normal +startup walk. + +**The `uptime > 3600` gate is load-bearing.** Every node walks +`disconnected → connected → syncing → tracking → full` once at boot; without the +gate, every restart pages. The trade-off is deliberate: flapping confined to the +first hour after boot is not alerted. + +Investigate in this order: `NodeStoreIOLatencyHigh` (most common cause), peer +connectivity, then clock sync. + +**NodeNotFull** — The node has been below `FULL` for 15m +(`0`=disconnected, `1`=connected, `2`=syncing, `3`=tracking, `4`=full). This is +deliberately a _separate_ rule from `NodeStateFlapping`: a node that drops to +syncing and stays there produces no further full-transitions, so the flapping +counter by definition cannot catch it. + +#### Overlay / manifests + +**ManifestJobQueueConvoy** — Manifest jobs are backing up in the job queue. Peers +send `TMManifests` dumps up to ~57MB (just under `kMaximumMessageSize`, see +`overlay/Message.h`), and `JtManifest` is registered with `maxLimit` +(`core/JobTypes.h`), so every peer's dump runs concurrently and they convoy on +`ManifestCache::mutex_`; `OverlayImpl::onManifests` also re-verifies the blob a +second time on Accept. Measured effect: each `RcvManifests` job took 16-18s and +the entire 8-worker pool was occupied. + +This is the most reliable manifest-flood signal because `jobq_manifest_waiting` +is `0` at the 99.9th percentile on every node over 24h — any sustained backlog is +a genuine outlier rather than normal variance. + +**ManifestFloodInbound** — Inbound manifest byte-rate exceeds 50 kB/s. Catches the +wire-level cause (a peer shipping oversized dumps) even when the job pool absorbs +it without a visible backlog. Measured steady state is 0.7-2.3 kB/s against a p99 +of 267-420 kB/s during the startup flood, so the threshold sits ~20x above normal +and well below a real flood. + +> **Both manifest rules deliberately suppress startup.** The manifest storm at +> boot is _measured normal behaviour_, so `ManifestFloodInbound` carries an +> `uptime > 1800` gate and `ManifestJobQueueConvoy` relies on a 10m dwell that the +> startup burst does not outlast. A flood confined to the first 30 minutes after +> boot will therefore not alert. + +**PeerResourceDisconnects** — The node dropped more than 5 peers in 30m for +exceeding resource budgets. Sustained disconnects starve the node of peers and +precede sync loss. + ### Tuning thresholds Thresholds live in @@ -2142,45 +2259,111 @@ to Slack **and** email; a warning goes to Slack only. Both group by #### Configure delivery (no secrets in git) -The Slack webhook and email address are **not** hard-coded — the YAML -references `${SLACK_WEBHOOK_URL}` and `${ALERT_EMAIL_TO}`, which Grafana -expands from the environment at startup. Supply them through a gitignored -env file: +The Slack webhook and email address are **not** hard-coded. `contactpoints.yaml` +ships deliberately unroutable placeholders — an `https://hooks.slack.invalid/…` +host and an `…@xrpld.invalid` address — which keep provisioning valid so the +stack boots with zero configuration while alerts route nowhere. + +To enable delivery, edit those two values **in place** with a real webhook and +address, and do not commit the result. ```bash cp docker/telemetry/.env.alerting.example docker/telemetry/.env.alerting -# edit .env.alerting — this file is gitignored, never commit the webhook/address +# edit .env.alerting — gitignored; holds the SMTP relay settings +$EDITOR docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml docker compose -f docker/telemetry/docker-compose.yml up -d grafana ``` -- **Slack** — set `SLACK_WEBHOOK_URL` to an incoming-webhook URL. Drives both - tiers. -- **Email** — set `ALERT_EMAIL_TO` (comma-separated) **and** point the - `GF_SMTP_*` vars at a real relay with `GF_SMTP_ENABLED=true`. Grafana can - only send mail once SMTP is configured. +- **Slack** — replace the placeholder `url:` with an incoming-webhook URL. Drives + both tiers. +- **Email** — replace the placeholder `addresses:` (comma- or semicolon-separated) + **and** point the `GF_SMTP_*` vars in `.env.alerting` at a real relay with + `GF_SMTP_ENABLED=true`. Grafana can only send mail once SMTP is configured. -Any variable left blank disables that path; the stack still runs. To add a -third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver to -the relevant contact point. +Three traps worth knowing before you edit this file: + +- **Do not substitute `${SLACK_WEBHOOK_URL}` / `${ALERT_EMAIL_TO}` here.** Grafana + expands `${VAR}` but does **not** support `${VAR:-default}`, so an unset variable + expands to empty, fails validation, and Grafana **exits 1** — taking the whole + telemetry stack down, not just alerting. A blank variable does not "disable that + path"; it breaks startup. +- **Never empty a `receivers:` list to disable a tier.** A contact point with no + receivers ceases to exist, the policy tree then references a missing receiver, + and Grafana refuses to boot. Point the route at a contact point that still + exists instead. +- **File provisioning is upsert-only.** Deleting a receiver from the YAML does not + remove it from an instance that already booted with it — the old receiver keeps + delivering. Removal needs an explicit `deleteContactPoints:` block listing the + uid (a commented example sits at the bottom of `contactpoints.yaml`). + +To add a third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver +to the relevant contact point. + +#### Deploying alerts to Grafana Cloud + +Grafana Cloud has **no provisioning filesystem**, so these `apiVersion: 1` files +cannot be loaded there. Cloud deployment goes through the REST API via +`docker/telemetry/upload_alerts_to_grafana.py`, which reads the same tracked +`rules.yaml` as the single source of truth (so local and Cloud cannot drift) and +applies the Cloud-specific transforms: the local `prometheus` datasource uid is +swapped for the Cloud one, the `folder:` _name_ becomes an existing `folderUID`, +and `interval` becomes integer seconds. + +```bash +cd docker/telemetry +python3 upload_alerts_to_grafana.py --dry-run # always dry-run first +python3 upload_alerts_to_grafana.py # create rules, paused +python3 upload_alerts_to_grafana.py --verify # read back what is deployed +``` + +Credentials come from `.env.grafanaserviceapi` (gitignored, a service-account +token with `alert.rules:write`); the recipient address comes from `ALERT_EMAIL_TO` +in `.env.alerting`. Neither is ever written to a tracked file. Use +`--no-delivery` to land the rules before a recipient is chosen, and `--activate` +only once the thresholds have been checked against the target fleet's baseline. + +> **The Cloud notification policy tree must not be pushed.** There is exactly one +> policy tree per org and the PUT endpoint **replaces it wholesale**. On a shared +> stack the root receiver and its sibling routes belong to other teams, so pushing +> an xrpld-shaped tree would silently re-route their alerts. The uploader +> therefore never touches the tree; instead each rule carries +> `notification_settings.receiver`, which routes that rule directly to the xrpld +> contact point and bypasses the tree entirely. Verify with a before/after hash of +> `GET /api/v1/provisioning/policies`. ### Verifying alert provisioning loaded After the stack is up: ```bash -# All six rules present? -curl -s http://localhost:3000/api/v1/provisioning/alert-rules | jq '.[].title' +# All thirteen rules present, and is each one paused? +curl -s http://localhost:3000/api/v1/provisioning/alert-rules | + jq -r '.[] | "\(.title)\tpaused=\(.isPaused)"' # Contact points present? curl -s http://localhost:3000/api/v1/provisioning/contact-points | jq '.[].name' ``` +Check `paused=true` explicitly rather than assuming it: a mis-spelled +`is_paused` is dropped without any error and the rule provisions **live**. + Grafana logs a provisioning error and skips the file if the YAML is malformed: ```bash docker compose -f docker/telemetry/docker-compose.yml logs grafana | grep -i alerting ``` +A malformed _expression_ fails differently and more quietly — the rule loads but +every evaluation errors. After a threshold or expr change, confirm each rule's +query still returns data: + +```bash +# Should print a numeric value per node, and no empty results +curl -sG http://localhost:9090/api/v1/query \ + --data-urlencode 'query=sum by (service_instance_id) (rate(ledgers_closed_total{service_name="xrpld"}[5m]))' | + jq '.data.result | length' +``` + ## Log-Trace Correlation When xrpld is built with `telemetry=ON`, log lines emitted within an active OpenTelemetry span automatically include `trace_id` and `span_id` fields: @@ -2204,6 +2387,12 @@ The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`). +For log-derived panels built on these queries, see the +[Log-Derived Insights](#log-derived-insights-log-derived-insights) dashboard and its +LogQL trap list — `partition`, `severity`, and `xrpl_network_type` are +**structured metadata**, not stream labels, so they must be filtered with `|` +after the selector and cannot be discovered by `label_values()`. + ```logql # Find all logs for a specific trace {service_name="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" @@ -2211,11 +2400,13 @@ The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrp # Error logs with trace context (log lines with ERR severity that have a trace_id) {service_name="xrpld"} |= "ERR" |= "trace_id=" -# All logs from a specific partition that were emitted during a span -{service_name="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" +# All logs from a specific partition that were emitted during a span. +# Prefer the structured-metadata filter over a line match: `|= "LedgerMaster"` +# also matches the substring anywhere in the message body. +{service_name="xrpld"} | partition = `LedgerMaster` | trace_id != "" # Logs from a specific subsystem during a span (e.g. LedgerConsensus) -{service_name="xrpld"} |= "LedgerConsensus" |= "trace_id=" +{service_name="xrpld"} | partition = `LedgerConsensus` | trace_id != "" # Logs from the last hour containing trace context {service_name="xrpld"} |= "trace_id=" | regexp `(?P\S+):(?P\S+)\s+trace_id=(?P[a-f0-9]+)` @@ -2232,6 +2423,120 @@ count_over_time({service_name="xrpld"} |= "trace_id=" [5m]) 4. Open Grafana at http://localhost:3000 -> Explore -> Loki and search for `{service_name="xrpld"} |= "trace_id="`. 5. Click the TraceID link to navigate to the corresponding trace in Tempo. +### Log-Derived Insights (`log-derived-insights`) + +The only **Loki/LogQL** dashboard. It surfaces detail that no metric or span +records, by parsing `debug.log` text. 41 panels in 10 rows: 8 stat, 18 +timeseries, 2 table, 1 state-timeline, 1 logs, 1 text, across 35 queries. + +> **REQUIRES DEBUG LOGS for most rows.** xrpld's default threshold is `Info` +> (`Severity thresh = Severity::Info`, `app/main/Main.cpp`). Rows tagged `[DBG]` +> read `DBG`-severity lines that a default node never writes, so those panels are +> **empty** on an unmodified node — and an empty panel means _not collecting_, not +> _no problem_. Rows tagged `[DEFAULT OK]` work as shipped. +> +> Enable per partition rather than globally (`Resource` alone emits ~329k +> lines/6h): +> +> ``` +> log_level ManifestCache debug +> log_level Resource debug +> log_level InboundLedger debug +> log_level Peer debug +> log_level PeerFinder debug +> ``` +> +> Those five cover every `[DBG]` row. The `[MIXED]` stat row additionally +> reads `LedgerConsensus` and `LoadMonitor`, both of which already emit at +> the default level, so its error/consensus/breach/sync panels populate +> without any change — only its manifest, fee, and fetch-waste panels need +> debug enabled. + +| Row | Gate | Key panels | +| --------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Worst Offenders — Node Ranking | `[MIXED]` | 8 stat panels ranking nodes by error volume, attack-like input, total fee charged, manifest rejections, consensus problems, job latency breaches, sync instability, and ledger fetch waste | +| Node Operating State Transitions | `[DEFAULT OK]` | Transition rate and state timeline from `STATE->` (`NetworkOPsImp::setMode`, info) | +| Log Volume & Severity Mix | `[DEFAULT OK]` | Line rate by severity; top-N partitions by rate | +| Manifests — Disposition & Producers | `[DBG]` | Disposition rate; accept-vs-reject; top-N master keys | +| Resource Fee Charges — Load Attribution | `[DBG]` | Charge rate by reason; fee-weighted load; top-N peers by IP and public key | +| Ledger Acquisition Efficiency | `[DBG]` | Duplicate ratio; good vs duplicate vs timeout | +| Peer Lifecycle & Disconnects | `[DBG]` | Disconnect reason breakdown; handshake and accept rate | +| Consensus Phase & Mode | `[DEFAULT OK]` | Phase transitions; operating-mode proxy; quorum and trusted-set size | +| Slow Job Latency Breaches | `[DEFAULT OK]` | Run p99, wait p99, breach rate by job (`LoadMonitor`, >500ms only) | +| Error & Warning Stream | `[DEFAULT OK]` | WRN/ERR/FTL rate by partition; live log tail | + +Filters: `$service_name`, `$deployment_environment`, `$node`, +`$xrpl_network_type`, `$severity`, plus log-derived `$consensus_phase`, +`$consensus_mode`, `$manifest_action`, `$charge_reason`, and `$topn`. + +#### LogQL traps this dashboard exposed + +Ten mistakes that fail **silently** — each cost a debugging cycle, so check them +before adding any LogQL panel. + +1. **`partition` is structured metadata, not a stream label.** + `{service_name="xrpld", partition="ManifestCache"}` returns **zero rows with + no error**. Correct form: `{service_name="xrpld"} | partition = \`ManifestCache\``. +Stream labels are only `service_name`, `service_instance_id`, +`deployment_environment`. Everything else — `partition`, `severity`, +`xrpl_network_type`, `message`, `trace_id` — is structured metadata. + +2. **`label_values()` cannot see structured metadata.** A `query`-type template + variable over `xrpl_network_type`, `severity`, or `partition` returns an empty + dropdown; only true stream labels populate. Use a `custom` variable with + enumerated values instead. This is why filters appeared blank. + +3. **A target with no datasource `uid` resolves to the DEFAULT datasource.** + The Prometheus dashboards use `{"type": "prometheus"}` with no uid and work + only because Prometheus _is_ the default. A Loki target written the same way + sends LogQL to Prometheus and returns nothing. Always pin + `{"type": "loki", "uid": "${DS_LOKI}"}`. + +4. **`$__rate_interval` is Prometheus-only — Loki panels must use `[$__auto]`.** + Grafana does not substitute `$__rate_interval` for a Loki target, so Loki + receives the literal string and fails with + `parse error: not a valid duration string: "$__rate_interval"`, which surfaces + as "No data". The other 14 dashboards all use `$__rate_interval` because they + are Prometheus-backed; do **not** align LogQL panels to that convention. + +5. **Loki caps a query at 2000 series.** Any per-key or per-IP aggregation must be + wrapped in `topk(N, ...)` or it fails with HTTP 400. A true distinct-key count + over a large key space is therefore not possible in a panel. + +6. **Loki tables need `labelsToFields` plus `reduce`.** Loki attaches labels to + the Value field instead of returning columns, so a table panel renders bare + Time/Value without `labelsToFields`. Grafana also runs a Loki table target as a + **range** query even when `instant: true` is set, producing one row per series + _per timestamp_ — visible as the same key repeated many times. Use + `reduce(lastNotNull, labelsToFields)` then `organize`, and note the value + column is then named `Last *`, which any field override must match. + +7. **Title Case legends need `label_format`, not value mappings.** A label-driven + legend renders the raw log value (`full`, `moderate peer request`). Grafana + value mappings do not help — they map the metric _value_, not label text in + `displayName`. Rewrite the label in the query: + `| label_format state=\`{{if eq .state "full"}}Full{{else}}{{.state}}{{end}}\``. + +8. **`unwrap` must be the last pipeline stage.** Any label filter or + `label_format` placed after `| unwrap ` makes the query invalid and it + returns zero frames. + +9. **Non-matching lines yield an empty label.** A line in the selected partition + that does not match the panel's `regexp` still passes through with an empty + extracted label, which renders as a blank legend entry. Guard with + `|