Merge branch 'pratik/otel-phase5-docs-deployment' into pratik/otel-phase6-statsd

# Conflicts:
#	src/libxrpl/telemetry/SpanGuard.cpp

docker/telemetry/grafana/dashboards/consensus-health.json

@@ -1036,7 +1036,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
         "type": "query",
-        "query": "label_values(traces_span_metrics_calls_total, service_instance_id)",
+        "query": "label_values(target_info, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "prometheus"

docker/telemetry/grafana/dashboards/ledger-operations.json

@@ -387,7 +387,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
         "type": "query",
-        "query": "label_values(traces_span_metrics_calls_total, service_instance_id)",
+        "query": "label_values(target_info, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "prometheus"

docker/telemetry/grafana/dashboards/peer-network.json

@@ -221,7 +221,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
         "type": "query",
-        "query": "label_values(traces_span_metrics_calls_total, service_instance_id)",
+        "query": "label_values(target_info, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "prometheus"

docker/telemetry/grafana/dashboards/rpc-performance.json

@@ -486,7 +486,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
         "type": "query",
-        "query": "label_values(traces_span_metrics_calls_total, service_instance_id)",
+        "query": "label_values(target_info, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "prometheus"

docker/telemetry/grafana/dashboards/transaction-overview.json

@@ -428,9 +428,9 @@
       }
     },
     {
-      "title": "TxQ Accept Status",
-      "description": "**What:** Outcomes when queued transactions are considered for a ledger: applied (included), failed (removed), or retried (kept for the next round).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** Applied should dominate when the queue is draining healthily.\n**Healthy range:** Mostly applied; workload-dependent.\n**Watch for:** A rising retried or failed share signals queue pressure and fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept",
-      "type": "piechart",
+      "title": "TxQ Accept: Applied Ratio per Node (State Timeline)",
+      "description": "**What:** Applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.\n**How it's computed:** Per node, applied accepts divided by applied-plus-failed accepts over a 5-minute window.\n**Reading it:** Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.\n**Healthy range:** At or near 100% applied when the queue is draining healthily; workload-dependent.\n**Watch for:** A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept",
+      "type": "state-timeline",
       "gridPos": {
         "h": 8,
         "w": 8,
@@ -438,13 +438,17 @@
         "y": 40
       },
       "options": {
+        "mergeValues": true,
+        "showValue": "auto",
+        "alignValue": "center",
+        "rowHeight": 0.85,
         "legend": {
-          "displayMode": "table",
-          "placement": "right",
-          "values": ["value", "percent"]
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
         "tooltip": {
-          "mode": "multi"
+          "mode": "single"
         }
       },
       "targets": [
@@ -452,13 +456,30 @@
           "datasource": {
             "type": "prometheus"
           },
-          "expr": "sum by (txq_status) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=~\"$txq_status\"}[5m]))",
-          "legendFormat": "{{txq_status}}"
+          "expr": "sum by (service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[5m]))\n/\nsum by (service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[5m]))",
+          "legendFormat": "{{service_instance_id}}"
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "unit": "short"
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": 0 },
+              { "color": "yellow", "value": 0.7 },
+              { "color": "green", "value": 0.9 }
+            ]
+          },
+          "custom": {
+            "fillOpacity": 80,
+            "lineWidth": 0
+          }
         },
         "overrides": []
       }
@@ -916,7 +937,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
         "type": "query",
-        "query": "label_values(traces_span_metrics_calls_total, service_instance_id)",
+        "query": "label_values(target_info, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "prometheus"
@@ -987,24 +1008,6 @@
         "sort": 1,
         "label": "Result Code"
       },
-      {
-        "name": "txq_status",
-        "type": "query",
-        "datasource": {
-          "type": "prometheus"
-        },
-        "query": "label_values(traces_span_metrics_calls_total{span_name=\"txq.accept_tx\", txq_status!=\"\"}, txq_status)",
-        "refresh": 2,
-        "includeAll": true,
-        "multi": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "sort": 1,
-        "label": "Queue Status"
-      },
       {
         "name": "stage",
         "type": "query",

docker/telemetry/grafana/dashboards/validate_dashboards.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Dashboard lint: cumulative metrics must be rate()-wrapped; tier filters present."""
+
+import json, re, sys
+
+# Prometheus gauges that hold a CUMULATIVE total -> must be rate()/increase()-wrapped.
+CUMULATIVE_PREFIXES = (
+    "total_bytes_",
+    "total_messages_",
+    "transactions_messages_",
+    "transactions_duplicate_messages_",
+    "proposals_",
+    "validations_",
+    "overlay_peer_disconnects",
+    "squelch_",
+    "overhead_",
+    "validator_lists_",
+    "set_get_",
+    "set_share_",
+    "have_transactions_",
+    "requested_transactions_",
+    "proof_path_",
+    "replay_delta_",
+    "ledger_data_",
+    "ledger_get_",
+    "ledger_share_",
+    "ledger_transaction_",
+    "ledger_account_state_",
+    "getobject_",
+    "jq_trans_overflow_total",
+)
+# nodestore_state{metric=...} cumulative sub-series (raw only inside these metric= labels).
+NODESTORE_CUMULATIVE = (
+    "node_reads_total",
+    "node_reads_hit",
+    "node_writes",
+    "node_read_bytes",
+    "node_written_bytes",
+    "node_reads_duration_us",
+)
+# state_accounting_*_duration are cumulative µs.
+STATE_DURATION = re.compile(r"state_accounting_\w+_duration")
+REQUIRED_FILTERS = (
+    "$node",
+    "$deployment_environment",
+    "$xrpl_network_type",
+    "$service_name",
+)
+
+
+def iter_panels(dash):
+    for p in dash.get("panels", []):
+        yield p
+        for sub in p.get("panels", []) or []:
+            yield sub
+
+
+def expr_is_wrapped(expr):
+    return (
+        "rate(" in expr
+        or "increase(" in expr
+        or "irate(" in expr
+        or "histogram_quantile(" in expr
+    )
+
+
+def check(path, forbid_5m):
+    errs = []
+    try:
+        dash = json.load(open(path))
+    except Exception as e:
+        return [f"{path}: INVALID JSON: {e}"]
+    for p in iter_panels(dash):
+        title = p.get("title", "<untitled>")
+        for tg in p.get("targets", []) or []:
+            expr = (tg.get("expr") or "").strip()
+            if not expr:
+                continue
+            # tier filters
+            if "{" in expr:
+                for f in REQUIRED_FILTERS:
+                    if f not in expr:
+                        errs.append(f"{path} [{title}]: missing {f} in expr")
+            # cumulative metric plotted raw?
+            hits = [m for m in CUMULATIVE_PREFIXES if m in expr]
+            nod = [m for m in NODESTORE_CUMULATIVE if f'"{m}"' in expr]
+            statedur = STATE_DURATION.search(expr)
+            if (hits or nod or statedur) and not expr_is_wrapped(expr):
+                who = hits or nod or [statedur.group(0)]
+                errs.append(
+                    f"{path} [{title}]: cumulative metric {who} plotted RAW (needs rate/increase)"
+                )
+            if "deriv(" in expr or "idelta(" in expr:
+                errs.append(
+                    f"{path} [{title}]: uses deriv()/idelta() on a cumulative series"
+                )
+            if forbid_5m and ("[5m]" in expr or "[1h]" in expr):
+                # Only a fixed [5m]/[1h] inside a top-level rate()/increase()/irate() on a
+                # plain counter is a violation. Legitimate fixed-window exprs -- histogram
+                # buckets (rate(..._bucket[5m])), avg_over_time windows, and subqueries
+                # ([5m:]) -- are intentional and must not be flagged.
+                if not ("_bucket" in expr or "avg_over_time" in expr or ":]" in expr):
+                    errs.append(
+                        f"{path} [{title}]: hardcoded range window; use [$__rate_interval]"
+                    )
+        # unit check
+        unit = p.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
+        if unit == "mps":
+            errs.append(f"{path} [{title}]: invalid Grafana unit 'mps' (use 'cps')")
+    return errs
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith("--")]
+    forbid_5m = "--no-5m" in sys.argv
+    all_errs = []
+    for path in args:
+        all_errs += check(path, forbid_5m)
+    if all_errs:
+        print("\n".join(all_errs))
+        print(f"\nFAIL: {len(all_errs)} violation(s)")
+        sys.exit(1)
+    print(f"OK: {len(args)} dashboard(s) passed")
+
+
+if __name__ == "__main__":
+    main()

docs/telemetry-runbook.md

@@ -628,16 +628,17 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 
 ### Transaction Overview (`transaction-overview`)
 
-| Panel                             | Type       | PromQL                                                                               | Labels Used   |
-| --------------------------------- | ---------- | ------------------------------------------------------------------------------------ | ------------- |
-| Transaction Processing Rate       | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name`   |
-| Transaction Processing Latency    | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})`                      | —             |
-| Transaction Path Distribution     | piechart   | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local`       |
-| Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])`                  | —             |
-| TX Processing Duration Heatmap    | heatmap    | `tx.process` histogram buckets                                                       | `le`          |
-| TX Apply Duration per Ledger      | timeseries | p95/p50 of `tx.apply`                                                                | —             |
-| Peer TX Receive Rate              | timeseries | `tx.receive` rate                                                                    | —             |
-| TX Apply Failed Rate              | stat       | `tx.apply` with `STATUS_CODE_ERROR`                                                  | `status_code` |
+| Panel                              | Type           | PromQL                                                                                              | Labels Used                         |
+| ---------------------------------- | -------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------- |
+| Transaction Processing Rate        | timeseries     | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive`                | `span_name`                         |
+| Transaction Processing Latency     | timeseries     | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})`                                     | —                                   |
+| Transaction Path Distribution      | piechart       | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))`                | `local`                             |
+| Transaction Receive vs Suppressed  | timeseries     | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])`                                 | —                                   |
+| TX Processing Duration Heatmap     | heatmap        | `tx.process` histogram buckets                                                                      | `le`                                |
+| TX Apply Duration per Ledger       | timeseries     | p95/p50 of `tx.apply`                                                                               | —                                   |
+| Peer TX Receive Rate               | timeseries     | `tx.receive` rate                                                                                   | —                                   |
+| TX Apply Failed Rate               | stat           | `tx.apply` with `STATUS_CODE_ERROR`                                                                 | `status_code`                       |
+| TxQ Accept: Applied Ratio per Node | state-timeline | applied / (applied+failed) of `traces_span_metrics_calls_total{span_name="txq.accept_tx"}` per node | `txq_status`, `service_instance_id` |
 
 ### Consensus Health (`consensus-health`)
 

src/libxrpl/telemetry/SpanGuard.cpp

@@ -46,7 +46,6 @@
 #include <cstdint>
 #include <cstring>
 #include <exception>
-#include <format>
 #include <initializer_list>
 #include <memory>
 #include <string>
@@ -188,7 +187,9 @@ SpanGuard::span(TraceCategory cat, std::string_view prefix, std::string_view nam
     auto* tel = Telemetry::getInstance();
     if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat))
         return {};
-    auto fullName = std::format("{}.{}", prefix, name);
+    std::string fullName;
+    fullName.reserve(prefix.size() + 1 + name.size());
+    fullName.append(prefix).append(1, '.').append(name);
     return SpanGuard(std::make_unique<Impl>(tel->startSpan(fullName, categoryToSpanKind(cat))));
 }