diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy index 163e1f9f19..278f866835 100644 --- a/docker/telemetry/alloy/config.alloy +++ b/docker/telemetry/alloy/config.alloy @@ -349,7 +349,6 @@ otelcol.connector.spanmetrics "xrpld" { // Overlay / peer dimensions. dimension { name = "local" } - dimension { name = "suppressed" } dimension { name = "proposal_trusted" } dimension { name = "validation_trusted" } diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index c637f172e6..71e8d79ca8 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -273,8 +273,8 @@ "id": 5 }, { - "title": "Transaction Receive vs Suppressed", - "description": "###### What this is:\n*Raw transactions received from peers, split by whether they were duplicates suppressed before processing.*\n\n###### How it's computed:\n*Per-second rate of received transactions over the window, split by suppressed flag and node.*\n\n###### Reading it:\n*A large suppressed share is normal, since the same transaction arrives from many peers.*\n\n###### Healthy range:\n*Workload-dependent; suppressed typically exceeds newly processed.*\n\n###### Watch for:\n*A collapse in suppression (duplicate filtering failing) or an overall receive flood.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::handleTransaction`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", + "title": "Transaction Receive Rate", + "description": "###### What this is:\n*Transactions received from peers that this node went on to process.*\n\n###### How it's computed:\n*Per-second rate of the tx.receive span over the window, per node.*\n\n###### Reading it:\n*Tracks the useful inbound transaction load. Copies dropped as duplicates are not counted, because the span starts after the duplicate check.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A fall to zero while peers are connected, meaning inbound relay has stopped.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer. Suppressed copies are excluded from this panel; their volume is the transactions_duplicate traffic category.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::handleTransaction`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", "type": "timeseries", "gridPos": { "h": 10, @@ -295,7 +295,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (suppressed, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{span_name=\"tx.receive\", tx_type=~\"$tx_type\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Suppressed $1\", \"suppressed\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{span_name=\"tx.receive\", tx_type=~\"$tx_type\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml index f89fa89eec..3b9862f722 100644 --- a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -206,11 +206,6 @@ datasources: operator: "=" scope: span type: dynamic - - id: tx-suppressed - tag: suppressed - operator: "=" - scope: span - type: dynamic - id: peer-version tag: peer_version operator: "=" diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 4f5c1b56e7..6ebdc81610 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -232,7 +232,6 @@ connectors: - name: close_time_correct - name: consensus_state - name: local - - name: suppressed - name: proposal_trusted - name: validation_trusted - name: tx_type diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 3fdb624e6c..cb6b580aff 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -214,7 +214,6 @@ connectors: - name: close_time_correct - name: consensus_state - name: local - - name: suppressed - name: proposal_trusted - name: validation_trusted - name: tx_type diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index aee41551e7..f4ce67b285 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -26,8 +26,7 @@ "stage", "txq_status", "close_time_correct", - "consensus_state", - "suppressed" + "consensus_state" ], "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)." }, diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 5e7f60f13b..cdbe3f498e 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -274,14 +274,14 @@ this span: count successes as total minus error, or filter on `status_code`. ### Transaction Spans -| Span Name | Source File | Attributes | Description | -| --------------- | --------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------- | -| `tx.process` | NetworkOPs.cpp | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied`, `current_ledger_seq` | Transaction submission and processing | -| `tx.receive` | PeerImp.cpp | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status`, `current_ledger_seq` | Transaction received from peer relay | -| `tx.apply` | BuildLedger.cpp | `tx_count`, `tx_failed` | Transaction set applied per ledger | -| `tx.preflight` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Stateless checks stage | -| `tx.preclaim` | applySteps.cpp | `stage`, `tx_type`, `ter_result`, `current_ledger_seq`, `current_ledger_hash` | Ledger-aware checks stage | -| `tx.transactor` | Transactor.cpp | `stage`, `tx_type`, `ter_result`, `applied`, `current_ledger_seq`, `current_ledger_hash` | Apply stage (transactor runs) | +| Span Name | Source File | Attributes | Description | +| --------------- | --------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------ | +| `tx.process` | NetworkOPs.cpp | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied`, `current_ledger_seq` | Transaction submission and processing | +| `tx.receive` | PeerImp.cpp | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `tx_status`, `current_ledger_seq` | Transaction this node will process, received from peer relay | +| `tx.apply` | BuildLedger.cpp | `tx_count`, `tx_failed` | Transaction set applied per ledger | +| `tx.preflight` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Stateless checks stage | +| `tx.preclaim` | applySteps.cpp | `stage`, `tx_type`, `ter_result`, `current_ledger_seq`, `current_ledger_hash` | Ledger-aware checks stage | +| `tx.transactor` | Transactor.cpp | `stage`, `tx_type`, `ter_result`, `applied`, `current_ledger_seq`, `current_ledger_hash` | Apply stage (transactor runs) | The three apply-pipeline spans (`tx.preflight`, `tx.preclaim`, `tx.transactor`) share a deterministic `trace_id` from `txID[0:16]`, so they group under one @@ -595,7 +595,7 @@ flowchart TB RELAYOUT(["Overlay::relay fan-out to N peers
(no span; if applied / terQUEUED,
shouldRelay, not tfInnerBatchTxn)"]):::plain PREDROP(["Diverged / needNetworkLedger
(no span — dropped before tx.receive)"]):::drop RCV["tx.receive
(peer TMTransaction in)"]:::span - RCVDROP["tx.receive
tx_status = rejected_inner_batch /
suppressed / dropped_no_sync /
dropped_queue_full"]:::drop + RCVDROP["tx.receive
tx_status = dropped_no_sync /
dropped_queue_full"]:::drop CHK(["checkTransaction
(JtTransaction worker, no span)"]):::plain PRELAY_IN(["TMTransaction in (no span)"]):::plain