diff --git a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json index 9d61fef2bf..25d537d98c 100644 --- a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json @@ -30,56 +30,56 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_get_Bytes_In", + "expr": "rippled_ledger_data_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Data Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_share_Bytes_In", + "expr": "rippled_ledger_data_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Data Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Transaction_Set_candidate_get_Bytes_In", + "expr": "rippled_ledger_data_Transaction_Set_candidate_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Set Candidate Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Transaction_Set_candidate_share_Bytes_In", + "expr": "rippled_ledger_data_Transaction_Set_candidate_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Set Candidate Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Transaction_Node_get_Bytes_In", + "expr": "rippled_ledger_data_Transaction_Node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Transaction_Node_share_Bytes_In", + "expr": "rippled_ledger_data_Transaction_Node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Account_State_Node_get_Bytes_In", + "expr": "rippled_ledger_data_Account_State_Node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Node Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_data_Account_State_Node_share_Bytes_In", + "expr": "rippled_ledger_data_Account_State_Node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Node Share" } ], @@ -118,56 +118,56 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_share_Bytes_In", + "expr": "rippled_ledger_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Share In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_get_Bytes_In", + "expr": "rippled_ledger_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Get In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Transaction_Set_candidate_share_Bytes_In", + "expr": "rippled_ledger_Transaction_Set_candidate_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Set Candidate Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Transaction_Set_candidate_get_Bytes_In", + "expr": "rippled_ledger_Transaction_Set_candidate_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Set Candidate Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Transaction_node_share_Bytes_In", + "expr": "rippled_ledger_Transaction_node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Transaction_node_get_Bytes_In", + "expr": "rippled_ledger_Transaction_node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Account_State_node_share_Bytes_In", + "expr": "rippled_ledger_Account_State_node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ledger_Account_State_node_get_Bytes_In", + "expr": "rippled_ledger_Account_State_node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Get" } ], @@ -206,56 +206,56 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Ledger_get_Bytes_In", + "expr": "rippled_getobject_Ledger_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Ledger_share_Bytes_In", + "expr": "rippled_getobject_Ledger_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_get_Bytes_In", + "expr": "rippled_getobject_Transaction_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Transaction Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_share_Bytes_In", + "expr": "rippled_getobject_Transaction_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Transaction Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_node_get_Bytes_In", + "expr": "rippled_getobject_Transaction_node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_node_share_Bytes_In", + "expr": "rippled_getobject_Transaction_node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Account_State_node_get_Bytes_In", + "expr": "rippled_getobject_Account_State_node_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Account_State_node_share_Bytes_In", + "expr": "rippled_getobject_Account_State_node_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Share" } ], @@ -294,49 +294,49 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_CAS_get_Bytes_In", + "expr": "rippled_getobject_CAS_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "CAS Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_CAS_share_Bytes_In", + "expr": "rippled_getobject_CAS_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "CAS Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Fetch_Pack_share_Bytes_In", + "expr": "rippled_getobject_Fetch_Pack_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Fetch Pack Share" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Fetch_Pack_get_Bytes_In", + "expr": "rippled_getobject_Fetch_Pack_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Fetch Pack Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transactions_get_Bytes_In", + "expr": "rippled_getobject_Transactions_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Transactions Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_get_Bytes_In", + "expr": "rippled_getobject_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Aggregate Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_share_Bytes_In", + "expr": "rippled_getobject_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Aggregate Share" } ], @@ -375,49 +375,49 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Ledger_get_Messages_In", + "expr": "rippled_getobject_Ledger_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Ledger Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_get_Messages_In", + "expr": "rippled_getobject_Transaction_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Transaction Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transaction_node_get_Messages_In", + "expr": "rippled_getobject_Transaction_node_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "TX Node Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Account_State_node_get_Messages_In", + "expr": "rippled_getobject_Account_State_node_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Account State Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_CAS_get_Messages_In", + "expr": "rippled_getobject_CAS_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "CAS Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Fetch_Pack_get_Messages_In", + "expr": "rippled_getobject_Fetch_Pack_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Fetch Pack Get" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_getobject_Transactions_get_Messages_In", + "expr": "rippled_getobject_Transactions_get_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Transactions Get" } ], @@ -463,7 +463,7 @@ "datasource": { "type": "prometheus" }, - "expr": "topk(20, {__name__=~\"rippled_.*_Bytes_In\", __name__!~\"rippled_total_.*\"})", + "expr": "topk(20, {exported_instance=~\"$node\", __name__=~\"rippled_.*_Bytes_In\", __name__!~\"rippled_total_.*\"})", "legendFormat": "{{__name__}}" } ], @@ -495,7 +495,28 @@ "schemaVersion": 39, "tags": ["rippled", "statsd", "ledger", "sync", "telemetry"], "templating": { - "list": [] + "list": [ + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(exported_instance)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] }, "time": { "from": "now-1h", diff --git a/docker/telemetry/grafana/dashboards/system-network-traffic.json b/docker/telemetry/grafana/dashboards/system-network-traffic.json index 193a553b72..e9dc47f10f 100644 --- a/docker/telemetry/grafana/dashboards/system-network-traffic.json +++ b/docker/telemetry/grafana/dashboards/system-network-traffic.json @@ -30,14 +30,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_Peer_Finder_Active_Inbound_Peers", + "expr": "rippled_Peer_Finder_Active_Inbound_Peers{exported_instance=~\"$node\"}", "legendFormat": "Inbound Peers" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_Peer_Finder_Active_Outbound_Peers", + "expr": "rippled_Peer_Finder_Active_Outbound_Peers{exported_instance=~\"$node\"}", "legendFormat": "Outbound Peers" } ], @@ -76,7 +76,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_Overlay_Peer_Disconnects", + "expr": "rippled_Overlay_Peer_Disconnects{exported_instance=~\"$node\"}", "legendFormat": "Disconnects" } ], @@ -115,14 +115,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_total_Bytes_In", + "expr": "rippled_total_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_total_Bytes_Out", + "expr": "rippled_total_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Bytes Out" } ], @@ -161,14 +161,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_total_Messages_In", + "expr": "rippled_total_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Messages In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_total_Messages_Out", + "expr": "rippled_total_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Messages Out" } ], @@ -207,21 +207,21 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_transactions_Messages_In", + "expr": "rippled_transactions_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "TX Messages In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_transactions_Messages_Out", + "expr": "rippled_transactions_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "TX Messages Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_transactions_duplicate_Messages_In", + "expr": "rippled_transactions_duplicate_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "TX Duplicate In" } ], @@ -260,28 +260,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_proposals_Messages_In", + "expr": "rippled_proposals_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Proposals In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proposals_Messages_Out", + "expr": "rippled_proposals_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Proposals Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proposals_untrusted_Messages_In", + "expr": "rippled_proposals_untrusted_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Untrusted In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proposals_duplicate_Messages_In", + "expr": "rippled_proposals_duplicate_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Duplicate In" } ], @@ -320,28 +320,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_validations_Messages_In", + "expr": "rippled_validations_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Validations In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validations_Messages_Out", + "expr": "rippled_validations_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Validations Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validations_untrusted_Messages_In", + "expr": "rippled_validations_untrusted_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Untrusted In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validations_duplicate_Messages_In", + "expr": "rippled_validations_duplicate_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Duplicate In" } ], @@ -380,7 +380,7 @@ "datasource": { "type": "prometheus" }, - "expr": "topk(10, {__name__=~\"rippled_.*_Bytes_In\", __name__!~\"rippled_total_.*\"})", + "expr": "topk(10, {exported_instance=~\"$node\", __name__=~\"rippled_.*_Bytes_In\", __name__!~\"rippled_total_.*\"})", "legendFormat": "{{__name__}}" } ], @@ -660,7 +660,28 @@ "schemaVersion": 39, "tags": ["rippled", "statsd", "network", "telemetry"], "templating": { - "list": [] + "list": [ + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(exported_instance)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] }, "time": { "from": "now-1h", diff --git a/docker/telemetry/grafana/dashboards/system-node-health.json b/docker/telemetry/grafana/dashboards/system-node-health.json index dc7f404889..49ac731528 100644 --- a/docker/telemetry/grafana/dashboards/system-node-health.json +++ b/docker/telemetry/grafana/dashboards/system-node-health.json @@ -30,7 +30,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_LedgerMaster_Validated_Ledger_Age", + "expr": "rippled_LedgerMaster_Validated_Ledger_Age{exported_instance=~\"$node\"}", "legendFormat": "Validated Age" } ], @@ -78,7 +78,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_LedgerMaster_Published_Ledger_Age", + "expr": "rippled_LedgerMaster_Published_Ledger_Age{exported_instance=~\"$node\"}", "legendFormat": "Published Age" } ], @@ -126,35 +126,35 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Full_duration", + "expr": "rippled_State_Accounting_Full_duration{exported_instance=~\"$node\"}", "legendFormat": "Full" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Tracking_duration", + "expr": "rippled_State_Accounting_Tracking_duration{exported_instance=~\"$node\"}", "legendFormat": "Tracking" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Syncing_duration", + "expr": "rippled_State_Accounting_Syncing_duration{exported_instance=~\"$node\"}", "legendFormat": "Syncing" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Connected_duration", + "expr": "rippled_State_Accounting_Connected_duration{exported_instance=~\"$node\"}", "legendFormat": "Connected" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Disconnected_duration", + "expr": "rippled_State_Accounting_Disconnected_duration{exported_instance=~\"$node\"}", "legendFormat": "Disconnected" } ], @@ -193,35 +193,35 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Full_transitions", + "expr": "rippled_State_Accounting_Full_transitions{exported_instance=~\"$node\"}", "legendFormat": "Full" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Tracking_transitions", + "expr": "rippled_State_Accounting_Tracking_transitions{exported_instance=~\"$node\"}", "legendFormat": "Tracking" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Syncing_transitions", + "expr": "rippled_State_Accounting_Syncing_transitions{exported_instance=~\"$node\"}", "legendFormat": "Syncing" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Connected_transitions", + "expr": "rippled_State_Accounting_Connected_transitions{exported_instance=~\"$node\"}", "legendFormat": "Connected" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_State_Accounting_Disconnected_transitions", + "expr": "rippled_State_Accounting_Disconnected_transitions{exported_instance=~\"$node\"}", "legendFormat": "Disconnected" } ], @@ -260,14 +260,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_ios_latency{quantile=\"0.95\"}", + "expr": "rippled_ios_latency{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95 I/O Latency" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_ios_latency{quantile=\"0.5\"}", + "expr": "rippled_ios_latency{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50 I/O Latency" } ], @@ -306,7 +306,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_job_count", + "expr": "rippled_job_count{exported_instance=~\"$node\"}", "legendFormat": "Job Queue Depth" } ], @@ -345,7 +345,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(rippled_ledger_fetches_total[5m])", + "expr": "rate(rippled_ledger_fetches_total{exported_instance=~\"$node\"}[5m])", "legendFormat": "Fetches / Sec" } ], @@ -377,7 +377,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(rippled_ledger_history_mismatch_total[5m])", + "expr": "rate(rippled_ledger_history_mismatch_total{exported_instance=~\"$node\"}[5m])", "legendFormat": "Mismatches / Sec" } ], @@ -404,7 +404,28 @@ "schemaVersion": 39, "tags": ["rippled", "statsd", "node-health", "telemetry"], "templating": { - "list": [] + "list": [ + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(exported_instance)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] }, "time": { "from": "now-1h", diff --git a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json index b1a20f3740..aa4c32b97c 100644 --- a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json @@ -30,42 +30,42 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_Messages_In", + "expr": "rippled_squelch_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Squelch In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_Messages_Out", + "expr": "rippled_squelch_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Squelch Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_suppressed_Messages_In", + "expr": "rippled_squelch_suppressed_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Suppressed In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_suppressed_Messages_Out", + "expr": "rippled_squelch_suppressed_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Suppressed Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_ignored_Messages_In", + "expr": "rippled_squelch_ignored_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Ignored In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_squelch_ignored_Messages_Out", + "expr": "rippled_squelch_ignored_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Ignored Out" } ], @@ -104,42 +104,42 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_Bytes_In", + "expr": "rippled_overhead_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Base Overhead In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_Bytes_Out", + "expr": "rippled_overhead_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Base Overhead Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_cluster_Bytes_In", + "expr": "rippled_overhead_cluster_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Cluster In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_cluster_Bytes_Out", + "expr": "rippled_overhead_cluster_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Cluster Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_manifest_Bytes_In", + "expr": "rippled_overhead_manifest_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Manifest In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_overhead_manifest_Bytes_Out", + "expr": "rippled_overhead_manifest_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Manifest Out" } ], @@ -178,28 +178,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_validator_lists_Bytes_In", + "expr": "rippled_validator_lists_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validator_lists_Bytes_Out", + "expr": "rippled_validator_lists_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Bytes Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validator_lists_Messages_In", + "expr": "rippled_validator_lists_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Messages In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_validator_lists_Messages_Out", + "expr": "rippled_validator_lists_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Messages Out" } ], @@ -255,28 +255,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_set_get_Bytes_In", + "expr": "rippled_set_get_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Set Get In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_set_get_Bytes_Out", + "expr": "rippled_set_get_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Set Get Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_set_share_Bytes_In", + "expr": "rippled_set_share_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Set Share In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_set_share_Bytes_Out", + "expr": "rippled_set_share_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Set Share Out" } ], @@ -315,28 +315,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_have_transactions_Messages_In", + "expr": "rippled_have_transactions_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Have TX In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_have_transactions_Messages_Out", + "expr": "rippled_have_transactions_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Have TX Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_requested_transactions_Messages_In", + "expr": "rippled_requested_transactions_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Requested TX In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_requested_transactions_Messages_Out", + "expr": "rippled_requested_transactions_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Requested TX Out" } ], @@ -375,28 +375,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_unknown_Bytes_In", + "expr": "rippled_unknown_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Unknown Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_unknown_Bytes_Out", + "expr": "rippled_unknown_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Unknown Bytes Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_unknown_Messages_In", + "expr": "rippled_unknown_Messages_In{exported_instance=~\"$node\"}", "legendFormat": "Unknown Messages In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_unknown_Messages_Out", + "expr": "rippled_unknown_Messages_Out{exported_instance=~\"$node\"}", "legendFormat": "Unknown Messages Out" } ], @@ -452,28 +452,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_proof_path_request_Bytes_In", + "expr": "rippled_proof_path_request_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Request Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proof_path_request_Bytes_Out", + "expr": "rippled_proof_path_request_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Request Bytes Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proof_path_response_Bytes_In", + "expr": "rippled_proof_path_response_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Response Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_proof_path_response_Bytes_Out", + "expr": "rippled_proof_path_response_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Response Bytes Out" } ], @@ -512,28 +512,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_replay_delta_request_Bytes_In", + "expr": "rippled_replay_delta_request_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Request Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_replay_delta_request_Bytes_Out", + "expr": "rippled_replay_delta_request_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Request Bytes Out" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_replay_delta_response_Bytes_In", + "expr": "rippled_replay_delta_response_Bytes_In{exported_instance=~\"$node\"}", "legendFormat": "Response Bytes In" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_replay_delta_response_Bytes_Out", + "expr": "rippled_replay_delta_response_Bytes_Out{exported_instance=~\"$node\"}", "legendFormat": "Response Bytes Out" } ], @@ -555,7 +555,28 @@ "schemaVersion": 39, "tags": ["rippled", "statsd", "overlay", "network", "telemetry"], "templating": { - "list": [] + "list": [ + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(exported_instance)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] }, "time": { "from": "now-1h", diff --git a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json index 2f64fa3d86..fa8104e522 100644 --- a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json @@ -30,7 +30,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(rippled_rpc_requests_total[5m])", + "expr": "rate(rippled_rpc_requests_total{exported_instance=~\"$node\"}[5m])", "legendFormat": "Requests / Sec" } ], @@ -62,14 +62,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.95\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95 Response Time" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.5\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50 Response Time" } ], @@ -108,14 +108,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_size{quantile=\"0.95\"}", + "expr": "rippled_rpc_size{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95 Response Size" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_size{quantile=\"0.5\"}", + "expr": "rippled_rpc_size{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50 Response Size" } ], @@ -154,28 +154,28 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.5\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.9\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.9\"}", "legendFormat": "P90" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.95\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_rpc_time{quantile=\"0.99\"}", + "expr": "rippled_rpc_time{exported_instance=~\"$node\", quantile=\"0.99\"}", "legendFormat": "P99" } ], @@ -214,14 +214,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_pathfind_fast{quantile=\"0.95\"}", + "expr": "rippled_pathfind_fast{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95 Fast Pathfind" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_pathfind_fast{quantile=\"0.5\"}", + "expr": "rippled_pathfind_fast{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50 Fast Pathfind" } ], @@ -260,14 +260,14 @@ "datasource": { "type": "prometheus" }, - "expr": "rippled_pathfind_full{quantile=\"0.95\"}", + "expr": "rippled_pathfind_full{exported_instance=~\"$node\", quantile=\"0.95\"}", "legendFormat": "P95 Full Pathfind" }, { "datasource": { "type": "prometheus" }, - "expr": "rippled_pathfind_full{quantile=\"0.5\"}", + "expr": "rippled_pathfind_full{exported_instance=~\"$node\", quantile=\"0.5\"}", "legendFormat": "P50 Full Pathfind" } ], @@ -306,7 +306,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(rippled_warn_total[5m])", + "expr": "rate(rippled_warn_total{exported_instance=~\"$node\"}[5m])", "legendFormat": "Warnings / Sec" } ], @@ -354,7 +354,7 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(rippled_drop_total[5m])", + "expr": "rate(rippled_drop_total{exported_instance=~\"$node\"}[5m])", "legendFormat": "Drops / Sec" } ], @@ -385,7 +385,28 @@ "schemaVersion": 39, "tags": ["rippled", "statsd", "rpc", "pathfinding", "telemetry"], "templating": { - "list": [] + "list": [ + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(exported_instance)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] }, "time": { "from": "now-1h", diff --git a/include/xrpl/beast/insight/OTelCollector.h b/include/xrpl/beast/insight/OTelCollector.h index 3e31df54de..ee0dd2c1b0 100644 --- a/include/xrpl/beast/insight/OTelCollector.h +++ b/include/xrpl/beast/insight/OTelCollector.h @@ -70,15 +70,22 @@ public: * collector that exports metrics via OTLP/HTTP. When telemetry is * disabled at compile time, returns a NullCollector. * - * @param endpoint OTLP/HTTP metrics endpoint URL - * (e.g. "http://localhost:4318/v1/metrics"). - * @param prefix Prefix prepended to all metric names - * (e.g. "rippled"). - * @param journal Journal for logging. + * @param endpoint OTLP/HTTP metrics endpoint URL + * (e.g. "http://localhost:4318/v1/metrics"). + * @param prefix Prefix prepended to all metric names + * (e.g. "rippled"). + * @param instanceId Unique identifier for this node instance, + * emitted as the `service.instance.id` OTel + * resource attribute. Defaults to empty string + * (attribute omitted when empty). + * @param journal Journal for logging. * @return Shared pointer to the created Collector. */ static std::shared_ptr - New(std::string const& endpoint, std::string const& prefix, Journal journal); + New(std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + Journal journal); }; } // namespace insight diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp index 34a0d917f8..fba3d03b8b 100644 --- a/src/libxrpl/beast/insight/OTelCollector.cpp +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -363,11 +363,17 @@ public: /** * @brief Construct the OTel collector and initialize the export pipeline. * - * @param endpoint OTLP/HTTP metrics endpoint URL. - * @param prefix Prefix for all metric names. - * @param journal Journal for logging. + * @param endpoint OTLP/HTTP metrics endpoint URL. + * @param prefix Prefix for all metric names. + * @param instanceId Value for the service.instance.id resource attribute. + * When empty, the attribute is omitted. + * @param journal Journal for logging. */ - OTelCollectorImp(std::string const& endpoint, std::string const& prefix, Journal journal); + OTelCollectorImp( + std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + Journal journal); /** * @brief Shut down the MeterProvider, flushing any pending exports. @@ -620,6 +626,7 @@ OTelMeterImpl::increment(value_type amount) OTelCollectorImp::OTelCollectorImp( std::string const& endpoint, std::string const& prefix, + std::string const& instanceId, Journal journal) : m_journal(journal), m_prefix(prefix) { @@ -641,9 +648,13 @@ OTelCollectorImp::OTelCollectorImp( metrics_sdk::PeriodicExportingMetricReaderFactory::Create(std::move(exporter), readerOpts); // Configure resource attributes matching the trace exporter. - auto resourceAttrs = resource::Resource::Create({ - {resource::SemanticConventions::kServiceName, "rippled"}, - }); + // Include service.instance.id when provided so Prometheus + // exported_instance labels distinguish multi-node deployments. + resource::ResourceAttributes attrs; + attrs[resource::SemanticConventions::kServiceName] = "rippled"; + if (!instanceId.empty()) + attrs[resource::SemanticConventions::kServiceInstanceId] = instanceId; + auto resourceAttrs = resource::Resource::Create(attrs); // Create MeterProvider. m_provider = metrics_sdk::MeterProviderFactory::Create(std::move(reader), resourceAttrs); @@ -782,9 +793,13 @@ OTelCollectorImp::formatName(std::string const& name) const //------------------------------------------------------------------------------ std::shared_ptr -OTelCollector::New(std::string const& endpoint, std::string const& prefix, Journal journal) +OTelCollector::New( + std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + Journal journal) { - return std::make_shared(endpoint, prefix, journal); + return std::make_shared(endpoint, prefix, instanceId, journal); } } // namespace insight @@ -805,6 +820,7 @@ std::shared_ptr OTelCollector::New( std::string const& /* endpoint */, std::string const& /* prefix */, + std::string const& /* instanceId */, Journal /* journal */) { return NullCollector::New(); diff --git a/src/xrpld/app/main/CollectorManager.cpp b/src/xrpld/app/main/CollectorManager.cpp index 2a2c5eeb1a..2dfbd87a28 100644 --- a/src/xrpld/app/main/CollectorManager.cpp +++ b/src/xrpld/app/main/CollectorManager.cpp @@ -32,7 +32,12 @@ public: endpoint = "http://localhost:4318/v1/metrics"; std::string const& prefix(get(params, "prefix")); - m_collector = beast::insight::OTelCollector::New(endpoint, prefix, journal); + // Read service_instance_id, same key as the [telemetry] + // section uses, so multi-node deployments can distinguish + // metric sources via the exported_instance Prometheus label. + std::string const instanceId = get(params, "service_instance_id"); + + m_collector = beast::insight::OTelCollector::New(endpoint, prefix, instanceId, journal); } else {