diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index a30a5f901c..961cf003e5 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -1988,8 +1988,8 @@ "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, __name__) (rate({__name__=~\"xrpld_(makeFetchPack|publishAcqLedger|untrustedValidation|manifest|localTransaction|ledgerReplayRequest|ledgerRequest|untrustedProposal|ledgerReplayTask|ledgerData|clientCommand|clientSubscribe|clientFeeChange|clientConsensus|clientAccountHistory|clientRPC|clientWebsocket|RPC|updatePaths|transaction|batch|advanceLedger|publishNewLedger|fetchTxnData|writeAhead|trustedValidation|writeObjects|acceptLedger|trustedProposal|sweep|clusterReport|heartbeat|administration|handleHaveTransactions|doTransactions)_milliseconds_bucket\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "legendFormat": "{{__name__}}" + "expr": "histogram_quantile($quantile, sum by (le, job_type) (rate(xrpld_job_running_duration_us_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "legendFormat": "{{job_type}}" } ], "fieldConfig": { @@ -2027,8 +2027,8 @@ "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, __name__) (rate({__name__=~\"xrpld_(makeFetchPack_q|publishAcqLedger_q|untrustedValidation_q|manifest_q|localTransaction_q|ledgerReplayRequest_q|ledgerRequest_q|untrustedProposal_q|ledgerReplayTask_q|ledgerData_q|clientCommand_q|clientSubscribe_q|clientFeeChange_q|clientConsensus_q|clientAccountHistory_q|clientRPC_q|clientWebsocket_q|RPC_q|updatePaths_q|transaction_q|batch_q|advanceLedger_q|publishNewLedger_q|fetchTxnData_q|writeAhead_q|trustedValidation_q|writeObjects_q|acceptLedger_q|trustedProposal_q|sweep_q|clusterReport_q|heartbeat_q|administration_q|handleHaveTransactions_q|doTransactions_q)_milliseconds_bucket\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "legendFormat": "{{__name__}}" + "expr": "histogram_quantile($quantile, sum by (le, job_type) (rate(xrpld_job_queued_duration_us_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "legendFormat": "{{job_type}}" } ], "fieldConfig": { diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp index 1bd6a018af..3271026664 100644 --- a/src/libxrpl/beast/insight/OTelCollector.cpp +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -613,6 +613,11 @@ OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void OTelGaugeImpl::~OTelGaugeImpl() { + // RemoveCallback must run before this object is destroyed so the SDK + // collection thread cannot invoke gaugeCallback on a dangling `this`. + // The SDK's ObservableRegistry guards its callback list and the Observe() + // pass with the same mutex, so RemoveCallback cannot return while a + // callback for this instrument is in flight — removal is synchronous. gauge_->RemoveCallback(gaugeCallback, this); collector_->removeGauge(this); } @@ -781,8 +786,16 @@ OTelCollectorImp::callHooks() if (!lastHookCallMs_.compare_exchange_strong(last, now, std::memory_order_acq_rel)) return; // Another thread won the race. - std::scoped_lock const lock(mutex_); - for (auto* hook : hooks_) + // Copy the hook list under the lock, then invoke handlers outside it. + // A handler may drop the last reference to an OTelHookImpl, whose + // destructor calls removeHook() and re-acquires mutex_; invoking + // handlers while holding the (non-recursive) lock would deadlock. + std::vector hooks; + { + std::scoped_lock const lock(mutex_); + hooks = hooks_; + } + for (auto* hook : hooks) hook->callHandler(); }