diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 1b74be09d5..b90c76ca4c 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -392,7 +392,7 @@ }, { "title": "Transaction Overflow Rate", - "description": "⚠ NOT YET INSTRUMENTED — jq_trans_overflow_total has no call sites; reads flat zero until wired.\n\n###### What this is:\n*Rate at which transaction jobs would be shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute.*\n\n###### Reading it:\n*This counter is not currently wired up in the code, so the panel reads empty or zero regardless of load.*\n\n###### Healthy range:\n*Not applicable until wired; currently always empty or zero.*\n\n###### Watch for:\n*n/a until wired — this counter does not rise under load today.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`(job-queue transaction overflow counter)`", + "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate — the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`", "type": "timeseries", "gridPos": { "h": 8, diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 1a9b73d939..501ff7d2fb 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -254,8 +254,30 @@ MetricsRegistry::initSyncInstruments() "validations_checked_total", "Total network validations received and checked"); stateChangesCounter_ = meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes"); - jqTransOverflowCounter_ = meter_->CreateUInt64Counter( + // jq_trans_overflow_total is observed from Overlay's existing cumulative + // atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay + // owns the only increment site (PeerImp), so an ObservableCounter reads the + // live total each collection cycle without threading a push path through + // develop-owned overlay code. + jqTransOverflowObservable_ = meter_->CreateInt64ObservableCounter( "jq_trans_overflow_total", "Total job queue transaction overflows"); + jqTransOverflowObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->app_.getOverlay().getJqTransOverflow())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); ledgerHistoryMismatchCounter_ = meter_->CreateUInt64Counter( "ledger_history_mismatch_total", "Total built-vs-validated ledger mismatches by reason"); txqExpiredCounter_ = meter_->CreateUInt64Counter( @@ -1455,15 +1477,6 @@ MetricsRegistry::incrementStateChanges() #endif } -void -MetricsRegistry::incrementJqTransOverflow() -{ -#ifdef XRPL_ENABLE_TELEMETRY - if (enabled_ && jqTransOverflowCounter_) - jqTransOverflowCounter_->Add(1); -#endif -} - void MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason) { diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index 280251516d..91eab98fb3 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -23,23 +23,22 @@ | +-- OtlpHttpMetricExporter | +-- Counters / Histograms (synchronous instruments) - | +-- xrpld_rpc_method_started_total - | +-- xrpld_rpc_method_finished_total - | +-- xrpld_rpc_method_errored_total - | +-- xrpld_rpc_method_duration_us (Histogram) - | +-- xrpld_job_queued_total - | +-- xrpld_job_started_total - | +-- xrpld_job_finished_total - | +-- xrpld_job_queued_duration_us (Histogram) - | +-- xrpld_job_running_duration_us (Histogram) - | +-- xrpld_ledgers_closed_total - | +-- xrpld_validations_sent_total - | +-- xrpld_validations_checked_total - | +-- xrpld_state_changes_total - | +-- xrpld_jq_trans_overflow_total - | +-- xrpld_ledger_history_mismatch_total{reason} - | +-- xrpld_txq_expired_total - | +-- xrpld_txq_dropped_total{reason} + | +-- rpc_method_started_total + | +-- rpc_method_finished_total + | +-- rpc_method_errored_total + | +-- rpc_method_us (Histogram) + | +-- job_queued_total + | +-- job_started_total + | +-- job_finished_total + | +-- job_queued_us (Histogram) + | +-- job_running_us (Histogram) + | +-- ledgers_closed_total + | +-- validations_sent_total + | +-- validations_checked_total + | +-- state_changes_total + | +-- ledger_history_mismatch_total{reason} + | +-- txq_expired_total + | +-- txq_dropped_total{reason} | +-- ValidationTracker (validation agreement tracker) | @@ -61,6 +60,7 @@ +-- State tracking (mode value, time in state) +-- Storage detail (NuDB sizes) +-- Validation agreement (1h/24h pct, counts) + +-- jq_trans_overflow_total (observed from Overlay) Control-flow for async gauges: @@ -347,13 +347,6 @@ public: void incrementStateChanges(); - /** Increment the jq_trans_overflow_total counter. - Called when the job queue transaction limit overflows (mirrors - Overlay::incJqTransOverflow()). - */ - void - incrementJqTransOverflow(); - /** Increment the ledger_history_mismatch_total counter for a reason. Called from LedgerHistory::handleMismatch() once the mismatch has been classified. The reason label turns fork diagnosis from a @@ -504,38 +497,39 @@ private: validationAgreementGauge_; // --- External dashboard parity counters (Task 7.14) --- - /// Counter: xrpld_ledgers_closed_total — incremented each consensus round. + /// Counter: ledgers_closed_total — incremented each consensus round. opentelemetry::nostd::unique_ptr> ledgersClosedCounter_; - /// Counter: xrpld_validations_sent_total — incremented when this node sends a validation. + /// Counter: validations_sent_total — incremented when this node sends a validation. opentelemetry::nostd::unique_ptr> validationsSentCounter_; - /// Counter: xrpld_validations_checked_total — incremented for each network validation + /// Counter: validations_checked_total — incremented for each network validation /// received. opentelemetry::nostd::unique_ptr> validationsCheckedCounter_; - /// Counter: xrpld_state_changes_total — incremented on operating mode transitions. + /// Counter: state_changes_total — incremented on operating mode transitions. opentelemetry::nostd::unique_ptr> stateChangesCounter_; - /// Counter: xrpld_jq_trans_overflow_total — incremented on job queue transaction overflows. - opentelemetry::nostd::unique_ptr> - jqTransOverflowCounter_; - /// Counter: xrpld_ledger_history_mismatch_total{reason} — incremented per classified + /// ObservableCounter: jq_trans_overflow_total — observed from + /// Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay). + opentelemetry::nostd::shared_ptr + jqTransOverflowObservable_; + /// Counter: ledger_history_mismatch_total{reason} — incremented per classified /// built-vs-validated ledger mismatch. opentelemetry::nostd::unique_ptr> ledgerHistoryMismatchCounter_; - /// Counter: xrpld_txq_expired_total — incremented per transaction expired out of the + /// Counter: txq_expired_total — incremented per transaction expired out of the /// transaction queue. opentelemetry::nostd::unique_ptr> txqExpiredCounter_; - /// Counter: xrpld_txq_dropped_total{reason} — incremented when a transaction is refused + /// Counter: txq_dropped_total{reason} — incremented when a transaction is refused /// admission to the queue. opentelemetry::nostd::unique_ptr> txqDroppedCounter_; - /// ObservableCounter: xrpld_validation_agreements_total — observed from + /// ObservableCounter: validation_agreements_total — observed from /// ValidationTracker::totalAgreementsEver() (monotonic gross lifetime /// tally, initial-classification semantics). opentelemetry::nostd::shared_ptr validationAgreementsObservable_; - /// ObservableCounter: xrpld_validation_missed_total — observed from + /// ObservableCounter: validation_missed_total — observed from /// ValidationTracker::totalMissedEver() (monotonic gross lifetime tally, /// initial-classification semantics). opentelemetry::nostd::shared_ptr