ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-07-23 15:10:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): wire jq_trans_overflow_total as observable counter

Browse Source 
Observe the overflow total from Overlay::getJqTransOverflow() via an
ObservableCounter instead of the unused push helper (which had no call
site, so the metric read flat zero). Keeps the increment path in the
overlay-owned code and removes the dead incrementJqTransOverflow() helper.
Update the job-queue panel to reflect the now-live metric, and refresh
stale xrpld_-prefixed metric names in the header comments.

NOTE: committed without a local build per instruction; needs compile +
clang-tidy verification before push.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pratik Mankawde
2026-07-10 17:16:35 +01:00
parent 535a95363f
commit 851bc75dfc
3 changed files with 54 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/telemetry/grafana/dashboards/job-queue.json
View File

@@ -392,7 +392,7 @@
},
{
"title": "Transaction Overflow Rate",
"description": "⚠ NOT YET INSTRUMENTED — jq_trans_overflow_total has no call sites; reads flat zero until wired.\n\n###### What this is:\n*Rate at which transaction jobs would be shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute.*\n\n###### Reading it:\n*This counter is not currently wired up in the code, so the panel reads empty or zero regardless of load.*\n\n###### Healthy range:\n*Not applicable until wired; currently always empty or zero.*\n\n###### Watch for:\n*n/a until wired — this counter does not rise under load today.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`(job-queue transaction overflow counter)`",
"description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate — the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`",
"type": "timeseries",
"gridPos": {
"h": 8,

33
src/xrpld/telemetry/MetricsRegistry.cpp
View File

@@ -254,8 +254,30 @@ MetricsRegistry::initSyncInstruments()
"validations_checked_total", "Total network validations received and checked");
stateChangesCounter_ =
meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes");
jqTransOverflowCounter_ = meter_->CreateUInt64Counter(
// jq_trans_overflow_total is observed from Overlay's existing cumulative
// atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay
// owns the only increment site (PeerImp), so an ObservableCounter reads the
// live total each collection cycle without threading a push path through
// develop-owned overlay code.
jqTransOverflowObservable_ = meter_->CreateInt64ObservableCounter(
"jq_trans_overflow_total", "Total job queue transaction overflows");
jqTransOverflowObservable_->AddCallback(
[](opentelemetry::metrics::ObserverResult result, void* state) {
auto* self = static_cast<MetricsRegistry*>(state);
if (self->callbacksDetached_.load(std::memory_order_acquire))
return;
try
{
opentelemetry::nostd::get<opentelemetry::nostd::shared_ptr<
opentelemetry::metrics::ObserverResultT<int64_t>>>(result)
->Observe(static_cast<int64_t>(self->app_.getOverlay().getJqTransOverflow()));
}
catch (...) // NOLINT(bugprone-empty-catch)
{
// Silently skip on error.
}
},
this);
ledgerHistoryMismatchCounter_ = meter_->CreateUInt64Counter(
"ledger_history_mismatch_total", "Total built-vs-validated ledger mismatches by reason");
txqExpiredCounter_ = meter_->CreateUInt64Counter(
@@ -1455,15 +1477,6 @@ MetricsRegistry::incrementStateChanges()
#endif
}
void
MetricsRegistry::incrementJqTransOverflow()
{
#ifdef XRPL_ENABLE_TELEMETRY
if (enabled_ && jqTransOverflowCounter_)
jqTransOverflowCounter_->Add(1);
#endif
}
void
MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason)
{

66
src/xrpld/telemetry/MetricsRegistry.h
View File

@@ -23,23 +23,22 @@
| +-- OtlpHttpMetricExporter
|
+-- Counters / Histograms (synchronous instruments)
| +-- xrpld_rpc_method_started_total
| +-- xrpld_rpc_method_finished_total
| +-- xrpld_rpc_method_errored_total
| +-- xrpld_rpc_method_duration_us (Histogram)
| +-- xrpld_job_queued_total
| +-- xrpld_job_started_total
| +-- xrpld_job_finished_total
| +-- xrpld_job_queued_duration_us (Histogram)
| +-- xrpld_job_running_duration_us (Histogram)
| +-- xrpld_ledgers_closed_total
| +-- xrpld_validations_sent_total
| +-- xrpld_validations_checked_total
| +-- xrpld_state_changes_total
| +-- xrpld_jq_trans_overflow_total
| +-- xrpld_ledger_history_mismatch_total{reason}
| +-- xrpld_txq_expired_total
| +-- xrpld_txq_dropped_total{reason}
| +-- rpc_method_started_total
| +-- rpc_method_finished_total
| +-- rpc_method_errored_total
| +-- rpc_method_us (Histogram)
| +-- job_queued_total
| +-- job_started_total
| +-- job_finished_total
| +-- job_queued_us (Histogram)
| +-- job_running_us (Histogram)
| +-- ledgers_closed_total
| +-- validations_sent_total
| +-- validations_checked_total
| +-- state_changes_total
| +-- ledger_history_mismatch_total{reason}
| +-- txq_expired_total
| +-- txq_dropped_total{reason}
|
+-- ValidationTracker (validation agreement tracker)
|
@@ -61,6 +60,7 @@
+-- State tracking (mode value, time in state)
+-- Storage detail (NuDB sizes)
+-- Validation agreement (1h/24h pct, counts)
+-- jq_trans_overflow_total (observed from Overlay)
Control-flow for async gauges:
@@ -347,13 +347,6 @@ public:
void
incrementStateChanges();
/** Increment the jq_trans_overflow_total counter.
Called when the job queue transaction limit overflows (mirrors
Overlay::incJqTransOverflow()).
*/
void
incrementJqTransOverflow();
/** Increment the ledger_history_mismatch_total counter for a reason.
Called from LedgerHistory::handleMismatch() once the mismatch has
been classified. The reason label turns fork diagnosis from a
@@ -504,38 +497,39 @@ private:
validationAgreementGauge_;
// --- External dashboard parity counters (Task 7.14) ---
/// Counter: xrpld_ledgers_closed_total — incremented each consensus round.
/// Counter: ledgers_closed_total — incremented each consensus round.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
ledgersClosedCounter_;
/// Counter: xrpld_validations_sent_total — incremented when this node sends a validation.
/// Counter: validations_sent_total — incremented when this node sends a validation.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
validationsSentCounter_;
/// Counter: xrpld_validations_checked_total — incremented for each network validation
/// Counter: validations_checked_total — incremented for each network validation
/// received.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
validationsCheckedCounter_;
/// Counter: xrpld_state_changes_total — incremented on operating mode transitions.
/// Counter: state_changes_total — incremented on operating mode transitions.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
stateChangesCounter_;
/// Counter: xrpld_jq_trans_overflow_total — incremented on job queue transaction overflows.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
jqTransOverflowCounter_;
/// Counter: xrpld_ledger_history_mismatch_total{reason} — incremented per classified
/// ObservableCounter: jq_trans_overflow_total — observed from
/// Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay).
opentelemetry::nostd::shared_ptr<opentelemetry::metrics::ObservableInstrument>
jqTransOverflowObservable_;
/// Counter: ledger_history_mismatch_total{reason} — incremented per classified
/// built-vs-validated ledger mismatch.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>
ledgerHistoryMismatchCounter_;
/// Counter: xrpld_txq_expired_total — incremented per transaction expired out of the
/// Counter: txq_expired_total — incremented per transaction expired out of the
/// transaction queue.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>> txqExpiredCounter_;
/// Counter: xrpld_txq_dropped_total{reason} — incremented when a transaction is refused
/// Counter: txq_dropped_total{reason} — incremented when a transaction is refused
/// admission to the queue.
opentelemetry::nostd::unique_ptr<opentelemetry::metrics::Counter<uint64_t>> txqDroppedCounter_;
/// ObservableCounter: xrpld_validation_agreements_total — observed from
/// ObservableCounter: validation_agreements_total — observed from
/// ValidationTracker::totalAgreementsEver() (monotonic gross lifetime
/// tally, initial-classification semantics).
opentelemetry::nostd::shared_ptr<opentelemetry::metrics::ObservableInstrument>
validationAgreementsObservable_;
/// ObservableCounter: xrpld_validation_missed_total — observed from
/// ObservableCounter: validation_missed_total — observed from
/// ValidationTracker::totalMissedEver() (monotonic gross lifetime tally,
/// initial-classification semantics).
opentelemetry::nostd::shared_ptr<opentelemetry::metrics::ObservableInstrument>