ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-08-21 22:30:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pratik/otel-phase10-workload-validation' into pratik/otel-sync-diagnostics

Browse Source
This commit is contained in:
Pratik Mankawde
2026-08-20 16:52:45 +01:00
parent 28055fdb96 8a011c6ecc
commit d450b16b71
19 changed files with 776 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
src/libxrpl/beast/insight/OTelCollector.cpp
View File

@@ -229,13 +229,9 @@ public:
* @param name Export-ready metric name, already run through
* formatName() by the collector: lowercase, with `.`
* and ` ` mapped to `_`.
* @param meter OTel Meter used to create the observable gauge.
* @param collector Owning collector, used to invoke hooks before reads.
*/
OTelGaugeImpl(
std::string const& name,
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
std::shared_ptr<OTelCollectorImp> const& collector);
OTelGaugeImpl(std::string name, std::shared_ptr<OTelCollectorImp> const& collector);
~OTelGaugeImpl() override;
@@ -273,6 +269,25 @@ public:
static void
gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state);
/**
* Create the observable instrument and register the callback, once.
*
* Called when the collector is told collection is ready, because the
* callback reads live application state.
*/
void
arm();
/**
* Remove the callback, so the reader thread stops observing this gauge.
*
* RemoveCallback is synchronous: the SDK guards its callback list and the
* observe pass with the same mutex, so no callback is running once this
* returns. Idempotent.
*/
void
disarm();
private:
/**
* Current gauge value, updated atomically by set()/increment().
@@ -280,10 +295,20 @@ private:
std::atomic<int64_t> value_{0};
/**
* OTel observable gauge handle (prevents deregistration).
* Export-ready metric name, held until arm() creates the instrument.
*/
std::string const name_;
/**
* OTel observable gauge handle, null until arm() runs.
*/
opentelemetry::nostd::shared_ptr<metrics_api::ObservableInstrument> gauge_;
/**
* Guards gauge_ against concurrent arm()/disarm().
*/
std::mutex armMutex_;
/**
* Owning collector, used to invoke hooks before reading gauge values.
*/
@@ -450,6 +475,12 @@ public:
Gauge
makeGauge(std::string const& name) override;
void
onCollectionReady() override;
void
onCollectionStopping() override;
Meter
makeMeter(std::string const& name) override;
/** @} */
@@ -503,6 +534,13 @@ public:
removeGauge(OTelGaugeImpl* gauge);
/** @} */
/**
* @brief The shared Meter, for gauges creating their instrument in arm().
* @return The Meter this collector resolved at construction.
*/
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
otelMeter() const;
/**
* @brief Format a raw metric name for export.
*
@@ -627,16 +665,37 @@ OTelEventImpl::notify(value_type const& value)
// OTelGaugeImpl
//------------------------------------------------------------------------------
OTelGaugeImpl::OTelGaugeImpl(
std::string const& name,
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
std::shared_ptr<OTelCollectorImp> const& collector)
: gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector)
OTelGaugeImpl::OTelGaugeImpl(std::string name, std::shared_ptr<OTelCollectorImp> const& collector)
: name_(std::move(name)), collector_(collector)
{
collector_->addGauge(this);
}
void
OTelGaugeImpl::arm()
{
// AddCallback arms the SDK reader thread against this gauge, and the
// callback runs hook handlers that read application services. The registry
// does not de-duplicate callbacks, so arm at most once.
std::scoped_lock const lock(armMutex_);
if (gauge_)
return;
gauge_ = collector_->otelMeter()->CreateInt64ObservableGauge(name_);
gauge_->AddCallback(gaugeCallback, this);
}
void
OTelGaugeImpl::disarm()
{
std::scoped_lock const lock(armMutex_);
if (!gauge_)
return;
gauge_->RemoveCallback(gaugeCallback, this);
gauge_ = nullptr;
}
void
OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state)
{
@@ -657,7 +716,8 @@ OTelGaugeImpl::~OTelGaugeImpl()
// The SDK's ObservableRegistry guards its callback list and the Observe()
// pass with the same mutex, so RemoveCallback cannot return while a
// callback for this instrument is in flight — removal is synchronous.
gauge_->RemoveCallback(gaugeCallback, this);
// A no-op when never armed, or already disarmed at shutdown.
disarm();
collector_->removeGauge(this);
}
@@ -786,7 +846,7 @@ OTelCollectorImp::makeEvent(std::string const& name)
Gauge
OTelCollectorImp::makeGauge(std::string const& name)
{
return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), otelMeter_, shared_from_this()));
return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), shared_from_this()));
}
Meter
@@ -852,6 +912,64 @@ OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge)
std::erase(gauges_, gauge);
}
void
OTelCollectorImp::onCollectionReady()
{
// Snapshot under the lock, arm outside it. arm() enters the SDK's
// observable registry lock, and the reader thread takes that lock before
// calling callHooks(), which wants mutex_. callHooks() copies its hook list
// for the same reason.
std::vector<OTelGaugeImpl*> gauges;
{
std::scoped_lock const lock(mutex_);
gauges = gauges_;
}
std::size_t armed = 0;
for (auto* gauge : gauges)
{
// Telemetry must never stop the node, so one bad instrument costs only
// its own metric.
try
{
gauge->arm();
++armed;
}
catch (std::exception const& e)
{
JLOG(journal_.error()) << "OTelCollector: could not register an observable gauge, "
"so that metric will not be exported: "
<< e.what();
}
}
JLOG(journal_.info()) << "OTelCollector: registered " << armed << " of " << gauges.size()
<< " observable gauges";
}
void
OTelCollectorImp::onCollectionStopping()
{
// Same lock discipline as onCollectionReady(): snapshot, then act outside
// the lock, because disarm() enters the SDK's observable registry lock.
std::vector<OTelGaugeImpl*> gauges;
{
std::scoped_lock const lock(mutex_);
gauges = gauges_;
}
for (auto* gauge : gauges)
gauge->disarm();
JLOG(journal_.info()) << "OTelCollector: stopped observing " << gauges.size() << " gauges";
}
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
OTelCollectorImp::otelMeter() const
{
return otelMeter_;
}
std::string
OTelCollectorImp::formatName(std::string const& name)
{

35
src/libxrpl/beast/insight/StatsDCollector.cpp
View File

@@ -23,6 +23,7 @@
#include <boost/system/detail/error_code.hpp>
#include <boost/system/system_error.hpp>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <deque>
@@ -218,6 +219,13 @@ private:
std::recursive_mutex metricsLock_;
List<StatsDMetricBase> metrics_;
/**
* Whether hook handlers may be called. False until onCollectionReady(),
* because the handlers read application services that are still being
* constructed while this collector exists.
*/
std::atomic<bool> polling_{false};
// Must come last for order of init
std::thread thread_;
@@ -255,6 +263,22 @@ public:
thread_.join();
}
void
onCollectionReady() override
{
polling_.store(true, std::memory_order_release);
}
void
onCollectionStopping() override
{
polling_.store(false, std::memory_order_release);
// onTimer holds metricsLock_ across the handler loop, so acquiring it
// here waits for a handler that is already running.
std::scoped_lock const _(metricsLock_);
}
Hook
makeHook(HookImpl::HandlerType const& handler) override
{
@@ -437,12 +461,15 @@ public:
return;
}
std::scoped_lock const _(metricsLock_);
if (polling_.load(std::memory_order_acquire))
{
std::scoped_lock const _(metricsLock_);
for (auto& m : metrics_)
m.doProcess();
for (auto& m : metrics_)
m.doProcess();
sendBuffers();
sendBuffers();
}
setTimer();
}

44
src/libxrpl/server/Wallet.cpp
View File

@@ -32,6 +32,7 @@
#include <format>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <unordered_set>
#include <utility>
@@ -147,27 +148,34 @@ clearNodeIdentity(soci::session& session)
session << "DELETE FROM NodeIdentity;";
}
std::optional<std::pair<PublicKey, SecretKey>>
readNodeIdentity(soci::session& session)
{
// SOCI requires boost::optional (not std::optional) as the parameter.
boost::optional<std::string> pubKO, priKO;
soci::statement st =
(session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;",
soci::into(pubKO),
soci::into(priKO));
st.execute();
while (st.fetch())
{
auto const sk = parseBase58<SecretKey>(TokenType::NodePrivate, priKO.value_or(""));
auto const pk = parseBase58<PublicKey>(TokenType::NodePublic, pubKO.value_or(""));
// Only use if the public and secret keys are a pair
if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk)))
return std::pair{*pk, *sk};
}
return std::nullopt;
}
std::pair<PublicKey, SecretKey>
getNodeIdentity(soci::session& session)
{
{
// SOCI requires boost::optional (not std::optional) as the parameter.
boost::optional<std::string> pubKO, priKO;
soci::statement st =
(session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;",
soci::into(pubKO),
soci::into(priKO));
st.execute();
while (st.fetch())
{
auto const sk = parseBase58<SecretKey>(TokenType::NodePrivate, priKO.value_or(""));
auto const pk = parseBase58<PublicKey>(TokenType::NodePublic, pubKO.value_or(""));
// Only use if the public and secret keys are a pair
if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk)))
return {*pk, *sk};
}
}
if (auto const stored = readNodeIdentity(session))
return *stored;
// If a valid identity wasn't found, we randomly generate a new one:
auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::Secp256k1);

54
src/libxrpl/telemetry/Telemetry.cpp
View File

@@ -66,6 +66,7 @@
#include <chrono>
#include <cstdint>
#include <exception>
#include <memory>
#include <string>
#include <string_view>
@@ -317,26 +318,41 @@ class TelemetryImpl : public Telemetry
*/
opentelemetry::nostd::shared_ptr<opentelemetry::context::RuntimeContextStorage> contextStorage_;
/**
* Set by stop(), so a second call does nothing.
*/
bool stopped_{false};
public:
TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal)
{
// Build the metrics pipeline NOW, in the constructor, so the global
// MeterProvider is published before any subsystem is constructed.
// beast::insight instruments are created eagerly in subsystem
// constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which
// run during ApplicationImp's member-init list — long before start().
// opentelemetry-cpp has no proxy MeterProvider, so an instrument
// created before SetMeterProvider() binds to the noop provider forever.
// Tracing does not have this problem because getTracer() is called
// fresh at each span creation (runtime, after start()).
// Publish the MeterProvider before any subsystem is constructed; see
// initMetrics(). setup_.serviceInstanceId is already resolved by the
// caller, so the resource is complete.
//
// The metrics resource uses setup_.serviceInstanceId as provided by
// config. A later setServiceInstanceId() (node-key fallback) cannot
// change this immutable resource, so operators relying on the node-key
// identity should set [telemetry] service_instance_id explicitly.
initMetrics();
// A failure must never stop the node starting: the global provider
// stays noop and every instrument call remains valid.
try
{
initMetrics();
}
catch (std::exception const& e)
{
JLOG(journal_.error()) << "Telemetry metrics pipeline failed to initialise, "
"continuing without metrics: "
<< e.what();
}
}
/**
* Override the service instance id, for callers that learn it late.
*
* Affects only the tracer resource, which start() builds. The metrics
* resource is built by the constructor and is immutable, so supply the id
* through Setup to have it on both.
*
* @param id The instance id to report on spans.
*/
void
setServiceInstanceId(std::string const& id) override
{
@@ -572,10 +588,16 @@ public:
void
stop() override
{
if (stopped_)
return;
stopped_ = true;
JLOG(journal_.info()) << "Telemetry stopping";
// Unregister global instance before tearing down the pipeline.
Telemetry::setInstance(nullptr);
// Unregister global instance before tearing down the pipeline, but only
// if this object is the one that published it.
if (Telemetry::getInstance() == this)
Telemetry::setInstance(nullptr);
if (sdkProvider_)
{

45
src/libxrpl/telemetry/TelemetryConfig.cpp
View File

@@ -112,24 +112,37 @@ makeTelemetrySetup(
setup.tlsClientCertPath = section.valueOr<std::string>(key::tlsClientCert, "");
setup.tlsClientKeyPath = section.valueOr<std::string>(key::tlsClientKey, "");
// Mutual TLS needs both the client certificate and its private key.
// Supplying only one fails later with a cryptic SSL handshake error, so
// reject the partial configuration here with an actionable message.
if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty())
// The mutual TLS (mTLS) checks below are fatal, so gate them on the one
// thing this parser can know: `enabled` is 1. With `enabled` 0 a leftover
// cert line must never stop the node from booting.
//
// The predicate is only that config switch, not whether an exporter can
// exist. This file has no preprocessor guard, so both checks also run in a
// -Dtelemetry=OFF build, where makeTelemetry() returns the null
// implementation whatever `enabled` says.
if (setup.enabled)
{
Throw<std::runtime_error>(
"[telemetry] tls_client_cert and tls_client_key must be set together "
"(set both for mutual TLS, or neither for one-way TLS).");
}
// mTLS needs both the client certificate and its private key.
// Supplying only one fails later with a cryptic SSL handshake error, so
// reject the partial configuration here with an actionable message.
if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty())
{
Throw<std::runtime_error>(
"[telemetry] tls_client_cert and tls_client_key must be set together "
"(set both for mutual TLS, or neither for one-way TLS).");
}
// Mutual TLS only takes effect when TLS is on. Certificate paths set with
// use_tls=0 would be silently ignored and the exporter would connect in
// plaintext, so reject that contradiction instead of failing open.
if (!setup.tlsClientCertPath.empty() && !setup.useTls)
{
Throw<std::runtime_error>(
"[telemetry] tls_client_cert/tls_client_key require use_tls=1 "
"(set use_tls=1 to enable mutual TLS, or remove the cert paths).");
// Still inside the enabled branch. mTLS only takes effect when TLS is
// on, so a client certificate set with use_tls=0 would be ignored and
// any exporter that did run would connect in plaintext. Reject that
// contradiction instead of failing open. tls_ca_cert is deliberately
// not checked this way.
if (!setup.tlsClientCertPath.empty() && !setup.useTls)
{
Throw<std::runtime_error>(
"[telemetry] tls_client_cert/tls_client_key require use_tls=1 "
"(set use_tls=1 to enable mutual TLS, or remove the cert paths).");
}
}
// Head sampling is intentionally fixed at 1.0 (sample everything) and is

174
src/tests/libxrpl/telemetry/TelemetryConfig.cpp
View File

@@ -2,12 +2,83 @@
#include <xrpl/config/BasicConfig.h>
#include <xrpl/telemetry/Telemetry.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <stdexcept>
using namespace xrpl;
using ::testing::HasSubstr;
using ::testing::ThrowsMessage;
namespace {
/**
* Shared inputs for the mutual TLS (mTLS) tests of makeTelemetrySetup().
*
* keyClientCert and keyClientKey are the config key names, named once so every
* test below spells them the same way, mirroring the `key::` constants the
* parser itself uses. A misspelling cannot hide here: the throwing case that
* names the misspelled key stops throwing, the use_tls case throws the pairing
* message instead and fails its matcher, and the value cases see an empty path
* or an unexpected throw. Tests that never set the key are unaffected. One
* source of truth still keeps the two files from drifting apart.
*
* clientCert and clientKey are the paths written to those keys. They are
* declared as `char const*` so they pass to Section::set() (which takes
* `std::string const&`) and compare against the parsed std::string members
* without an explicit conversion, exactly as a literal would.
*
* pairingError and useTlsError are message fragments. Both guards throw
* std::runtime_error, so the exception type alone cannot tell them apart.
* Each fragment occurs in exactly one of the two messages, so matching it
* proves which guard fired.
*/
namespace mtls {
constexpr char const* keyClientCert = "tls_client_cert";
constexpr char const* keyClientKey = "tls_client_key";
constexpr char const* clientCert = "/etc/ssl/client.pem";
constexpr char const* clientKey = "/etc/ssl/client.key";
constexpr char const* pairingError = "must be set together";
constexpr char const* useTlsError = "require use_tls=1";
/**
* Build a [telemetry] section carrying only the `enabled` key.
*
* Every mTLS test states `enabled` explicitly, because the validation
* guards run only when telemetry is on. Each test then adds the TLS keys its
* own case needs on top of the returned section.
*
* @param telemetryEnabled Value written to the `enabled` key.
* @return The section, ready for further set() calls.
*/
Section
makeSection(bool telemetryEnabled)
{
Section section;
section.set("enabled", telemetryEnabled ? "1" : "0");
return section;
}
/**
* Parse a [telemetry] section with a fixed placeholder node identity.
*
* Keeps the node key, version and network ID out of the individual cases,
* which vary only in their TLS keys.
*
* @param section The section to parse.
* @return The populated Setup struct.
*/
telemetry::Telemetry::Setup
parseSection(Section const& section)
{
return telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0);
}
} // namespace mtls
} // namespace
TEST(TelemetryConfig, setup_defaults)
{
telemetry::Telemetry::Setup const s;
@@ -88,39 +159,110 @@ TEST(TelemetryConfig, parse_full_section)
TEST(TelemetryConfig, mtls_cert_and_key_both_set)
{
Section section;
// Telemetry on and use_tls=1, so both guards run and neither may fire.
Section section = mtls::makeSection(true);
section.set("use_tls", "1");
section.set("tls_client_cert", "/etc/ssl/client.pem");
section.set("tls_client_key", "/etc/ssl/client.key");
section.set(mtls::keyClientCert, mtls::clientCert);
section.set(mtls::keyClientKey, mtls::clientKey);
auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0);
EXPECT_EQ(setup.tlsClientCertPath, "/etc/ssl/client.pem");
EXPECT_EQ(setup.tlsClientKeyPath, "/etc/ssl/client.key");
auto const setup = mtls::parseSection(section);
EXPECT_TRUE(setup.enabled);
EXPECT_TRUE(setup.useTls);
EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert);
EXPECT_EQ(setup.tlsClientKeyPath, mtls::clientKey);
}
TEST(TelemetryConfig, mtls_cert_without_key_throws)
{
Section section;
section.set("tls_client_cert", "/etc/ssl/client.pem");
EXPECT_THROW(
telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error);
// Only the cert is set, so the pairing guard is the one that must fire.
Section section = mtls::makeSection(true);
section.set(mtls::keyClientCert, mtls::clientCert);
EXPECT_THAT(
[&section] { mtls::parseSection(section); },
ThrowsMessage<std::runtime_error>(HasSubstr(mtls::pairingError)));
}
TEST(TelemetryConfig, mtls_key_without_cert_throws)
{
Section section;
section.set("tls_client_key", "/etc/ssl/client.key");
EXPECT_THROW(
telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error);
// Only the key is set, the mirror image of the case above.
Section section = mtls::makeSection(true);
section.set(mtls::keyClientKey, mtls::clientKey);
EXPECT_THAT(
[&section] { mtls::parseSection(section); },
ThrowsMessage<std::runtime_error>(HasSubstr(mtls::pairingError)));
}
TEST(TelemetryConfig, mtls_cert_key_without_use_tls_throws)
{
// Both paths are set, so the pairing guard cannot fire; use_tls is absent
// and defaults to 0, so the use_tls guard is the only reachable throw.
Section section = mtls::makeSection(true);
section.set(mtls::keyClientCert, mtls::clientCert);
section.set(mtls::keyClientKey, mtls::clientKey);
EXPECT_THAT(
[&section] { mtls::parseSection(section); },
ThrowsMessage<std::runtime_error>(HasSubstr(mtls::useTlsError)));
}
TEST(TelemetryConfig, mtls_contradiction_ignored_when_telemetry_disabled)
{
// The use_tls contradiction with telemetry off: parsing must succeed so a
// stale cert line cannot stop the node from booting.
Section section = mtls::makeSection(false);
section.set(mtls::keyClientCert, mtls::clientCert);
section.set(mtls::keyClientKey, mtls::clientKey);
auto const setup = mtls::parseSection(section);
EXPECT_FALSE(setup.enabled);
EXPECT_FALSE(setup.useTls);
EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert);
EXPECT_EQ(setup.tlsClientKeyPath, mtls::clientKey);
}
TEST(TelemetryConfig, mtls_cert_without_key_ignored_when_telemetry_disabled)
{
// The pairing violation with telemetry off: also parsed, not rejected.
Section section = mtls::makeSection(false);
section.set(mtls::keyClientCert, mtls::clientCert);
auto const setup = mtls::parseSection(section);
EXPECT_FALSE(setup.enabled);
EXPECT_FALSE(setup.useTls);
EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert);
EXPECT_TRUE(setup.tlsClientKeyPath.empty());
}
TEST(TelemetryConfig, mtls_default_no_client_tls_is_accepted)
{
// The documented default with telemetry on: no client certificate, and
// use_tls absent so it defaults to 0. Both guards run and neither may
// fire. The use_tls guard tests the certificate path first; drop that
// conjunct and this config is rejected, so no default node could boot.
Section const section = mtls::makeSection(true);
telemetry::Telemetry::Setup setup;
ASSERT_NO_THROW(setup = mtls::parseSection(section));
EXPECT_TRUE(setup.enabled);
EXPECT_FALSE(setup.useTls);
EXPECT_TRUE(setup.tlsClientCertPath.empty());
EXPECT_TRUE(setup.tlsClientKeyPath.empty());
}
TEST(TelemetryConfig, mtls_neither_set_is_one_way_tls)
{
Section section;
// Telemetry is on so the guards run, and this config must pass both:
// one-way TLS with a CA bundle and no client certificate.
Section section = mtls::makeSection(true);
section.set("use_tls", "1");
section.set("tls_ca_cert", "/etc/ssl/ca.pem");
auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0);
auto const setup = mtls::parseSection(section);
EXPECT_TRUE(setup.enabled);
EXPECT_TRUE(setup.useTls);
EXPECT_EQ(setup.tlsCertPath, "/etc/ssl/ca.pem");
EXPECT_TRUE(setup.tlsClientCertPath.empty());
EXPECT_TRUE(setup.tlsClientKeyPath.empty());
}

132
src/xrpld/app/main/Application.cpp
View File

@@ -330,7 +330,8 @@ public:
ApplicationImp(
std::unique_ptr<Config> config,
std::unique_ptr<Logs> logs,
std::unique_ptr<TimeKeeper> timeKeeper)
std::unique_ptr<TimeKeeper> timeKeeper,
std::optional<std::string> const& nodePublicKey)
: BasicApp(numberOfThreads(*config))
, config_(std::move(config))
, logs_(std::move(logs))
@@ -344,14 +345,26 @@ public:
*this,
logs_->journal("PerfLog"),
[this] { signalStop("PerfLog"); }))
// Telemetry publishes the MeterProvider on construction, so it must
// precede collectorManager_ below and every subsystem that creates an
// instrument. Its resource is immutable, so the instance id has to be
// supplied now; empty means this run reports none.
, telemetry_(
telemetry::makeTelemetry(
telemetry::makeTelemetrySetup(
config_->section("telemetry"),
"", // Updated later via setServiceInstanceId()
nodePublicKey.value_or(""),
build_info::getVersionString(),
config_->networkId),
logs_->journal("Telemetry")))
// Built here, not in setup(): getMetricsRegistry() is read from the job
// queue and io threads, which are already running, so assigning the
// handle later would race with those reads.
, metricsRegistry_(
std::make_unique<telemetry::MetricsRegistry>(
telemetry_->isEnabled(),
*this,
logs_->journal("MetricsRegistry")))
, txMaster_(*this)
, collectorManager_(makeCollectorManager(
@@ -529,6 +542,33 @@ public:
add(ledgerCleaner_.get());
}
/**
* Stop observing and stop telemetry before the members are destroyed.
*
* The metrics reader thread runs callbacks that read the services member
* destruction is about to tear down. telemetry_ is declared early because
* the collector needs its MeterProvider, so reverse-order member destruction
* would take it down last.
*
* run() does both on the normal path; this covers the paths that never
* reach it -- every `return false` in setup(), and the unit tests. Both
* calls are idempotent.
*/
~ApplicationImp() override
{
// A shutdown diagnostic must never terminate the process, and a
// destructor is implicitly noexcept.
try
{
collectorManager_->collector()->onCollectionStopping();
telemetry_->stop();
}
catch (std::exception const& e)
{
JLOG(journal_.error()) << "Error stopping telemetry: " << e.what();
}
}
//--------------------------------------------------------------------------
bool
@@ -1270,15 +1310,15 @@ private:
*
* Rule for keeping this call site valid: only telemetry work that reads
* NO application subsystem may run here. That holds today — this phase
* uses the config strings and the node identity, and creates only
* push-model counters and histograms, which app code records into once
* it is ready. Anything that registers a callback reading a subsystem
* must go in startTelemetryGauges() instead, because a callback
* registered here can fire on the metrics reader thread while the rest of
* the application is still being built.
* uses the config strings and creates only push-model counters and
* histograms, which app code records into once it is ready. Anything that
* registers a callback reading a subsystem must go in
* startTelemetryGauges() instead, because a callback registered here can
* fire on the metrics reader thread while the rest of the application is
* still being built.
*
* @pre nodeIdentity_ is populated (needed for the service_instance_id
* fallback) and metricsRegistry_ is constructed.
* The resource attributes, including service.instance.id, were supplied at
* construction.
*/
void
startTelemetry() const;
@@ -1402,11 +1442,8 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
nodeIdentity_ = getNodeIdentity(*this, cmdline);
// Now that the node identity is known, inject it into the telemetry
// resource attributes — but only if the user didn't already set a
// custom service_instance_id in [telemetry]. The Telemetry object
// was constructed with an empty serviceInstanceId because
// nodeIdentity_ is not available in the member initializer list.
// The metrics resource was fixed at construction, but the tracer resource is
// built by start() below, so a key minted just now can still reach spans.
if (!config_->section("telemetry").exists("service_instance_id"))
telemetry_->setServiceInstanceId(toBase58(TokenType::NodePublic, nodeIdentity_->first));
@@ -1415,12 +1452,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
// stable per-node key whatever [telemetry] says.
telemetry_->setNodeId(toBase58(TokenType::NodePublic, nodeIdentity_->first));
// Create the OTel MetricsRegistry for gap-fill metrics (counters,
// histograms, observable gauges). It must exist before startTelemetry(),
// which starts the metrics half of the pipeline.
metricsRegistry_ = std::make_unique<telemetry::MetricsRegistry>(
telemetry_->isEnabled(), *this, logs_->journal("MetricsRegistry"));
// Start telemetry here, not in start(). Spans and metrics are both emitted
// during the rest of setup() — the first consensus round in
// beginConsensus() below emits spans and records the process's only
@@ -1600,14 +1631,15 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
collectorManager_->collector());
add(*overlay_); // add to PropertyStream
// Register the observable instruments now that overlay_ exists. This arms
// the metrics reader thread to invoke their callbacks, several of which
// read getOverlay() — registering earlier would let the reader observe a
// half-built application. The reader thread itself already started in
// startTelemetry() above; this is as early as the callbacks can safely be
// attached, and it is still before beginConsensus() so the gauges cover
// the first round.
// Register the observable instruments now that overlay_ exists — the last of
// the services their callbacks read. Registering earlier would let the
// metrics reader thread observe a half-built application. Two independent
// sets: the MetricsRegistry gauges, and the insight collector's, whose
// callbacks additionally run the hook handlers in ledgerMaster_,
// networkOPs_, the peer finder and the job queue. Both are still before
// beginConsensus() below, so they cover the first round.
startTelemetryGauges();
collectorManager_->collector()->onCollectionReady();
// start first consensus round
if (!networkOPs_->beginConsensus(ledgerMaster_->getClosedLedger()->header().hash, {}))
@@ -1858,21 +1890,31 @@ ApplicationImp::run()
return getValidators().trustedPublisher(pubKey);
});
// Stop observing before any service below is stopped: the collector's gauge
// callbacks run hook handlers that read ledgerMaster_, networkOPs_, the peer
// finder, the job queue and overlay_. Returns once no callback is running.
collectorManager_->collector()->onCollectionStopping();
// The order of these stop calls is delicate.
// Re-ordering them risks undefined behavior.
loadManager_->stop();
// Detach MetricsRegistry observable-gauge callbacks BEFORE stopping
// any service the callbacks read from. The callbacks run on the OTel
// reader thread and touch nodeStore_, overlay_, networkOPs_,
// ledgerMaster, inboundLedgers, etc. A final tick that fires after
// one of those services has shut down would dereference dangling
// state. detachCallbacks() flips an atomic flag every callback
// acquire-loads at its entry, so subsequent ticks become no-ops.
// The final provider teardown still happens in metricsRegistry_->stop()
// farther down.
// Stop the metrics pipeline BEFORE any service its callbacks read. Those
// callbacks run on the OTel reader thread and touch nodeStore_, overlay_,
// networkOPs_, ledgerMaster, inboundLedgers and more, so a tick arriving
// after one of them has stopped would read dangling state.
//
// detachCallbacks() alone would not be enough: it flips a flag that each
// callback checks on entry, which leaves a callback that is already past
// that check running. stop() shuts the provider down, which joins the
// reader thread, so once it returns no callback is running or can start.
// The cost is that metrics recorded during the remaining shutdown steps
// are not exported.
if (metricsRegistry_)
{
metricsRegistry_->detachCallbacks();
metricsRegistry_->stop();
}
shaMapStore_->stop();
jobQueue_->stop();
@@ -1887,10 +1929,6 @@ ApplicationImp::run()
ledgerCleaner_->stop();
nodeStore_->stop();
perfLog_->stop();
// Stop metrics pipeline before telemetry — gauge callbacks reference
// Application services that may be shutting down.
if (metricsRegistry_)
metricsRegistry_->stop();
// Telemetry must stop last among trace-producing components.
// serverHandler_, overlay_, and jobQueue_ are already stopped above,
// so no threads should be calling startSpan() at this point.
@@ -2458,9 +2496,19 @@ makeApplication(
std::unique_ptr<Config> config,
std::unique_ptr<Logs> logs,
std::unique_ptr<TimeKeeper> timeKeeper)
{
return makeApplication(std::move(config), std::move(logs), std::move(timeKeeper), std::nullopt);
}
std::unique_ptr<Application>
makeApplication(
std::unique_ptr<Config> config,
std::unique_ptr<Logs> logs,
std::unique_ptr<TimeKeeper> timeKeeper,
std::optional<std::string> const& nodePublicKey)
{
return std::make_unique<ApplicationImp>(
std::move(config), std::move(logs), std::move(timeKeeper));
std::move(config), std::move(logs), std::move(timeKeeper), nodePublicKey);
}
void

15
src/xrpld/app/main/Application.h
View File

@@ -174,4 +174,19 @@ makeApplication(
std::unique_ptr<Logs> logs,
std::unique_ptr<TimeKeeper> timeKeeper);
/**
* Construct the application with a known node public key.
*
* Telemetry builds its resource attributes during construction and they are
* immutable, so the base58 node public key must be supplied here. Pass
* std::nullopt when it is unknown; that run reports no instance id. See
* resolveNodePublicKey().
*/
std::unique_ptr<Application>
makeApplication(
std::unique_ptr<Config> config,
std::unique_ptr<Logs> logs,
std::unique_ptr<TimeKeeper> timeKeeper,
std::optional<std::string> const& nodePublicKey);
} // namespace xrpl

57
src/xrpld/app/main/Main.cpp
View File

@@ -1,4 +1,5 @@
#include <xrpld/app/main/Application.h>
#include <xrpld/app/main/NodeIdentity.h>
#include <xrpld/core/Config.h>
#include <xrpld/core/TimeKeeper.h>
#include <xrpld/rpc/RPCCall.h>
@@ -12,6 +13,7 @@
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/unit_test/suite_info.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/config/Constants.h>
#include <xrpl/core/StartUpType.h>
#include <xrpl/git/Git.h>
@@ -36,6 +38,7 @@
#include <exception>
#include <iostream>
#include <memory>
#include <optional>
#include <ostream>
#include <string>
#include <vector>
@@ -804,8 +807,58 @@ run(int argc, char** argv)
if (vm.contains("debug"))
setDebugLogSink(logs->makeSink("Debug", beast::Severity::Trace));
auto app =
makeApplication(std::move(config), std::move(logs), std::make_unique<TimeKeeper>());
// Telemetry needs the node public key at construction, so read it here
// where a config error can still be reported and the process can exit
// cleanly. getNodeIdentity() in setup() stays authoritative.
std::optional<std::string> nodePublicKey;
try
{
nodePublicKey = resolveNodePublicKey(*config, vm, logs->journal("Application"));
}
catch (std::exception const& e)
{
std::cerr << "Unable to start " << systemName() << ": " << e.what() << std::endl;
return -1;
}
if (!nodePublicKey)
{
JLOG(logs->journal("Application").warn())
<< "Telemetry: no node identity available yet, so this run reports an empty "
"service.instance.id. Set [telemetry] service_instance_id, or restart once "
"the node key exists.";
}
// Application construction runs member initializers that validate
// config (for example the [telemetry] section) and can throw. A throw
// from a member-initializer list cannot be recovered inside the
// constructor, so catch it here. Left uncaught it reaches
// std::terminate, whose default handler prints a C++ terminate dump
// and raises SIGABRT, leaving a core file where the system allows one;
// the catch replaces that with two operator-readable lines on stderr
// and a non-zero exit status.
//
// Only the construction is covered. The [telemetry] section is parsed
// near the top of the member list, before the job queue and node store
// are built, so unwinding that throw destroys very little. setup() is
// left outside deliberately: it starts subsystems whose shutdown order
// is delicate, and only the normal stop sequence gets that order right.
std::unique_ptr<Application> app;
try
{
app = makeApplication(
std::move(config), std::move(logs), std::make_unique<TimeKeeper>(), nodePublicKey);
}
catch (std::exception const& e)
{
std::cerr << "Unable to start " << systemName() << ": " << e.what() << std::endl;
std::cerr << "Fix the reported problem and start again." << std::endl;
return -1;
}
// Construction succeeded, so app holds an object: makeApplication never
// returns null and the catch above is the only other way out.
XRPL_ASSERT(app, "xrpl::run : non-null application");
if (!app->setup(vm))
return -1;

83
src/xrpld/app/main/NodeIdentity.cpp
View File

@@ -8,13 +8,18 @@
#include <xrpl/protocol/KeyType.h>
#include <xrpl/protocol/SecretKey.h>
#include <xrpl/protocol/Seed.h>
#include <xrpl/rdb/DBInit.h>
#include <xrpl/rdb/DatabaseCon.h>
#include <xrpl/server/Wallet.h>
#include <boost/program_options/variables_map.hpp>
#include <array>
#include <filesystem>
#include <optional>
#include <stdexcept>
#include <string>
#include <system_error>
#include <utility>
namespace xrpl {
@@ -58,4 +63,82 @@ getNodeIdentity(Application& app, boost::program_options::variables_map const& c
return getNodeIdentity(*db);
}
std::optional<std::string>
resolveNodePublicKey(
Config const& config,
boost::program_options::variables_map const& cmdline,
beast::Journal journal)
{
std::optional<Seed> seed;
bool seedConfigured = false;
if (cmdline.contains("nodeid"))
{
seedConfigured = true;
seed = parseGenericSeed(cmdline["nodeid"].as<std::string>(), false);
}
else if (config.exists(Sections::kNodeSeed))
{
seedConfigured = true;
if (auto const& lines = config.section(Sections::kNodeSeed).lines(); !lines.empty())
seed = parseBase58<Seed>(lines.front());
}
// A configured seed decides the identity outright. A malformed or missing
// one is reported by getNodeIdentity(), which runs later.
if (seedConfigured)
{
if (!seed)
return std::nullopt;
auto const secretKey = generateSecretKey(KeyType::Secp256k1, *seed);
return toBase58(TokenType::NodePublic, derivePublicKey(KeyType::Secp256k1, secretKey));
}
// --newnodeid discards whatever is stored.
if (cmdline.contains("newnodeid"))
return std::nullopt;
try
{
auto setup = setupDatabaseCon(config, journal);
// Standalone uses a temporary database, so nothing is persisted and this
// run will mint a fresh key.
if (setup.standAlone && setup.startUp != StartUpType::Load &&
setup.startUp != StartUpType::LoadFile && setup.startUp != StartUpType::Replay)
{
return std::nullopt;
}
// The global pragmas include journal_mode, which rewrites the database
// header. The wallet is opened without them everywhere else.
setup.useGlobalPragma = false;
// Only read an existing file: SQLite would otherwise create one.
if (std::error_code ec; !std::filesystem::exists(setup.dataDir / kWalletDbName, ec))
{
return std::nullopt;
}
// Empty init SQL: open the existing schema, never create it.
DatabaseCon walletDb{
setup,
kWalletDbName,
std::array<std::string, 0>{},
std::array<char const*, 0>{},
journal};
auto db = walletDb.checkoutDb();
if (auto const stored = readNodeIdentity(*db))
return toBase58(TokenType::NodePublic, stored->first);
}
catch (std::exception const& e)
{
JLOG(journal.warn()) << "Could not read the node identity: " << e.what();
}
return std::nullopt;
}
} // namespace xrpl

26
src/xrpld/app/main/NodeIdentity.h
View File

@@ -1,12 +1,16 @@
#pragma once
#include <xrpld/app/main/Application.h>
#include <xrpld/core/Config.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/protocol/PublicKey.h>
#include <xrpl/protocol/SecretKey.h>
#include <boost/program_options.hpp>
#include <optional>
#include <string>
#include <utility>
namespace xrpl {
@@ -20,4 +24,26 @@ namespace xrpl {
std::pair<PublicKey, SecretKey>
getNodeIdentity(Application& app, boost::program_options::variables_map const& cmdline);
/**
* This server's public key, read without creating or modifying anything.
*
* For callers that need the identity before the Application exists, such as
* telemetry building its resource attributes in the member-init list. Derives
* from a configured seed when there is one, otherwise reads the wallet database
* only if it already exists.
*
* getNodeIdentity() remains authoritative and mints a key when none exists.
*
* @param config The server configuration.
* @param cmdline The command line parameters passed into the application.
* @param journal Journal for reporting an unreadable database.
* @return The base58-encoded node public key, or std::nullopt if none can be
* read.
*/
std::optional<std::string>
resolveNodePublicKey(
Config const& config,
boost::program_options::variables_map const& cmdline,
beast::Journal journal);
} // namespace xrpl