mirror of
https://github.com/XRPLF/rippled.git
synced 2026-08-21 14:20:56 +00:00
Merge branch 'pratik/otel-phase8-log-correlation' into pratik/otel-phase9-metric-gap-fill
# Conflicts: # docker/telemetry/xrpld-telemetry.cfg # src/libxrpl/beast/insight/OTelCollector.cpp # src/libxrpl/telemetry/Telemetry.cpp # src/xrpld/app/main/Application.cpp
This commit is contained in:
@@ -121,6 +121,7 @@ processors:
|
||||
- context: datapoint
|
||||
statements:
|
||||
- set(attributes["service_instance_id"], resource.attributes["service.instance.id"])
|
||||
- set(attributes["xrpl_node_id"], resource.attributes["xrpl.node.id"])
|
||||
- set(attributes["deployment_environment"], resource.attributes["deployment.environment"])
|
||||
- set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"])
|
||||
|
||||
@@ -141,6 +142,7 @@ connectors:
|
||||
# series from distinct nodes/tiers grouped separately.
|
||||
resource_metrics_key_attributes:
|
||||
- service.instance.id
|
||||
- xrpl.node.id
|
||||
- deployment.environment
|
||||
- xrpl.network.type
|
||||
histogram:
|
||||
|
||||
@@ -136,6 +136,7 @@ connectors:
|
||||
# nodes/tiers grouped separately.
|
||||
resource_metrics_key_attributes:
|
||||
- service.instance.id
|
||||
- xrpl.node.id
|
||||
- deployment.environment
|
||||
- xrpl.network.type
|
||||
histogram:
|
||||
|
||||
@@ -30,6 +30,29 @@ public:
|
||||
|
||||
virtual ~Collector() = 0;
|
||||
|
||||
/**
|
||||
* Called once the services that hook handlers read are constructed.
|
||||
*
|
||||
* Implementations that poll their producers must not do so before this:
|
||||
* hook handlers read live application state. Default is a no-op, for
|
||||
* collectors that only push.
|
||||
*/
|
||||
virtual void
|
||||
onCollectionReady()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Called before those services are shut down.
|
||||
*
|
||||
* Polling must have stopped by the time this returns. Paired with
|
||||
* onCollectionReady().
|
||||
*/
|
||||
virtual void
|
||||
onCollectionStopping()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a hook.
|
||||
*
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
@@ -88,6 +89,19 @@ addValidatorManifest(soci::session& session, std::string const& serialized);
|
||||
void
|
||||
clearNodeIdentity(soci::session& session);
|
||||
|
||||
/**
|
||||
* Returns this node's stored keypair, if the database holds a valid one.
|
||||
*
|
||||
* Read-only: unlike getNodeIdentity(), never generates or persists a key. A row
|
||||
* counts only when its public and secret keys are a pair.
|
||||
*
|
||||
* @param session Session with the database.
|
||||
*
|
||||
* @return The stored keypair, or std::nullopt.
|
||||
*/
|
||||
std::optional<std::pair<PublicKey, SecretKey>>
|
||||
readNodeIdentity(soci::session& session);
|
||||
|
||||
/**
|
||||
* Returns a stable public and private key for this node.
|
||||
*
|
||||
|
||||
@@ -229,13 +229,9 @@ public:
|
||||
* @param name Export-ready metric name, already run through
|
||||
* formatName() by the collector: lowercase, with `.`
|
||||
* and ` ` mapped to `_`.
|
||||
* @param meter OTel Meter used to create the observable gauge.
|
||||
* @param collector Owning collector, used to invoke hooks before reads.
|
||||
*/
|
||||
OTelGaugeImpl(
|
||||
std::string const& name,
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
|
||||
std::shared_ptr<OTelCollectorImp> const& collector);
|
||||
OTelGaugeImpl(std::string name, std::shared_ptr<OTelCollectorImp> const& collector);
|
||||
|
||||
~OTelGaugeImpl() override;
|
||||
|
||||
@@ -273,6 +269,25 @@ public:
|
||||
static void
|
||||
gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state);
|
||||
|
||||
/**
|
||||
* Create the observable instrument and register the callback, once.
|
||||
*
|
||||
* Called when the collector is told collection is ready, because the
|
||||
* callback reads live application state.
|
||||
*/
|
||||
void
|
||||
arm();
|
||||
|
||||
/**
|
||||
* Remove the callback, so the reader thread stops observing this gauge.
|
||||
*
|
||||
* RemoveCallback is synchronous: the SDK guards its callback list and the
|
||||
* observe pass with the same mutex, so no callback is running once this
|
||||
* returns. Idempotent.
|
||||
*/
|
||||
void
|
||||
disarm();
|
||||
|
||||
private:
|
||||
/**
|
||||
* Current gauge value, updated atomically by set()/increment().
|
||||
@@ -280,10 +295,20 @@ private:
|
||||
std::atomic<int64_t> value_{0};
|
||||
|
||||
/**
|
||||
* OTel observable gauge handle (prevents deregistration).
|
||||
* Export-ready metric name, held until arm() creates the instrument.
|
||||
*/
|
||||
std::string const name_;
|
||||
|
||||
/**
|
||||
* OTel observable gauge handle, null until arm() runs.
|
||||
*/
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::ObservableInstrument> gauge_;
|
||||
|
||||
/**
|
||||
* Guards gauge_ against concurrent arm()/disarm().
|
||||
*/
|
||||
std::mutex armMutex_;
|
||||
|
||||
/**
|
||||
* Owning collector, used to invoke hooks before reading gauge values.
|
||||
*/
|
||||
@@ -450,6 +475,12 @@ public:
|
||||
Gauge
|
||||
makeGauge(std::string const& name) override;
|
||||
|
||||
void
|
||||
onCollectionReady() override;
|
||||
|
||||
void
|
||||
onCollectionStopping() override;
|
||||
|
||||
Meter
|
||||
makeMeter(std::string const& name) override;
|
||||
/** @} */
|
||||
@@ -503,6 +534,13 @@ public:
|
||||
removeGauge(OTelGaugeImpl* gauge);
|
||||
/** @} */
|
||||
|
||||
/**
|
||||
* @brief The shared Meter, for gauges creating their instrument in arm().
|
||||
* @return The Meter this collector resolved at construction.
|
||||
*/
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
|
||||
otelMeter() const;
|
||||
|
||||
/**
|
||||
* @brief Format a raw metric name for export.
|
||||
*
|
||||
@@ -627,16 +665,37 @@ OTelEventImpl::notify(value_type const& value)
|
||||
// OTelGaugeImpl
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
OTelGaugeImpl::OTelGaugeImpl(
|
||||
std::string const& name,
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
|
||||
std::shared_ptr<OTelCollectorImp> const& collector)
|
||||
: gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector)
|
||||
OTelGaugeImpl::OTelGaugeImpl(std::string name, std::shared_ptr<OTelCollectorImp> const& collector)
|
||||
: name_(std::move(name)), collector_(collector)
|
||||
{
|
||||
collector_->addGauge(this);
|
||||
}
|
||||
|
||||
void
|
||||
OTelGaugeImpl::arm()
|
||||
{
|
||||
// AddCallback arms the SDK reader thread against this gauge, and the
|
||||
// callback runs hook handlers that read application services. The registry
|
||||
// does not de-duplicate callbacks, so arm at most once.
|
||||
std::scoped_lock const lock(armMutex_);
|
||||
if (gauge_)
|
||||
return;
|
||||
|
||||
gauge_ = collector_->otelMeter()->CreateInt64ObservableGauge(name_);
|
||||
gauge_->AddCallback(gaugeCallback, this);
|
||||
}
|
||||
|
||||
void
|
||||
OTelGaugeImpl::disarm()
|
||||
{
|
||||
std::scoped_lock const lock(armMutex_);
|
||||
if (!gauge_)
|
||||
return;
|
||||
|
||||
gauge_->RemoveCallback(gaugeCallback, this);
|
||||
gauge_ = nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state)
|
||||
{
|
||||
@@ -657,7 +716,8 @@ OTelGaugeImpl::~OTelGaugeImpl()
|
||||
// The SDK's ObservableRegistry guards its callback list and the Observe()
|
||||
// pass with the same mutex, so RemoveCallback cannot return while a
|
||||
// callback for this instrument is in flight — removal is synchronous.
|
||||
gauge_->RemoveCallback(gaugeCallback, this);
|
||||
// A no-op when never armed, or already disarmed at shutdown.
|
||||
disarm();
|
||||
collector_->removeGauge(this);
|
||||
}
|
||||
|
||||
@@ -786,7 +846,7 @@ OTelCollectorImp::makeEvent(std::string const& name)
|
||||
Gauge
|
||||
OTelCollectorImp::makeGauge(std::string const& name)
|
||||
{
|
||||
return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), otelMeter_, shared_from_this()));
|
||||
return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), shared_from_this()));
|
||||
}
|
||||
|
||||
Meter
|
||||
@@ -852,6 +912,64 @@ OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge)
|
||||
std::erase(gauges_, gauge);
|
||||
}
|
||||
|
||||
void
|
||||
OTelCollectorImp::onCollectionReady()
|
||||
{
|
||||
// Snapshot under the lock, arm outside it. arm() enters the SDK's
|
||||
// observable registry lock, and the reader thread takes that lock before
|
||||
// calling callHooks(), which wants mutex_. callHooks() copies its hook list
|
||||
// for the same reason.
|
||||
std::vector<OTelGaugeImpl*> gauges;
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
gauges = gauges_;
|
||||
}
|
||||
|
||||
std::size_t armed = 0;
|
||||
for (auto* gauge : gauges)
|
||||
{
|
||||
// Telemetry must never stop the node, so one bad instrument costs only
|
||||
// its own metric.
|
||||
try
|
||||
{
|
||||
gauge->arm();
|
||||
++armed;
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal_.error()) << "OTelCollector: could not register an observable gauge, "
|
||||
"so that metric will not be exported: "
|
||||
<< e.what();
|
||||
}
|
||||
}
|
||||
|
||||
JLOG(journal_.info()) << "OTelCollector: registered " << armed << " of " << gauges.size()
|
||||
<< " observable gauges";
|
||||
}
|
||||
|
||||
void
|
||||
OTelCollectorImp::onCollectionStopping()
|
||||
{
|
||||
// Same lock discipline as onCollectionReady(): snapshot, then act outside
|
||||
// the lock, because disarm() enters the SDK's observable registry lock.
|
||||
std::vector<OTelGaugeImpl*> gauges;
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
gauges = gauges_;
|
||||
}
|
||||
|
||||
for (auto* gauge : gauges)
|
||||
gauge->disarm();
|
||||
|
||||
JLOG(journal_.info()) << "OTelCollector: stopped observing " << gauges.size() << " gauges";
|
||||
}
|
||||
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
|
||||
OTelCollectorImp::otelMeter() const
|
||||
{
|
||||
return otelMeter_;
|
||||
}
|
||||
|
||||
std::string
|
||||
OTelCollectorImp::formatName(std::string const& name)
|
||||
{
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
#include <boost/system/system_error.hpp>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <deque>
|
||||
@@ -218,6 +219,13 @@ private:
|
||||
std::recursive_mutex metricsLock_;
|
||||
List<StatsDMetricBase> metrics_;
|
||||
|
||||
/**
|
||||
* Whether hook handlers may be called. False until onCollectionReady(),
|
||||
* because the handlers read application services that are still being
|
||||
* constructed while this collector exists.
|
||||
*/
|
||||
std::atomic<bool> polling_{false};
|
||||
|
||||
// Must come last for order of init
|
||||
std::thread thread_;
|
||||
|
||||
@@ -255,6 +263,22 @@ public:
|
||||
thread_.join();
|
||||
}
|
||||
|
||||
void
|
||||
onCollectionReady() override
|
||||
{
|
||||
polling_.store(true, std::memory_order_release);
|
||||
}
|
||||
|
||||
void
|
||||
onCollectionStopping() override
|
||||
{
|
||||
polling_.store(false, std::memory_order_release);
|
||||
|
||||
// onTimer holds metricsLock_ across the handler loop, so acquiring it
|
||||
// here waits for a handler that is already running.
|
||||
std::scoped_lock const _(metricsLock_);
|
||||
}
|
||||
|
||||
Hook
|
||||
makeHook(HookImpl::HandlerType const& handler) override
|
||||
{
|
||||
@@ -437,12 +461,15 @@ public:
|
||||
return;
|
||||
}
|
||||
|
||||
std::scoped_lock const _(metricsLock_);
|
||||
if (polling_.load(std::memory_order_acquire))
|
||||
{
|
||||
std::scoped_lock const _(metricsLock_);
|
||||
|
||||
for (auto& m : metrics_)
|
||||
m.doProcess();
|
||||
for (auto& m : metrics_)
|
||||
m.doProcess();
|
||||
|
||||
sendBuffers();
|
||||
sendBuffers();
|
||||
}
|
||||
|
||||
setTimer();
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
#include <format>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
@@ -147,27 +148,34 @@ clearNodeIdentity(soci::session& session)
|
||||
session << "DELETE FROM NodeIdentity;";
|
||||
}
|
||||
|
||||
std::optional<std::pair<PublicKey, SecretKey>>
|
||||
readNodeIdentity(soci::session& session)
|
||||
{
|
||||
// SOCI requires boost::optional (not std::optional) as the parameter.
|
||||
boost::optional<std::string> pubKO, priKO;
|
||||
soci::statement st =
|
||||
(session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;",
|
||||
soci::into(pubKO),
|
||||
soci::into(priKO));
|
||||
st.execute();
|
||||
while (st.fetch())
|
||||
{
|
||||
auto const sk = parseBase58<SecretKey>(TokenType::NodePrivate, priKO.value_or(""));
|
||||
auto const pk = parseBase58<PublicKey>(TokenType::NodePublic, pubKO.value_or(""));
|
||||
|
||||
// Only use if the public and secret keys are a pair
|
||||
if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk)))
|
||||
return std::pair{*pk, *sk};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::pair<PublicKey, SecretKey>
|
||||
getNodeIdentity(soci::session& session)
|
||||
{
|
||||
{
|
||||
// SOCI requires boost::optional (not std::optional) as the parameter.
|
||||
boost::optional<std::string> pubKO, priKO;
|
||||
soci::statement st =
|
||||
(session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;",
|
||||
soci::into(pubKO),
|
||||
soci::into(priKO));
|
||||
st.execute();
|
||||
while (st.fetch())
|
||||
{
|
||||
auto const sk = parseBase58<SecretKey>(TokenType::NodePrivate, priKO.value_or(""));
|
||||
auto const pk = parseBase58<PublicKey>(TokenType::NodePublic, pubKO.value_or(""));
|
||||
|
||||
// Only use if the public and secret keys are a pair
|
||||
if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk)))
|
||||
return {*pk, *sk};
|
||||
}
|
||||
}
|
||||
if (auto const stored = readNodeIdentity(session))
|
||||
return *stored;
|
||||
|
||||
// If a valid identity wasn't found, we randomly generate a new one:
|
||||
auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::Secp256k1);
|
||||
|
||||
@@ -66,6 +66,7 @@
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <exception>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
@@ -317,26 +318,41 @@ class TelemetryImpl : public Telemetry
|
||||
*/
|
||||
opentelemetry::nostd::shared_ptr<opentelemetry::context::RuntimeContextStorage> contextStorage_;
|
||||
|
||||
/**
|
||||
* Set by stop(), so a second call does nothing.
|
||||
*/
|
||||
bool stopped_{false};
|
||||
|
||||
public:
|
||||
TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal)
|
||||
{
|
||||
// Build the metrics pipeline NOW, in the constructor, so the global
|
||||
// MeterProvider is published before any subsystem is constructed.
|
||||
// beast::insight instruments are created eagerly in subsystem
|
||||
// constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which
|
||||
// run during ApplicationImp's member-init list — long before start().
|
||||
// opentelemetry-cpp has no proxy MeterProvider, so an instrument
|
||||
// created before SetMeterProvider() binds to the noop provider forever.
|
||||
// Tracing does not have this problem because getTracer() is called
|
||||
// fresh at each span creation (runtime, after start()).
|
||||
// Publish the MeterProvider before any subsystem is constructed; see
|
||||
// initMetrics(). setup_.serviceInstanceId is already resolved by the
|
||||
// caller, so the resource is complete.
|
||||
//
|
||||
// The metrics resource uses setup_.serviceInstanceId as provided by
|
||||
// config. A later setServiceInstanceId() (node-key fallback) cannot
|
||||
// change this immutable resource, so operators relying on the node-key
|
||||
// identity should set [telemetry] service_instance_id explicitly.
|
||||
initMetrics();
|
||||
// A failure must never stop the node starting: the global provider
|
||||
// stays noop and every instrument call remains valid.
|
||||
try
|
||||
{
|
||||
initMetrics();
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal_.error()) << "Telemetry metrics pipeline failed to initialise, "
|
||||
"continuing without metrics: "
|
||||
<< e.what();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Override the service instance id, for callers that learn it late.
|
||||
*
|
||||
* Affects only the tracer resource, which start() builds. The metrics
|
||||
* resource is built by the constructor and is immutable, so supply the id
|
||||
* through Setup to have it on both.
|
||||
*
|
||||
* @param id The instance id to report on spans.
|
||||
*/
|
||||
void
|
||||
setServiceInstanceId(std::string const& id) override
|
||||
{
|
||||
@@ -572,10 +588,16 @@ public:
|
||||
void
|
||||
stop() override
|
||||
{
|
||||
if (stopped_)
|
||||
return;
|
||||
stopped_ = true;
|
||||
|
||||
JLOG(journal_.info()) << "Telemetry stopping";
|
||||
|
||||
// Unregister global instance before tearing down the pipeline.
|
||||
Telemetry::setInstance(nullptr);
|
||||
// Unregister global instance before tearing down the pipeline, but only
|
||||
// if this object is the one that published it.
|
||||
if (Telemetry::getInstance() == this)
|
||||
Telemetry::setInstance(nullptr);
|
||||
|
||||
if (sdkProvider_)
|
||||
{
|
||||
|
||||
@@ -328,7 +328,8 @@ public:
|
||||
ApplicationImp(
|
||||
std::unique_ptr<Config> config,
|
||||
std::unique_ptr<Logs> logs,
|
||||
std::unique_ptr<TimeKeeper> timeKeeper)
|
||||
std::unique_ptr<TimeKeeper> timeKeeper,
|
||||
std::optional<std::string> const& nodePublicKey)
|
||||
: BasicApp(numberOfThreads(*config))
|
||||
, config_(std::move(config))
|
||||
, logs_(std::move(logs))
|
||||
@@ -342,11 +343,15 @@ public:
|
||||
*this,
|
||||
logs_->journal("PerfLog"),
|
||||
[this] { signalStop("PerfLog"); }))
|
||||
// Telemetry publishes the MeterProvider on construction, so it must
|
||||
// precede collectorManager_ below and every subsystem that creates an
|
||||
// instrument. Its resource is immutable, so the instance id has to be
|
||||
// supplied now; empty means this run reports none.
|
||||
, telemetry_(
|
||||
telemetry::makeTelemetry(
|
||||
telemetry::makeTelemetrySetup(
|
||||
config_->section("telemetry"),
|
||||
"", // Updated later via setServiceInstanceId()
|
||||
nodePublicKey.value_or(""),
|
||||
build_info::getVersionString(),
|
||||
config_->networkId),
|
||||
logs_->journal("Telemetry")))
|
||||
@@ -535,6 +540,33 @@ public:
|
||||
add(ledgerCleaner_.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop observing and stop telemetry before the members are destroyed.
|
||||
*
|
||||
* The metrics reader thread runs callbacks that read the services member
|
||||
* destruction is about to tear down. telemetry_ is declared early because
|
||||
* the collector needs its MeterProvider, so reverse-order member destruction
|
||||
* would take it down last.
|
||||
*
|
||||
* run() does both on the normal path; this covers the paths that never
|
||||
* reach it -- every `return false` in setup(), and the unit tests. Both
|
||||
* calls are idempotent.
|
||||
*/
|
||||
~ApplicationImp() override
|
||||
{
|
||||
// A shutdown diagnostic must never terminate the process, and a
|
||||
// destructor is implicitly noexcept.
|
||||
try
|
||||
{
|
||||
collectorManager_->collector()->onCollectionStopping();
|
||||
telemetry_->stop();
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal_.error()) << "Error stopping telemetry: " << e.what();
|
||||
}
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
bool
|
||||
@@ -1206,15 +1238,15 @@ private:
|
||||
*
|
||||
* Rule for keeping this call site valid: only telemetry work that reads
|
||||
* NO application subsystem may run here. That holds today — this phase
|
||||
* uses the config strings and the node identity, and creates only
|
||||
* push-model counters and histograms, which app code records into once
|
||||
* it is ready. Anything that registers a callback reading a subsystem
|
||||
* must go in startTelemetryGauges() instead, because a callback
|
||||
* registered here can fire on the metrics reader thread while the rest of
|
||||
* the application is still being built.
|
||||
* uses the config strings and creates only push-model counters and
|
||||
* histograms, which app code records into once it is ready. Anything that
|
||||
* registers a callback reading a subsystem must go in
|
||||
* startTelemetryGauges() instead, because a callback registered here can
|
||||
* fire on the metrics reader thread while the rest of the application is
|
||||
* still being built.
|
||||
*
|
||||
* @pre nodeIdentity_ is populated (needed for the service_instance_id
|
||||
* fallback) and metricsRegistry_ is constructed.
|
||||
* The resource attributes, including service.instance.id, were supplied at
|
||||
* construction.
|
||||
*/
|
||||
void
|
||||
startTelemetry() const;
|
||||
@@ -1338,11 +1370,8 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
|
||||
|
||||
nodeIdentity_ = getNodeIdentity(*this, cmdline);
|
||||
|
||||
// Now that the node identity is known, inject it into the telemetry
|
||||
// resource attributes — but only if the user didn't already set a
|
||||
// custom service_instance_id in [telemetry]. The Telemetry object
|
||||
// was constructed with an empty serviceInstanceId because
|
||||
// nodeIdentity_ is not available in the member initializer list.
|
||||
// The metrics resource was fixed at construction, but the tracer resource is
|
||||
// built by start() below, so a key minted just now can still reach spans.
|
||||
if (!config_->section("telemetry").exists("service_instance_id"))
|
||||
telemetry_->setServiceInstanceId(toBase58(TokenType::NodePublic, nodeIdentity_->first));
|
||||
|
||||
@@ -1530,14 +1559,15 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
|
||||
collectorManager_->collector());
|
||||
add(*overlay_); // add to PropertyStream
|
||||
|
||||
// Register the observable instruments now that overlay_ exists. This arms
|
||||
// the metrics reader thread to invoke their callbacks, several of which
|
||||
// read getOverlay() — registering earlier would let the reader observe a
|
||||
// half-built application. The reader thread itself already started in
|
||||
// startTelemetry() above; this is as early as the callbacks can safely be
|
||||
// attached, and it is still before beginConsensus() so the gauges cover
|
||||
// the first round.
|
||||
// Register the observable instruments now that overlay_ exists — the last of
|
||||
// the services their callbacks read. Registering earlier would let the
|
||||
// metrics reader thread observe a half-built application. Two independent
|
||||
// sets: the MetricsRegistry gauges, and the insight collector's, whose
|
||||
// callbacks additionally run the hook handlers in ledgerMaster_,
|
||||
// networkOPs_, the peer finder and the job queue. Both are still before
|
||||
// beginConsensus() below, so they cover the first round.
|
||||
startTelemetryGauges();
|
||||
collectorManager_->collector()->onCollectionReady();
|
||||
|
||||
// start first consensus round
|
||||
if (!networkOPs_->beginConsensus(ledgerMaster_->getClosedLedger()->header().hash, {}))
|
||||
@@ -1788,6 +1818,11 @@ ApplicationImp::run()
|
||||
return getValidators().trustedPublisher(pubKey);
|
||||
});
|
||||
|
||||
// Stop observing before any service below is stopped: the collector's gauge
|
||||
// callbacks run hook handlers that read ledgerMaster_, networkOPs_, the peer
|
||||
// finder, the job queue and overlay_. Returns once no callback is running.
|
||||
collectorManager_->collector()->onCollectionStopping();
|
||||
|
||||
// The order of these stop calls is delicate.
|
||||
// Re-ordering them risks undefined behavior.
|
||||
loadManager_->stop();
|
||||
@@ -2389,9 +2424,19 @@ makeApplication(
|
||||
std::unique_ptr<Config> config,
|
||||
std::unique_ptr<Logs> logs,
|
||||
std::unique_ptr<TimeKeeper> timeKeeper)
|
||||
{
|
||||
return makeApplication(std::move(config), std::move(logs), std::move(timeKeeper), std::nullopt);
|
||||
}
|
||||
|
||||
std::unique_ptr<Application>
|
||||
makeApplication(
|
||||
std::unique_ptr<Config> config,
|
||||
std::unique_ptr<Logs> logs,
|
||||
std::unique_ptr<TimeKeeper> timeKeeper,
|
||||
std::optional<std::string> const& nodePublicKey)
|
||||
{
|
||||
return std::make_unique<ApplicationImp>(
|
||||
std::move(config), std::move(logs), std::move(timeKeeper));
|
||||
std::move(config), std::move(logs), std::move(timeKeeper), nodePublicKey);
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
@@ -174,4 +174,19 @@ makeApplication(
|
||||
std::unique_ptr<Logs> logs,
|
||||
std::unique_ptr<TimeKeeper> timeKeeper);
|
||||
|
||||
/**
|
||||
* Construct the application with a known node public key.
|
||||
*
|
||||
* Telemetry builds its resource attributes during construction and they are
|
||||
* immutable, so the base58 node public key must be supplied here. Pass
|
||||
* std::nullopt when it is unknown; that run reports no instance id. See
|
||||
* resolveNodePublicKey().
|
||||
*/
|
||||
std::unique_ptr<Application>
|
||||
makeApplication(
|
||||
std::unique_ptr<Config> config,
|
||||
std::unique_ptr<Logs> logs,
|
||||
std::unique_ptr<TimeKeeper> timeKeeper,
|
||||
std::optional<std::string> const& nodePublicKey);
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include <xrpld/app/main/Application.h>
|
||||
#include <xrpld/app/main/NodeIdentity.h>
|
||||
#include <xrpld/core/Config.h>
|
||||
#include <xrpld/core/TimeKeeper.h>
|
||||
#include <xrpld/rpc/RPCCall.h>
|
||||
@@ -36,6 +37,7 @@
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -804,8 +806,30 @@ run(int argc, char** argv)
|
||||
if (vm.contains("debug"))
|
||||
setDebugLogSink(logs->makeSink("Debug", beast::Severity::Trace));
|
||||
|
||||
auto app =
|
||||
makeApplication(std::move(config), std::move(logs), std::make_unique<TimeKeeper>());
|
||||
// Telemetry needs the node public key at construction, so read it here
|
||||
// where a config error can still be reported and the process can exit
|
||||
// cleanly. getNodeIdentity() in setup() stays authoritative.
|
||||
std::optional<std::string> nodePublicKey;
|
||||
try
|
||||
{
|
||||
nodePublicKey = resolveNodePublicKey(*config, vm, logs->journal("Application"));
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
std::cerr << "Unable to start " << systemName() << ": " << e.what() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!nodePublicKey)
|
||||
{
|
||||
JLOG(logs->journal("Application").warn())
|
||||
<< "Telemetry: no node identity available yet, so this run reports an empty "
|
||||
"service.instance.id. Set [telemetry] service_instance_id, or restart once "
|
||||
"the node key exists.";
|
||||
}
|
||||
|
||||
auto app = makeApplication(
|
||||
std::move(config), std::move(logs), std::make_unique<TimeKeeper>(), nodePublicKey);
|
||||
|
||||
if (!app->setup(vm))
|
||||
return -1;
|
||||
|
||||
@@ -8,13 +8,18 @@
|
||||
#include <xrpl/protocol/KeyType.h>
|
||||
#include <xrpl/protocol/SecretKey.h>
|
||||
#include <xrpl/protocol/Seed.h>
|
||||
#include <xrpl/rdb/DBInit.h>
|
||||
#include <xrpl/rdb/DatabaseCon.h>
|
||||
#include <xrpl/server/Wallet.h>
|
||||
|
||||
#include <boost/program_options/variables_map.hpp>
|
||||
|
||||
#include <array>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -58,4 +63,82 @@ getNodeIdentity(Application& app, boost::program_options::variables_map const& c
|
||||
return getNodeIdentity(*db);
|
||||
}
|
||||
|
||||
std::optional<std::string>
|
||||
resolveNodePublicKey(
|
||||
Config const& config,
|
||||
boost::program_options::variables_map const& cmdline,
|
||||
beast::Journal journal)
|
||||
{
|
||||
std::optional<Seed> seed;
|
||||
bool seedConfigured = false;
|
||||
|
||||
if (cmdline.contains("nodeid"))
|
||||
{
|
||||
seedConfigured = true;
|
||||
seed = parseGenericSeed(cmdline["nodeid"].as<std::string>(), false);
|
||||
}
|
||||
else if (config.exists(Sections::kNodeSeed))
|
||||
{
|
||||
seedConfigured = true;
|
||||
if (auto const& lines = config.section(Sections::kNodeSeed).lines(); !lines.empty())
|
||||
seed = parseBase58<Seed>(lines.front());
|
||||
}
|
||||
|
||||
// A configured seed decides the identity outright. A malformed or missing
|
||||
// one is reported by getNodeIdentity(), which runs later.
|
||||
if (seedConfigured)
|
||||
{
|
||||
if (!seed)
|
||||
return std::nullopt;
|
||||
|
||||
auto const secretKey = generateSecretKey(KeyType::Secp256k1, *seed);
|
||||
return toBase58(TokenType::NodePublic, derivePublicKey(KeyType::Secp256k1, secretKey));
|
||||
}
|
||||
|
||||
// --newnodeid discards whatever is stored.
|
||||
if (cmdline.contains("newnodeid"))
|
||||
return std::nullopt;
|
||||
|
||||
try
|
||||
{
|
||||
auto setup = setupDatabaseCon(config, journal);
|
||||
|
||||
// Standalone uses a temporary database, so nothing is persisted and this
|
||||
// run will mint a fresh key.
|
||||
if (setup.standAlone && setup.startUp != StartUpType::Load &&
|
||||
setup.startUp != StartUpType::LoadFile && setup.startUp != StartUpType::Replay)
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// The global pragmas include journal_mode, which rewrites the database
|
||||
// header. The wallet is opened without them everywhere else.
|
||||
setup.useGlobalPragma = false;
|
||||
|
||||
// Only read an existing file: SQLite would otherwise create one.
|
||||
if (std::error_code ec; !std::filesystem::exists(setup.dataDir / kWalletDbName, ec))
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Empty init SQL: open the existing schema, never create it.
|
||||
DatabaseCon walletDb{
|
||||
setup,
|
||||
kWalletDbName,
|
||||
std::array<std::string, 0>{},
|
||||
std::array<char const*, 0>{},
|
||||
journal};
|
||||
|
||||
auto db = walletDb.checkoutDb();
|
||||
if (auto const stored = readNodeIdentity(*db))
|
||||
return toBase58(TokenType::NodePublic, stored->first);
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal.warn()) << "Could not read the node identity: " << e.what();
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpld/app/main/Application.h>
|
||||
#include <xrpld/core/Config.h>
|
||||
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/protocol/PublicKey.h>
|
||||
#include <xrpl/protocol/SecretKey.h>
|
||||
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -20,4 +24,26 @@ namespace xrpl {
|
||||
std::pair<PublicKey, SecretKey>
|
||||
getNodeIdentity(Application& app, boost::program_options::variables_map const& cmdline);
|
||||
|
||||
/**
|
||||
* This server's public key, read without creating or modifying anything.
|
||||
*
|
||||
* For callers that need the identity before the Application exists, such as
|
||||
* telemetry building its resource attributes in the member-init list. Derives
|
||||
* from a configured seed when there is one, otherwise reads the wallet database
|
||||
* only if it already exists.
|
||||
*
|
||||
* getNodeIdentity() remains authoritative and mints a key when none exists.
|
||||
*
|
||||
* @param config The server configuration.
|
||||
* @param cmdline The command line parameters passed into the application.
|
||||
* @param journal Journal for reporting an unreadable database.
|
||||
* @return The base58-encoded node public key, or std::nullopt if none can be
|
||||
* read.
|
||||
*/
|
||||
std::optional<std::string>
|
||||
resolveNodePublicKey(
|
||||
Config const& config,
|
||||
boost::program_options::variables_map const& cmdline,
|
||||
beast::Journal journal);
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
Reference in New Issue
Block a user