Merge branch 'pratik/otel-phase10-workload-validation' into pratik/otel-sync-diagnostics

# Conflicts:
#	src/xrpld/telemetry/MetricsRegistry.cpp

include/xrpl/beast/insight/Collector.h

@@ -6,6 +6,7 @@
 #include <xrpl/beast/insight/Hook.h>
 #include <xrpl/beast/insight/HookImpl.h>
 #include <xrpl/beast/insight/Meter.h>
+#include <xrpl/beast/insight/Unit.h>
 
 #include <memory>
 #include <string>
@@ -103,6 +104,24 @@ public:
     virtual Event
     makeEvent(std::string const& name) = 0;
 
+    /**
+     * Create an event whose samples measure `unit` rather than milliseconds.
+     *
+     * The default delegates to the millisecond overload, so a collector that
+     * cannot act on a unit keeps working unchanged -- the StatsD collector
+     * relies on this. Collectors that map a unit onto an export format, such
+     * as the OTel collector, override it.
+     *
+     * @param name Metric name, already prefixed if it came through a Group.
+     * @param unit What the samples measure.
+     */
+    virtual Event
+    makeEvent(std::string const& name, Unit unit)
+    {
+        (void)unit;
+        return makeEvent(name);
+    }
+
     Event
     makeEvent(std::string const& prefix, std::string const& name)
     {
@@ -110,6 +129,14 @@ public:
             return makeEvent(name);
         return makeEvent(prefix + "." + name);
     }
+
+    Event
+    makeEvent(std::string const& prefix, std::string const& name, Unit unit)
+    {
+        if (prefix.empty())
+            return makeEvent(name, unit);
+        return makeEvent(prefix + "." + name, unit);
+    }
     /** @} */
 
     /**

include/xrpl/beast/insight/Event.h

@@ -51,6 +51,24 @@ public:
             impl_->notify(ceil<value_type>(value));
     }
 
+    /**
+     * Push a raw integral sample.
+     *
+     * For Events whose unit is not a duration, such as a byte count. The
+     * value is stored in the same integral field the duration overload uses
+     * and is interpreted per the Event's unit by the backend.
+     *
+     * Prefer this over constructing an `Event::value_type` at the call site:
+     * wrapping a byte count in a `std::chrono::milliseconds` compiles, but
+     * reads as a duration to everything downstream.
+     */
+    void
+    notify(std::uint64_t value) const
+    {
+        if (impl_)
+            impl_->notify(value_type{value});
+    }
+
     [[nodiscard]] std::shared_ptr<EventImpl> const&
     impl() const
     {

include/xrpl/beast/insight/EventImpl.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <xrpl/beast/insight/Unit.h>
+
 #include <chrono>
 #include <memory>
 
@@ -10,11 +12,48 @@ class Event;
 class EventImpl : public std::enable_shared_from_this<EventImpl>
 {
 public:
+    /**
+     * The integral type every sample is stored as.
+     *
+     * Named for the common case -- durations -- and deliberately left as a
+     * duration type. Widening it would change the wire value of every
+     * existing StatsD timer, and metrics that need finer resolution than a
+     * whole millisecond use the OTel-native microsecond instruments instead.
+     * A sample whose unit() is not a duration is carried in the same integral
+     * field and interpreted per unit() by the backend.
+     */
     using value_type = std::chrono::milliseconds;
 
     virtual ~EventImpl() = 0;
     virtual void
     notify(value_type const& value) = 0;
+
+    /**
+     * @brief What this Event's samples measure. Fixed at construction.
+     *
+     * The OTel backend reads this to choose the instrument's declared unit
+     * and, through that, its bucket ladder. The StatsD backend ignores it.
+     */
+    [[nodiscard]] Unit
+    unit() const noexcept
+    {
+        return unit_;
+    }
+
+protected:
+    /**
+     * @param unit What the samples measure. Defaults to milliseconds so
+     *             existing implementations keep their behaviour unchanged.
+     */
+    explicit EventImpl(Unit unit = Unit::Millis) : unit_(unit)
+    {
+    }
+
+private:
+    /**
+     * What the samples measure; selects the export unit and bucket ladder.
+     */
+    Unit unit_;
 };
 
 }  // namespace beast::insight

include/xrpl/beast/insight/Unit.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include <cstdint>
+
+namespace beast::insight {
+
+/**
+ * @brief What an Event's samples measure.
+ *
+ * `Event` documents itself as carrying "a millisecond time, or other integral
+ * value", but both backends used to assume the first case: the OTel bridge
+ * declared every instrument with unit `ms`, and StatsD tagged every sample
+ * `|ms`. A size metric therefore exported under a `_milliseconds` name and
+ * inherited a latency bucket ladder, which censored a quarter of its samples
+ * and pinned its p95 to a constant.
+ *
+ * Naming the unit at creation time is what lets the OTel bridge pick both the
+ * instrument unit and the matching bucket ladder:
+ *
+ *     makeEvent("time", Unit::Millis) --> OTel unit "ms" --> millisecond ladder
+ *     makeEvent("size", Unit::Bytes)  --> OTel unit "By" --> byte ladder
+ *
+ * The StatsD backend deliberately ignores this and keeps emitting `|ms` for
+ * every Event. That path is retired here -- its UDP port is commented out of
+ * the compose file and the integration test fails if anything is listening on
+ * 8125 -- so changing its wire format would alter a legacy contract for no
+ * local benefit and with no way to verify it.
+ *
+ * @note Adding a member requires extending otelUnitCode(), which switches
+ *       exhaustively so a new member is a compile error rather than a silent
+ *       fallthrough to milliseconds.
+ */
+enum class Unit : std::uint8_t {
+    /**
+     * Whole milliseconds. The default, and what every duration Event uses.
+     */
+    Millis,
+
+    /**
+     * A byte count, such as a serialized response size.
+     */
+    Bytes
+};
+
+/**
+ * @brief The OTel (UCUM) unit code for a Unit.
+ *
+ * The collector's Prometheus exporter derives the exported metric-name suffix
+ * from this code, so `ms` yields `_milliseconds` and `By` yields `_bytes`. It
+ * is also the key the histogram views match on, which is how each unit gets
+ * its own bucket ladder.
+ *
+ * @param unit The unit to translate.
+ * @return A static, null-terminated UCUM code.
+ */
+constexpr char const*
+otelUnitCode(Unit unit) noexcept
+{
+    switch (unit)
+    {
+        case Unit::Bytes:
+            return "By";
+        case Unit::Millis:
+            break;
+    }
+    return "ms";
+}
+
+/**
+ * @brief Human-readable description for an instrument of this unit.
+ *
+ * Exported alongside the metric, so this is the text an operator reads in a
+ * metric catalogue. A byte-valued instrument that describes itself as a
+ * duration is exactly the confusion this whole type exists to remove, so the
+ * description is derived from the unit rather than written out at each
+ * instrument site.
+ *
+ * @param unit The unit to describe.
+ * @return A static, null-terminated description.
+ */
+constexpr char const*
+otelUnitDescription(Unit unit) noexcept
+{
+    switch (unit)
+    {
+        case Unit::Bytes:
+            return "Size in bytes";
+        case Unit::Millis:
+            break;
+    }
+    return "Duration in ms";
+}
+
+}  // namespace beast::insight

include/xrpl/telemetry/HistogramBuckets.h

@@ -0,0 +1,244 @@
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <span>
+#include <vector>
+
+namespace xrpl::telemetry::buckets {
+
+/**
+ * @file HistogramBuckets.h
+ * @brief Explicit histogram bucket edges for xrpld's OTel instruments.
+ *
+ * One header owns every ladder so a reviewer sees all of them at once and a
+ * test can assert their invariants. Before this existed the edges lived as
+ * file-local `namespace {}` constants, unreachable from any test, and they
+ * drifted apart.
+ *
+ * Why a ladder is worth this much care: when a quantile falls in the `+Inf`
+ * bucket, Prometheus returns the *second-highest* edge, not `+Inf`. A
+ * saturated histogram therefore reports a believable constant instead of an
+ * obvious error. The same trap exists at the bottom -- if nearly every
+ * sample lands in bucket 0, `histogram_quantile` interpolates inside it and
+ * invents a value. A ladder is correct only when its floor sits below the
+ * mass of the distribution and its ceiling above the tail.
+ *
+ *   sample --> [ SDK lower_bound over edges ] --> per-bucket counter
+ *                          |                              |
+ *                  edges come from                        v
+ *                    THIS header                     OTLP export
+ *                                                         |
+ *                                                         v
+ *                                        histogram_quantile() in Grafana
+ *
+ * Ladders are `std::array<double, N>` so they are constant-initialised and
+ * usable in a `static_assert`. The OTel SDK wants `std::vector<double>` in
+ * its aggregation config, so call toVector() at the registration site
+ * rather than storing vectors here.
+ *
+ * Example -- register a view with the millisecond ladder:
+ * @code
+ * auto config = std::make_shared<HistogramAggregationConfig>();
+ * config->boundaries_ = buckets::toVector(buckets::kMillisecondBuckets);
+ * @endcode
+ *
+ * Example -- the edge case that motivated a second ladder. An Event whose
+ * samples are sizes rather than durations must not borrow a latency ladder,
+ * or a quarter of its samples land in `+Inf` and every quantile reads back
+ * as the top edge:
+ * @code
+ * config->boundaries_ = buckets::toVector(buckets::kByteBuckets);
+ * @endcode
+ *
+ * @note Thread safety: every member is `constexpr` and immutable, so
+ *       reading them from any thread is safe. toVector() allocates and is
+ *       meant for start-up registration paths, never for a record path.
+ * @note Limitation: changing a ladder changes the exported series count and
+ *       ends bucket comparability across the change -- existing series keep
+ *       their old `le` values, so panels show a break at restart. Grafana
+ *       Cloud bills per series, so re-measure the series count after any
+ *       edit here.
+ */
+
+/**
+ * Bucket edges, in milliseconds, for whole-millisecond `beast::insight`
+ * Events: job queue wait and run times, io latency, RPC time, pathfinding.
+ *
+ * **This list must contain every representable edge of the collector's
+ * spanmetrics ladder, and may extend above it.** Agreement over the shared
+ * range is deliberate: it lets a span-derived latency panel and a native
+ * histogram panel be read on the same scale. It was specified that way
+ * originally, then silently broken when the collector ladder alone was
+ * extended, which left this side capped at 5 s while spans reached 30 s and
+ * censored every quantile above 5 s. `check_bucket_parity.py` now enforces
+ * the containment -- add a collector edge, add it here too.
+ *
+ * The sub-millisecond edges the collector carries (0.01 to 0.5 ms) are
+ * deliberately absent. `beast::insight::Event` rounds every duration up to
+ * a whole millisecond before it reaches the histogram, so those edges would
+ * collect nothing. Metrics that genuinely need finer resolution belong on
+ * the microsecond ladder, on the OTel-native path.
+ *
+ * The 60 s and 120 s edges exceed the collector's 30 s top on purpose,
+ * because jobs outlive spans: the updatepaths job type was measured
+ * averaging about 60 s, so a 30 s ceiling would censor its quantiles just
+ * as 5 s censors them today. All these Events share one ladder, so its
+ * ceiling has to cover the slowest member rather than the typical one.
+ *
+ * The 2, 3 and 4 s edges resolve second-scale work that previously had to
+ * interpolate across a single four-second-wide bucket.
+ */
+inline constexpr std::array kMillisecondBuckets{
+    1.0,
+    5.0,
+    10.0,
+    25.0,
+    50.0,
+    100.0,
+    250.0,
+    500.0,
+    1'000.0,
+    2'000.0,
+    3'000.0,
+    4'000.0,
+    5'000.0,
+    10'000.0,
+    30'000.0,
+    60'000.0,
+    120'000.0};
+
+/**
+ * Bucket edges, in bytes, for `beast::insight` Events whose samples are
+ * sizes rather than durations. Currently only the RPC response size.
+ *
+ * Placed from the measured distribution rather than from a guess about how
+ * large a response could theoretically be. Measured over 24 h: mean 2131 B,
+ * half of all responses under 1 kB, three quarters under 5 kB. The tail
+ * above 5 kB has a mean of at most 7538 B, which bounds p99 near 80 kB and
+ * p99.75 below 256 kB.
+ *
+ * So the resolution belongs between 512 B and 64 kB, where the
+ * distribution actually turns, and two further edges are ample headroom.
+ * Spending edges at the megabyte scale would cost cardinality on a range
+ * nothing measured occupies. If a genuinely multi-megabyte response ever
+ * shows up in the top bucket, extend this -- but extend it on evidence.
+ */
+inline constexpr std::array kByteBuckets{
+    512.0,
+    1'024.0,
+    2'048.0,
+    4'096.0,
+    8'192.0,
+    16'384.0,
+    32'768.0,
+    65'536.0,
+    262'144.0,
+    1'048'576.0};
+
+/**
+ * Bucket edges, in microseconds, for the OTel-native duration instruments
+ * created directly on MetricsRegistry: job queue wait and run times, RPC
+ * method latency, and GetObject lookup latency.
+ *
+ * The edges from 1 to 1000 us are the ones that matter most. An earlier
+ * version of this ladder started at 100 us, which sat ABOVE the mass of every
+ * instrument using it: 99.3% of job_queued_us samples, 92.5% of
+ * job_running_us and 90.4% of getobject_lookup_us fell in that first bucket.
+ * `histogram_quantile` then interpolated inside bucket 0 and returned the
+ * boundary scaled by the requested quantile -- p75/p95/p99 of job_queued_us
+ * read 75.5/95.7/99.7 us, which is arithmetic on the bucket edge, not a
+ * latency. A warm nodestore read is around 1.5 us, so single-microsecond
+ * resolution is not excessive here.
+ *
+ * The upper edges reach a minute so multi-second stalls stay measurable. The
+ * SDK's own default ladder stops at 10,000, which every one of these
+ * instruments exceeds during catch-up.
+ */
+inline constexpr std::array kMicrosecondBuckets{
+    1.0,
+    2.0,
+    5.0,
+    10.0,
+    25.0,
+    50.0,
+    100.0,
+    250.0,
+    500.0,
+    1'000.0,
+    5'000.0,
+    25'000.0,
+    100'000.0,
+    500'000.0,
+    1'000'000.0,
+    5'000'000.0,
+    10'000'000.0,
+    30'000'000.0,
+    60'000'000.0};
+
+/**
+ * Bucket edges for the GetObject request object count.
+ *
+ * Counts run from 1 to the hard reply cap (kHardMaxReplyNodes, 12288). The
+ * honest sync path asks for at most 8 objects, so the low edges are
+ * fine-grained; the upper ones follow the charge size bands up to the cap.
+ * Because the top edge IS the hard cap, this ladder cannot saturate.
+ */
+inline constexpr std::array
+    kObjectCountBuckets{1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0};
+
+/**
+ * Bucket edges for the GetObject resource charge.
+ *
+ * Charges span 0 (the free tier) to roughly 99k for a full-size all-miss
+ * request. The edges bracket the two thresholds that decide a peer's fate --
+ * the warning threshold at 5000 and the drop threshold at 25000 -- so a
+ * dashboard can show how close charges run to each.
+ */
+inline constexpr std::array
+    kChargeBuckets{0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0};
+
+/**
+ * @brief Check that a ladder is strictly ascending and non-negative.
+ *
+ * The SDK places a sample with `std::lower_bound` over the edges, which
+ * silently misbuckets when edges repeat or descend. Checking at compile
+ * time makes that class of typo impossible to ship.
+ *
+ * @param ladder Bucket upper bounds to check.
+ * @return true when the ladder is non-empty, starts at or above zero, and
+ *         every later edge is strictly greater than its predecessor.
+ */
+constexpr bool
+isAscendingNonNegative(std::span<double const> ladder) noexcept
+{
+    if (ladder.empty() || ladder.front() < 0.0)
+        return false;
+
+    for (std::size_t i = 1; i < ladder.size(); ++i)
+    {
+        if (!(ladder[i] > ladder[i - 1]))
+            return false;
+    }
+    return true;
+}
+
+static_assert(isAscendingNonNegative(kMillisecondBuckets));
+static_assert(isAscendingNonNegative(kByteBuckets));
+static_assert(isAscendingNonNegative(kMicrosecondBuckets));
+static_assert(isAscendingNonNegative(kObjectCountBuckets));
+static_assert(isAscendingNonNegative(kChargeBuckets));
+
+/**
+ * @brief Copy a ladder into the `std::vector<double>` the OTel SDK wants.
+ *
+ * @param ladder Bucket upper bounds.
+ * @return A vector holding the same edges in the same order.
+ */
+inline std::vector<double>
+toVector(std::span<double const> ladder)
+{
+    return std::vector<double>(ladder.begin(), ladder.end());
+}
+
+}  // namespace xrpl::telemetry::buckets