ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-06-02 16:26:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pratik/otel-phase3-tx-tracing' into pratik/otel-phase4-consensus-tracing

Browse Source
This commit is contained in:
Pratik Mankawde
2026-05-13 16:01:13 +01:00
parent c8674d61b8 e339ba1f6b
commit 7eeddd3ad9
12 changed files with 199 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/build/telemetry.md vendored
View File

@@ -185,15 +185,15 @@ Traced RPC operations produce a span hierarchy like:
```
rpc.request
└── rpc.command.server_info (xrpl.rpc.command=server_info, xrpl.rpc.status=success)
└── rpc.command.server_info (command=server_info, rpc_status=success)
```
Each span includes attributes:
- `xrpl.rpc.command` — the RPC method name
- `xrpl.rpc.version` — API version
- `xrpl.rpc.role` — `admin` or `user`
- `xrpl.rpc.status` — `success` or `error`
- `command` — the RPC method name
- `version` — API version
- `rpc_role` — `admin` or `user`
- `rpc_status` — `success` or `error`
## Running Tests

4
include/xrpl/telemetry/SpanGuard.h
View File

@@ -53,7 +53,7 @@
auto span = SpanGuard::span(
TraceCategory::Rpc, rpc_span::prefix::command, "submit");
span.setAttribute(rpc_span::attr::command, "submit");
span.setAttribute(rpc_span::attr::status, rpc_span::val::success);
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success);
// span ended automatically on scope exit
@endcode
@@ -86,7 +86,7 @@
TraceCategory::Rpc, rpc_span::prefix::rpc, "request");
if (span) {
// expensive attribute computation only when active
span.setAttribute(rpc_span::attr::payloadSize, computeSize());
span.setAttribute(rpc_span::attr::requestPayloadSize, computeSize());
}
@endcode

25
include/xrpl/telemetry/SpanNames.h
View File

@@ -16,9 +16,12 @@
* concatenation support. boost::static_string is not constexpr.
* StaticStr<N> exists specifically for compile-time dot-join composition.
*
* Naming conventions follow OpenTelemetry semantic conventions:
* - Attribute keys: "xrpl.<subsystem>.<field>"
* - Span prefixes: "<subsystem>[.<component>]"
* Naming conventions (see spec 2026-05-13-span-attr-naming-design):
* - Per-span attribute keys: bare field name (span name carries the domain).
* - Collision qualifier: <domain>_<field> when bare name collides across
* domains or with OTel reserved `status` (e.g. rpc_status, grpc_status).
* - Resource attribute keys: xrpl.<subsystem>.<field> (process-identity).
* - Span prefixes: <subsystem>[.<component>].
*/
#include <cstddef>
@@ -98,14 +101,22 @@ inline constexpr auto link = makeStr("link");
namespace attr {
inline constexpr auto networkId = join(join(seg::xrpl, seg::network), makeStr("id"));
inline constexpr auto networkType = join(join(seg::xrpl, seg::network), makeStr("type"));
inline constexpr auto linkType = join(join(seg::xrpl, seg::link), makeStr("type"));
inline constexpr auto linkType = makeStr("link_type");
/// Node health attributes (cross-cutting, used by RPC/consensus/tx spans).
/// Node health attributes — RESOURCE-ONLY (process identity, not per-span).
/// Set at Tracer init via resource::Resource::Create and refreshed on state
/// transitions. Do NOT use with span.setAttribute().
inline constexpr auto xrplNode = join(seg::xrpl, makeStr("node"));
/// "xrpl.node.amendment_blocked"
/// "xrpl.node.amendment_blocked" — resource attribute key.
inline constexpr auto nodeAmendmentBlocked = join(xrplNode, makeStr("amendment_blocked"));
/// "xrpl.node.server_state"
/// "xrpl.node.server_state" — resource attribute key.
inline constexpr auto nodeServerState = join(xrplNode, makeStr("server_state"));
/// Canonical shared attrs (rule 5 — kept xrpl.<domain>.* form).
/// Defined once here, aliased by domain-specific headers.
inline constexpr auto txHash = join(join(seg::xrpl, seg::tx), makeStr("hash"));
inline constexpr auto peerId = join(join(seg::xrpl, seg::peer), makeStr("id"));
inline constexpr auto ledgerSeq = join(join(seg::xrpl, seg::ledger), makeStr("seq"));
} // namespace attr
// ===== Shared attribute values =============================================

16
src/xrpld/app/main/GrpcSpanNames.h
View File

@@ -11,7 +11,7 @@
* +-------------------------------------------------------+
* | grpc.request |
* | CallData::process(coro) |
* | attrs: method, role, status |
* | attrs: method, grpc_role, grpc_status |
* +-------------------------------------------------------+
*
* Unlike the HTTP/WS RPC path, gRPC has a flat single-span structure
@@ -38,14 +38,12 @@ inline constexpr auto request = makeStr("request");
// ===== Attribute keys ======================================================
namespace attr {
inline constexpr auto xrplGrpc = join(seg::xrpl, makeStr("grpc"));
/// "xrpl.grpc.method"
inline constexpr auto method = join(xrplGrpc, makeStr("method"));
/// "xrpl.grpc.role"
inline constexpr auto role = join(xrplGrpc, makeStr("role"));
/// "xrpl.grpc.status"
inline constexpr auto status = join(xrplGrpc, makeStr("status"));
/// "method" — gRPC method name (e.g. GetLedger).
inline constexpr auto method = makeStr("method");
/// "grpc_role" — Domain-qualified: collides with rpc_role.
inline constexpr auto grpcRole = makeStr("grpc_role");
/// "grpc_status" — Domain-qualified: avoids OTel reserved span status.
inline constexpr auto grpcStatus = makeStr("grpc_status");
} // namespace attr
// ===== Attribute values ====================================================

2
src/xrpld/app/misc/NetworkOPs.cpp
View File

@@ -1325,7 +1325,7 @@ NetworkOPsImp::processTransaction(
{
using namespace telemetry;
auto span = std::make_shared<SpanGuard>(txProcessSpan(transaction->getID()));
span->setAttribute(tx_span::attr::hash, to_string(transaction->getID()).c_str());
span->setAttribute(tx_span::attr::txHash, to_string(transaction->getID()).c_str());
span->setAttribute(tx_span::attr::local, bLocal);
auto ev = m_job_queue.makeLoadEvent(jtTXN_PROC, "ProcessTXN");

31
src/xrpld/app/misc/TxSpanNames.h
View File

@@ -41,25 +41,20 @@ inline constexpr auto process = join(prefix::tx, op::process);
// ===== Attribute keys ======================================================
namespace attr {
inline constexpr auto xrplTx = join(seg::xrpl, seg::tx);
/// Canonical shared constants (defined in SpanNames.h).
using ::xrpl::telemetry::attr::peerId;
using ::xrpl::telemetry::attr::txHash;
/// "xrpl.tx.hash"
inline constexpr auto hash = join(xrplTx, makeStr("hash"));
/// "xrpl.tx.local"
inline constexpr auto local = join(xrplTx, makeStr("local"));
/// "xrpl.tx.path"
inline constexpr auto path = join(xrplTx, makeStr("path"));
/// "xrpl.tx.suppressed"
inline constexpr auto suppressed = join(xrplTx, makeStr("suppressed"));
/// "xrpl.tx.status"
inline constexpr auto status = join(xrplTx, makeStr("status"));
inline constexpr auto xrplPeer = join(seg::xrpl, seg::peer);
/// "xrpl.peer.id"
inline constexpr auto peerId = join(xrplPeer, makeStr("id"));
/// "xrpl.peer.version"
inline constexpr auto peerVersion = join(xrplPeer, makeStr("version"));
/// "local" — whether tx originated locally.
inline constexpr auto local = makeStr("local");
/// "path" — sync or async processing path.
inline constexpr auto path = makeStr("path");
/// "suppressed" — whether tx was suppressed as duplicate.
inline constexpr auto suppressed = makeStr("suppressed");
/// "tx_status" — domain-qualified (collides with rpc_status, txq_status).
inline constexpr auto txStatus = makeStr("tx_status");
/// "peer_version" — version of peer that sent the tx.
inline constexpr auto peerVersion = makeStr("peer_version");
} // namespace attr
// ===== Attribute values ====================================================

44
src/xrpld/app/misc/detail/TxQSpanNames.h
View File

@@ -71,30 +71,28 @@ inline constexpr auto cleanup = makeStr("cleanup");
// ===== Attribute keys ======================================================
namespace attr {
inline constexpr auto xrplTxq = join(seg::xrpl, makeStr("txq"));
/// Canonical shared constants (defined in SpanNames.h).
using ::xrpl::telemetry::attr::ledgerSeq;
using ::xrpl::telemetry::attr::txHash;
/// "xrpl.txq.tx_hash"
inline constexpr auto txHash = join(xrplTxq, makeStr("tx_hash"));
/// "xrpl.txq.status"
inline constexpr auto status = join(xrplTxq, makeStr("status"));
/// "xrpl.txq.fee_level_paid"
inline constexpr auto feeLevelPaid = join(xrplTxq, makeStr("fee_level_paid"));
/// "xrpl.txq.required_fee_level"
inline constexpr auto requiredFeeLevel = join(xrplTxq, makeStr("required_fee_level"));
/// "xrpl.txq.queue_size"
inline constexpr auto queueSize = join(xrplTxq, makeStr("queue_size"));
/// "xrpl.txq.ledger_changed"
inline constexpr auto ledgerChanged = join(xrplTxq, makeStr("ledger_changed"));
/// "xrpl.txq.ledger_seq"
inline constexpr auto ledgerSeq = join(xrplTxq, makeStr("ledger_seq"));
/// "xrpl.txq.expired_count"
inline constexpr auto expiredCount = join(xrplTxq, makeStr("expired_count"));
/// "xrpl.txq.ter_code"
inline constexpr auto terCode = join(xrplTxq, makeStr("ter_code"));
/// "xrpl.txq.retries_remaining"
inline constexpr auto retriesRemaining = join(xrplTxq, makeStr("retries_remaining"));
/// "xrpl.txq.num_cleared"
inline constexpr auto numCleared = join(xrplTxq, makeStr("num_cleared"));
/// "txq_status" — domain-qualified (collides with tx_status, rpc_status).
inline constexpr auto txqStatus = makeStr("txq_status");
/// "fee_level_paid" — fee level paid by queued tx.
inline constexpr auto feeLevelPaid = makeStr("fee_level_paid");
/// "required_fee_level" — minimum fee level for inclusion.
inline constexpr auto requiredFeeLevel = makeStr("required_fee_level");
/// "queue_size" — current TxQ depth.
inline constexpr auto queueSize = makeStr("queue_size");
/// "ledger_changed" — whether ledger changed since last attempt.
inline constexpr auto ledgerChanged = makeStr("ledger_changed");
/// "expired_count" — number of expired entries cleared.
inline constexpr auto expiredCount = makeStr("expired_count");
/// "ter_code" — transaction engine result code.
inline constexpr auto terCode = makeStr("ter_code");
/// "retries_remaining" — retries left before discard.
inline constexpr auto retriesRemaining = makeStr("retries_remaining");
/// "num_cleared" — entries cleared in batch.
inline constexpr auto numCleared = makeStr("num_cleared");
} // namespace attr
// ===== Attribute values ====================================================

9
src/xrpld/overlay/detail/PeerImp.cpp
View File

@@ -1445,10 +1445,15 @@ PeerImp::handleTransaction(
using namespace telemetry;
auto span = std::make_shared<SpanGuard>(txReceiveSpan(txID, *m));
span->setAttribute(tx_span::attr::hash, to_string(txID).c_str());
span->setAttribute(tx_span::attr::txHash, to_string(txID).c_str());
span->setAttribute(tx_span::attr::peerId, static_cast<int64_t>(id_));
if (auto const version = getVersion(); !version.empty())
span->setAttribute(tx_span::attr::peerVersion, version.c_str());
// Set defaults for conditional attributes so they are always present
// on the span. The suppressed path overrides these when the
// transaction has already been seen via HashRouter.
span->setAttribute(tx_span::attr::suppressed, false);
span->setAttribute(tx_span::attr::txStatus, "new");
// Charge strongly for attempting to relay a txn with tfInnerBatchTxn
// LCOV_EXCL_START
@@ -1486,7 +1491,7 @@ PeerImp::handleTransaction(
// we have seen this transaction recently
if (any(flags & HashRouterFlags::BAD))
{
span->setAttribute(tx_span::attr::status, tx_span::val::knownBad);
span->setAttribute(tx_span::attr::txStatus, tx_span::val::knownBad);
fee_.update(Resource::feeUselessData, "known bad");
JLOG(p_journal_.debug()) << "Ignoring known bad tx " << txID;
}

36
src/xrpld/rpc/detail/PathFindSpanNames.h
View File

@@ -63,24 +63,24 @@ inline constexpr auto rank = makeStr("rank");
// ===== Attribute keys ======================================================
namespace attr {
inline constexpr auto xrplPathfind = join(seg::xrpl, makeStr("pathfind"));
/// "xrpl.pathfind.source_account"
inline constexpr auto sourceAccount = join(xrplPathfind, makeStr("source_account"));
/// "xrpl.pathfind.dest_account"
inline constexpr auto destAccount = join(xrplPathfind, makeStr("dest_account"));
/// "xrpl.pathfind.fast"
inline constexpr auto fast = join(xrplPathfind, makeStr("fast"));
/// "xrpl.pathfind.search_level"
inline constexpr auto searchLevel = join(xrplPathfind, makeStr("search_level"));
/// "xrpl.pathfind.num_complete_paths"
inline constexpr auto numCompletePaths = join(xrplPathfind, makeStr("num_complete_paths"));
/// "xrpl.pathfind.num_paths"
inline constexpr auto numPaths = join(xrplPathfind, makeStr("num_paths"));
/// "xrpl.pathfind.num_requests"
inline constexpr auto numRequests = join(xrplPathfind, makeStr("num_requests"));
/// "xrpl.pathfind.ledger_index"
inline constexpr auto ledgerIndex = join(xrplPathfind, makeStr("ledger_index"));
/// "source_account" — originating account for path search.
inline constexpr auto sourceAccount = makeStr("source_account");
/// "dest_account" — destination account.
inline constexpr auto destAccount = makeStr("dest_account");
/// "fast" — whether fast pathfinding mode enabled.
inline constexpr auto fast = makeStr("fast");
/// "search_level" — depth of graph exploration.
inline constexpr auto searchLevel = makeStr("search_level");
/// "num_complete_paths" — complete paths found.
inline constexpr auto numCompletePaths = makeStr("num_complete_paths");
/// "num_paths" — total paths returned.
inline constexpr auto numPaths = makeStr("num_paths");
/// "num_requests" — active path requests.
inline constexpr auto numRequests = makeStr("num_requests");
/// "xrpl.pathfind.ledger_index" — kept qualified (rule 5): pathfind target
/// ledger is distinct from xrpl.ledger.seq.
inline constexpr auto ledgerIndex =
join(join(seg::xrpl, makeStr("pathfind")), makeStr("ledger_index"));
} // namespace attr
} // namespace xrpl::telemetry::pathfind_span

8
src/xrpld/rpc/detail/RPCHandler.cpp
View File

@@ -168,11 +168,9 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object&
span.setAttribute(rpc_span::attr::command, name.c_str());
span.setAttribute(rpc_span::attr::version, static_cast<int64_t>(context.apiVersion));
span.setAttribute(
rpc_span::attr::role,
rpc_span::attr::rpcRole,
context.role == Role::ADMIN ? std::string_view(rpc_span::val::admin)
: std::string_view(rpc_span::val::user));
span.setAttribute(attr::nodeAmendmentBlocked, context.app.getOPs().isAmendmentBlocked());
span.setAttribute(attr::nodeServerState, context.app.getOPs().strOperatingMode());
static std::atomic<std::uint64_t> requestId{0};
auto& perfLog = context.app.getPerfLog();
@@ -189,7 +187,7 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object&
JLOG(context.j.debug()) << "RPC call " << name << " completed in "
<< ((end - start).count() / 1000000000.0) << "seconds";
perfLog.rpcFinish(name, curId);
span.setAttribute(rpc_span::attr::status, rpc_span::val::success);
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success);
return ret;
}
catch (std::exception& e)
@@ -197,7 +195,7 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object&
perfLog.rpcError(name, curId);
JLOG(context.j.info()) << "Caught throw: " << e.what();
span.recordException(e);
span.setAttribute(rpc_span::attr::status, rpc_span::val::error);
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error);
if (context.loadType == Resource::feeReferenceRPC)
context.loadType = Resource::feeExceptionRPC;

115
src/xrpld/rpc/detail/RpcSpanNames.h
View File

@@ -14,8 +14,99 @@
* auto span = SpanGuard::span(
* TraceCategory::Rpc, rpc_span::prefix::command, "submit");
* span.setAttribute(rpc_span::attr::command, "submit");
* span.setAttribute(rpc_span::attr::status, rpc_span::val::success);
* span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success);
* @endcode
*
* Span hierarchy (automatic nesting via OTel thread-local context):
*
* HTTP JSON-RPC path (single request):
*
* +-------------------------------------------------------+
* | rpc.http_request |
* | ServerHandler::processSession(Session) |
* | |
* | +--------------------------------------------------+ |
* | | rpc.process | |
* | | ServerHandler::processRequest() | |
* | | | |
* | | +---------------------------------------------+ | |
* | | | rpc.command.{name} | | |
* | | | RPC::callMethod() | | |
* | | | attrs: command, version, rpc_role, rpc_status | | |
* | | +---------------------------------------------+ | |
* | +--------------------------------------------------+ |
* +-------------------------------------------------------+
*
* HTTP batch path (multiple commands per request):
*
* +-------------------------------------------------------+
* | rpc.http_request |
* | |
* | +--------------------------------------------------+ |
* | | rpc.process | |
* | | | |
* | | +------------------+ +------------------+ | |
* | | | rpc.command.{a} | | rpc.command.{b} | ... | |
* | | +------------------+ +------------------+ | |
* | +--------------------------------------------------+ |
* +-------------------------------------------------------+
*
* WebSocket path:
*
* +-------------------------------------------------------+
* | rpc.ws_message |
* | ServerHandler::processSession(WSSession) |
* | |
* | +--------------------------------------------------+ |
* | | rpc.command.{name} | |
* | | RPC::callMethod() | |
* | | attrs: command, version, rpc_role, rpc_status | |
* | +--------------------------------------------------+ |
* +-------------------------------------------------------+
*
* WebSocket error paths:
*
* +-------------------------------------------------------+
* | rpc.ws_message (error: invalid_json) |
* | ServerHandler::onWSMessage() — parse failure |
* +-------------------------------------------------------+
*
* +-------------------------------------------------------+
* | rpc.ws_upgrade |
* | ServerHandler::onHandoff() — upgrade try/catch |
* +-------------------------------------------------------+
*
* Command dispatch error path:
*
* +-------------------------------------------------------+
* | rpc.command.{name} (error: too_busy/unknown/etc) |
* | RPC::doCommand() — fillHandler() rejection |
* +-------------------------------------------------------+
*
* gRPC path (see GrpcSpanNames.h for constants):
*
* +-------------------------------------------------------+
* | grpc.request |
* | CallData::process(coro) |
* | attrs: method, grpc_status |
* +-------------------------------------------------------+
*
* Covered paths:
* - HTTP JSON-RPC (single and batch requests)
* - WebSocket RPC commands
* - WebSocket message parse errors (invalid JSON, oversized)
* - WebSocket upgrade failures (protocol handshake errors)
* - Admin CLI (connects via HTTP internally)
* - Command dispatch rejections (unknown cmd, too busy, no perm)
* - gRPC endpoints (GetLedger, GetLedgerData, GetLedgerDiff,
* GetLedgerEntry)
* - Command execution: timing, success/failure, exceptions
* - Per-command attributes: name, API version, rpc_role, rpc_status
*
* Known gaps (not yet instrumented):
* - Early validation errors in processRequest() before rpc.process
* span (malformed JSON, auth failures, oversized requests)
* - Subscription push notifications (server-initiated, not RPC)
*/
#include <xrpl/telemetry/SpanNames.h>
@@ -43,18 +134,16 @@ inline constexpr auto process = makeStr("process");
// ===== Attribute keys ======================================================
namespace attr {
inline constexpr auto xrplRpc = join(seg::xrpl, seg::rpc);
/// "xrpl.rpc.command"
inline constexpr auto command = join(xrplRpc, makeStr("command"));
/// "xrpl.rpc.version"
inline constexpr auto version = join(xrplRpc, makeStr("version"));
/// "xrpl.rpc.role"
inline constexpr auto role = join(xrplRpc, makeStr("role"));
/// "xrpl.rpc.status"
inline constexpr auto status = join(xrplRpc, makeStr("status"));
/// "xrpl.rpc.payload_size"
inline constexpr auto payloadSize = join(xrplRpc, makeStr("payload_size"));
/// "command" — RPC method name.
inline constexpr auto command = makeStr("command");
/// "version" — api_version per request.
inline constexpr auto version = makeStr("version");
/// "rpc_role" — admin|user. Domain-qualified: collides with grpc_role.
inline constexpr auto rpcRole = makeStr("rpc_role");
/// "rpc_status" — success|error. Domain-qualified: avoids OTel reserved span status.
inline constexpr auto rpcStatus = makeStr("rpc_status");
/// "request_payload_size" — bytes of inbound request payload.
inline constexpr auto requestPayloadSize = makeStr("request_payload_size");
} // namespace attr
// ===== Attribute values ====================================================

4
src/xrpld/rpc/detail/ServerHandler.cpp
View File

@@ -513,7 +513,7 @@ ServerHandler::processSession(
JLOG(m_journal.error()) << "Exception while processing WS: " << ex.what() << "\n"
<< "Input JSON: " << Json::Compact{Json::Value{jv}};
span.recordException(ex);
span.setAttribute(rpc_span::attr::status, rpc_span::val::error);
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error);
// LCOV_EXCL_STOP
}
@@ -904,7 +904,7 @@ ServerHandler::processRequest(
<< "Internal error : " << ex.what()
<< " when processing request: " << Json::Compact{Json::Value{params}};
span.recordException(ex);
span.setAttribute(rpc_span::attr::status, rpc_span::val::error);
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error);
// LCOV_EXCL_STOP
}