From ba2714fa22efdd7101afacf47f07e16281c1895f Mon Sep 17 00:00:00 2001 From: Nik Bougalis Date: Fri, 21 Jun 2019 12:10:06 -0700 Subject: [PATCH] Make protocol message counters more granular: A running instance of the server tracks the number of protocol messages and the number of bytes it sends and receives. This commit makes the counters more granular, allowing server operators to better track and understand bandwidth usage. --- src/ripple/overlay/Message.h | 4 +- src/ripple/overlay/impl/Message.cpp | 3 +- src/ripple/overlay/impl/OverlayImpl.cpp | 30 ++-- src/ripple/overlay/impl/TrafficCount.cpp | 172 +++++++++--------- src/ripple/overlay/impl/TrafficCount.h | 212 +++++++++++++++++------ 5 files changed, 260 insertions(+), 161 deletions(-) diff --git a/src/ripple/overlay/Message.h b/src/ripple/overlay/Message.h index 6d8eefd0d..498f70ca8 100644 --- a/src/ripple/overlay/Message.h +++ b/src/ripple/overlay/Message.h @@ -75,7 +75,7 @@ public: } /** Get the traffic category */ - int + std::size_t getCategory () const { return mCategory; @@ -167,7 +167,7 @@ private: std::vector mBuffer; - int mCategory; + std::size_t mCategory; }; } diff --git a/src/ripple/overlay/impl/Message.cpp b/src/ripple/overlay/impl/Message.cpp index 122ab998d..c73dee6c1 100644 --- a/src/ripple/overlay/impl/Message.cpp +++ b/src/ripple/overlay/impl/Message.cpp @@ -39,8 +39,7 @@ Message::Message (::google::protobuf::Message const& message, int type) message.SerializeToArray (&mBuffer [Message::kHeaderBytes], messageBytes); } - mCategory = safe_cast(TrafficCount::categorize - (message, type, false)); + mCategory = TrafficCount::categorize(message, type, false); } bool Message::operator== (Message const& other) const diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index c9c8be22e..d3136b9a4 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -593,26 +593,18 @@ void OverlayImpl::onWrite (beast::PropertyStream::Map& stream) { beast::PropertyStream::Set set ("traffic", stream); - auto stats = m_traffic.getCounts(); - for (auto& i : stats) + auto const stats = m_traffic.getCounts(); + for (auto const& i : stats) { - if (! i.second.messagesIn && ! i.second.messagesOut) - continue; - - beast::PropertyStream::Map item (set); - item["category"] = i.first; - item["bytes_in"] = - beast::lexicalCast - (i.second.bytesIn.load()); - item["messages_in"] = - beast::lexicalCast - (i.second.messagesIn.load()); - item["bytes_out"] = - beast::lexicalCast - (i.second.bytesOut.load()); - item["messages_out"] = - beast::lexicalCast - (i.second.messagesOut.load()); + if (i) + { + beast::PropertyStream::Map item(set); + item["category"] = i.name; + item["bytes_in"] = std::to_string(i.bytesIn.load()); + item["messages_in"] = std::to_string(i.messagesIn.load()); + item["bytes_out"] = std::to_string(i.bytesOut.load()); + item["messages_out"] = std::to_string(i.messagesOut.load()); + } } } diff --git a/src/ripple/overlay/impl/TrafficCount.cpp b/src/ripple/overlay/impl/TrafficCount.cpp index fc160e04e..abb4789db 100644 --- a/src/ripple/overlay/impl/TrafficCount.cpp +++ b/src/ripple/overlay/impl/TrafficCount.cpp @@ -21,118 +21,128 @@ namespace ripple { -const char* TrafficCount::getName (category c) -{ - switch (c) - { - case category::CT_base: - return "overhead"; - case category::CT_overlay: - return "overlay"; - case category::CT_transaction: - return "transactions"; - case category::CT_proposal: - return "proposals"; - case category::CT_validation: - return "validations"; - case category::CT_get_ledger: - return "ledger_get"; - case category::CT_share_ledger: - return "ledger_share"; - case category::CT_get_trans: - return "transaction_set_get"; - case category::CT_share_trans: - return "transaction_set_share"; - case category::CT_unknown: - assert (false); - return "unknown"; - default: - assert (false); - return "truly_unknow"; - } -} - TrafficCount::category TrafficCount::categorize ( ::google::protobuf::Message const& message, int type, bool inbound) { if ((type == protocol::mtHELLO) || (type == protocol::mtPING) || - (type == protocol::mtCLUSTER) || (type == protocol::mtSTATUS_CHANGE)) - return TrafficCount::category::CT_base; + return TrafficCount::category::base; - if ((type == protocol::mtMANIFESTS) || - (type == protocol::mtENDPOINTS) || - (type == protocol::mtGET_SHARD_INFO) || - (type == protocol::mtSHARD_INFO) || - (type == protocol::mtGET_PEER_SHARD_INFO) || - (type == protocol::mtPEER_SHARD_INFO) || + if (type == protocol::mtCLUSTER) + return TrafficCount::category::cluster; + + if (type == protocol::mtMANIFESTS) + return TrafficCount::category::manifests; + + if ((type == protocol::mtENDPOINTS) || (type == protocol::mtPEERS) || (type == protocol::mtGET_PEERS)) - return TrafficCount::category::CT_overlay; + return TrafficCount::category::overlay; + + if ((type == protocol::mtGET_SHARD_INFO) || + (type == protocol::mtSHARD_INFO) || + (type == protocol::mtGET_PEER_SHARD_INFO) || + (type == protocol::mtPEER_SHARD_INFO)) + return TrafficCount::category::shards; if (type == protocol::mtTRANSACTION) - return TrafficCount::category::CT_transaction; + return TrafficCount::category::transaction; if (type == protocol::mtVALIDATION) - return TrafficCount::category::CT_validation; + return TrafficCount::category::validation; if (type == protocol::mtPROPOSE_LEDGER) - return TrafficCount::category::CT_proposal; + return TrafficCount::category::proposal; if (type == protocol::mtHAVE_SET) - return inbound ? TrafficCount::category::CT_get_trans : - TrafficCount::category::CT_share_trans; + return inbound ? + TrafficCount::category::get_set: + TrafficCount::category::share_set; + if (auto msg = dynamic_cast(&message)) { - auto msg = dynamic_cast - (&message); - if (msg) - { - // We have received ledger data - if (msg->type() == protocol::liTS_CANDIDATE) - return (inbound && !msg->has_requestcookie()) ? - TrafficCount::category::CT_get_trans : - TrafficCount::category::CT_share_trans; + if (msg->type() == protocol::liTS_CANDIDATE) return (inbound && !msg->has_requestcookie()) ? - TrafficCount::category::CT_get_ledger : - TrafficCount::category::CT_share_ledger; - } + TrafficCount::category::ld_tsc_get: + TrafficCount::category::ld_tsc_share; + + if (msg->type() == protocol::liTX_NODE) + return (inbound && !msg->has_requestcookie()) ? + TrafficCount::category::ld_txn_get : + TrafficCount::category::ld_txn_share; + + if (msg->type() == protocol::liAS_NODE) + return (inbound && !msg->has_requestcookie()) ? + TrafficCount::category::ld_asn_get : + TrafficCount::category::ld_asn_share; + + return (inbound && !msg->has_requestcookie()) ? + TrafficCount::category::ld_get : + TrafficCount::category::ld_share; } + if (auto msg = dynamic_cast(&message)) { - auto msg = - dynamic_cast - (&message); - if (msg) - { - if (msg->itype() == protocol::liTS_CANDIDATE) - return (inbound || msg->has_requestcookie()) ? - TrafficCount::category::CT_share_trans : - TrafficCount::category::CT_get_trans; + if (msg->itype() == protocol::liTS_CANDIDATE) return (inbound || msg->has_requestcookie()) ? - TrafficCount::category::CT_share_ledger : - TrafficCount::category::CT_get_ledger; - } + TrafficCount::category::gl_tsc_share : + TrafficCount::category::gl_tsc_get; + + if (msg->itype() == protocol::liTX_NODE) + return (inbound || msg->has_requestcookie()) ? + TrafficCount::category::gl_txn_share : + TrafficCount::category::gl_txn_get; + + if (msg->itype() == protocol::liAS_NODE) + return (inbound || msg->has_requestcookie()) ? + TrafficCount::category::gl_asn_share : + TrafficCount::category::gl_asn_get; + + return (inbound || msg->has_requestcookie()) ? + TrafficCount::category::gl_share : + TrafficCount::category::gl_get; } + if (auto msg = dynamic_cast(&message)) { - auto msg = - dynamic_cast - (&message); - if (msg) - { - // inbound queries and outbound responses are sharing - // outbound queries and inbound responses are getting + if (msg->type() == protocol::TMGetObjectByHash::otLEDGER) return (msg->query() == inbound) ? - TrafficCount::category::CT_share_ledger : - TrafficCount::category::CT_get_ledger; - } + TrafficCount::category::share_hash_ledger : + TrafficCount::category::get_hash_ledger; + + if (msg->type() == protocol::TMGetObjectByHash::otTRANSACTION) + return (msg->query() == inbound) ? + TrafficCount::category::share_hash_tx : + TrafficCount::category::get_hash_tx; + + if (msg->type() == protocol::TMGetObjectByHash::otTRANSACTION_NODE) + return (msg->query() == inbound) ? + TrafficCount::category::share_hash_txnode : + TrafficCount::category::get_hash_txnode; + + if (msg->type() == protocol::TMGetObjectByHash::otSTATE_NODE) + return (msg->query() == inbound) ? + TrafficCount::category::share_hash_asnode : + TrafficCount::category::get_hash_asnode; + + if (msg->type() == protocol::TMGetObjectByHash::otCAS_OBJECT) + return (msg->query() == inbound) ? + TrafficCount::category::share_cas_object : + TrafficCount::category::get_cas_object; + + if (msg->type() == protocol::TMGetObjectByHash::otFETCH_PACK) + return (msg->query() == inbound) ? + TrafficCount::category::share_fetch_pack : + TrafficCount::category::get_fetch_pack; + + return (msg->query() == inbound) ? + TrafficCount::category::share_hash : + TrafficCount::category::get_hash; } - assert (false); - return TrafficCount::category::CT_unknown; + return TrafficCount::category::unknown; } } // ripple diff --git a/src/ripple/overlay/impl/TrafficCount.h b/src/ripple/overlay/impl/TrafficCount.h index ea5d99980..a118532ae 100644 --- a/src/ripple/overlay/impl/TrafficCount.h +++ b/src/ripple/overlay/impl/TrafficCount.h @@ -23,37 +23,38 @@ #include #include +#include #include -#include +#include namespace ripple { class TrafficCount { - public: - - using count_t = std::atomic ; - class TrafficStats { - public: + public: + std::string const name; - count_t bytesIn; - count_t bytesOut; - count_t messagesIn; - count_t messagesOut; + std::atomic bytesIn {0}; + std::atomic bytesOut {0}; + std::atomic messagesIn {0}; + std::atomic messagesOut {0}; - TrafficStats() : bytesIn(0), bytesOut(0), - messagesIn(0), messagesOut(0) - { ; } + TrafficStats(char const* n) + : name (n) + { + } - TrafficStats(const TrafficStats& ts) - : bytesIn (ts.bytesIn.load()) + TrafficStats(TrafficStats const& ts) + : name (ts.name) + , bytesIn (ts.bytesIn.load()) , bytesOut (ts.bytesOut.load()) , messagesIn (ts.messagesIn.load()) , messagesOut (ts.messagesOut.load()) - { ; } + { + } operator bool () const { @@ -61,70 +62,167 @@ public: } }; - - enum class category + // If you add entries to this enum, you need to update the initialization + // of the array at the bottom of this file which maps array numbers to + // human-readable names. + enum category : std::size_t { - CT_base, // basic peer overhead, must be first - CT_overlay, // overlay management - CT_transaction, - CT_proposal, - CT_validation, - CT_get_ledger, // ledgers we try to get - CT_share_ledger, // ledgers we share - CT_get_trans, // transaction sets we try to get - CT_share_trans, // transaction sets we get - CT_unknown // must be last + base, // basic peer overhead, must be first + + cluster, // cluster overhead + overlay, // overlay management + manifests, // manifest management + transaction, + proposal, + validation, + shards, // shard-related traffic + + // TMHaveSet message: + get_set, // transaction sets we try to get + share_set, // transaction sets we get + + // TMLedgerData: transaction set candidate + ld_tsc_get, + ld_tsc_share, + + // TMLedgerData: transaction node + ld_txn_get, + ld_txn_share, + + // TMLedgerData: account state node + ld_asn_get, + ld_asn_share, + + // TMLedgerData: generic + ld_get, + ld_share, + + // TMGetLedger: transaction set candidate + gl_tsc_share, + gl_tsc_get, + + // TMGetLedger: transaction node + gl_txn_share, + gl_txn_get, + + // TMGetLedger: account state node + gl_asn_share, + gl_asn_get, + + // TMGetLedger: generic + gl_share, + gl_get, + + // TMGetObjectByHash: + share_hash_ledger, + get_hash_ledger, + + // TMGetObjectByHash: + share_hash_tx, + get_hash_tx, + + // TMGetObjectByHash: transaction node + share_hash_txnode, + get_hash_txnode, + + // TMGetObjectByHash: account state node + share_hash_asnode, + get_hash_asnode, + + // TMGetObjectByHash: CAS + share_cas_object, + get_cas_object, + + // TMGetObjectByHash: fetch packs + share_fetch_pack, + get_fetch_pack, + + // TMGetObjectByHash: generic + share_hash, + get_hash, + + unknown // must be last }; - static const char* getName (category c); - + /** Given a protocol message, determine which traffic category it belongs to */ static category categorize ( ::google::protobuf::Message const& message, int type, bool inbound); - void addCount (category cat, bool inbound, int number) + /** Account for traffic associated with the given category */ + void addCount (category cat, bool inbound, int bytes) { + assert (cat <= category::unknown); + if (inbound) { - counts_[cat].bytesIn += number; + counts_[cat].bytesIn += bytes; ++counts_[cat].messagesIn; } else { - counts_[cat].bytesOut += number; + counts_[cat].bytesOut += bytes; ++counts_[cat].messagesOut; } } - TrafficCount() - { - for (category i = category::CT_base; - i <= category::CT_unknown; - i = safe_cast(safe_cast>(i) + 1)) - { - counts_[i]; - } - } + TrafficCount() = default; - std::map + /** An up-to-date copy of all the counters + + @return an object which satisfies the requirements of Container + */ + auto getCounts () const { - std::map ret; - - for (auto& i : counts_) - { - if (i.second) - ret.emplace (std::piecewise_construct, - std::forward_as_tuple (getName (i.first)), - std::forward_as_tuple (i.second)); - } - - return ret; + return counts_; } - protected: - - std::map counts_; +protected: + std::array counts_ + {{ + { "overhead" }, // category::base + { "overhead: cluster" }, // category::cluster + { "overhead: overlay" }, // category::overlay + { "overhead: manifest" }, // category::manifests + { "transactions" }, // category::transaction + { "proposals" }, // category::proposal + { "validations" }, // category::validation + { "shards" }, // category::shards + { "set (get)" }, // category::get_set + { "set (share)" }, // category::share_set + { "ledger data: Transaction Set candidate (get)" }, // category::ld_tsc_get + { "ledger data: Transaction Set candidate (share)" }, // category::ld_tsc_share + { "ledger data: Transaction Node (get)" }, // category::ld_txn_get + { "ledger data: Transaction Node (share)" }, // category::ld_txn_share + { "ledger data: Account State Node (get)" }, // category::ld_asn_get + { "ledger data: Account State Node (share)" }, // category::ld_asn_share + { "ledger data (get)" }, // category::ld_get + { "ledger data (share)" }, // category::ld_share + { "ledger: Transaction Set candidate (share)" }, // category::gl_tsc_share + { "ledger: Transaction Set candidate (get)" }, // category::gl_tsc_get + { "ledger: Transaction node (share)" }, // category::gl_txn_share + { "ledger: Transaction node (get)" }, // category::gl_txn_get + { "ledger: Account State node (share)" }, // category::gl_asn_share + { "ledger: Account State node (get)" }, // category::gl_asn_get + { "ledger (share)" }, // category::gl_share + { "ledger (get)" }, // category::gl_get + { "getobject: Ledger (share)" }, // category::share_hash_ledger + { "getobject: Ledger (get)" }, // category::get_hash_ledger + { "getobject: Transaction (share)" }, // category::share_hash_tx + { "getobject: Transaction (get)" }, // category::get_hash_tx + { "getobject: Transaction node (share)" }, // category::share_hash_txnode + { "getobject: Transaction node (get)" }, // category::get_hash_txnode + { "getobject: Account State node (share)" }, // category::share_hash_asnode + { "getobject: Account State node (get)" }, // category::get_hash_asnode + { "getobject: CAS (share)" }, // category::share_cas_object + { "getobject: CAS (get)" }, // category::get_cas_object + { "getobject: Fetch Pack (share)" }, // category::share_fetch_pack + { "getobject: Fetch Pack (get)" }, // category::get_fetch_pack + { "getobject (share)" }, // category::share_hash + { "getobject (get)" }, // category::get_hash + { "unknown" } // category::unknown + }}; }; }