mirror of
				https://github.com/Xahau/xahaud.git
				synced 2025-11-04 10:45:50 +00:00 
			
		
		
		
	Enchance /crawl API endpoint with local server information (RIPD-1644):
The /crawl API endpoint allows developers to examine the structure of the XRP Ledger's overlay network. This commit adds additional information about the local server to the /crawl endpoint, making it possible for developers to create data-rich network-wide status dashboards. Related: - https://developers.ripple.com/peer-protocol.html - https://github.com/ripple/rippled-network-crawler
This commit is contained in:
		
				
					committed by
					
						
						Nik Bougalis
					
				
			
			
				
	
			
			
			
						parent
						
							ea76103d5f
						
					
				
				
					commit
					494724578a
				
			@@ -20,7 +20,9 @@
 | 
			
		||||
#
 | 
			
		||||
#   7. Voting
 | 
			
		||||
#
 | 
			
		||||
#   8. Example Settings
 | 
			
		||||
#   8. Misc Settings
 | 
			
		||||
#
 | 
			
		||||
#   9. Example Settings
 | 
			
		||||
#
 | 
			
		||||
#-------------------------------------------------------------------------------
 | 
			
		||||
#
 | 
			
		||||
@@ -1039,6 +1041,46 @@
 | 
			
		||||
#   [signing_support]
 | 
			
		||||
#   true
 | 
			
		||||
#
 | 
			
		||||
# [crawl]
 | 
			
		||||
#
 | 
			
		||||
#   List of options to control what data is reported through the /crawl endpoint
 | 
			
		||||
#   See https://developers.ripple.com/peer-protocol.html#peer-crawler
 | 
			
		||||
#
 | 
			
		||||
#   <flag>
 | 
			
		||||
#
 | 
			
		||||
#       Enable or disable access to /crawl requests. Default is '1'
 | 
			
		||||
#
 | 
			
		||||
#   overlay = <flag>
 | 
			
		||||
#
 | 
			
		||||
#       Report information about peers this server is connected to, similar
 | 
			
		||||
#       to the "peers" RPC API. Default is '1'.
 | 
			
		||||
#
 | 
			
		||||
#   server = <flag>
 | 
			
		||||
#
 | 
			
		||||
#       Report information about the local server, similar to the "server_state"
 | 
			
		||||
#       RPC API. Default is '1'.
 | 
			
		||||
#
 | 
			
		||||
#   counts = <flag>
 | 
			
		||||
#
 | 
			
		||||
#       Report information about the local server health counters, similar to
 | 
			
		||||
#       the "get_counts" RPC API. Default is '0'.
 | 
			
		||||
#
 | 
			
		||||
#   unl = <flag>
 | 
			
		||||
#
 | 
			
		||||
#       Report information about the local server's validator lists, similar to
 | 
			
		||||
#       the "validators" and "validator_list_sites" RPC APIs. Default is '1'.
 | 
			
		||||
#
 | 
			
		||||
#   Example:
 | 
			
		||||
#
 | 
			
		||||
#   [crawl]
 | 
			
		||||
#   0
 | 
			
		||||
#
 | 
			
		||||
#   [crawl]
 | 
			
		||||
#   overlay = 1   # report peer overlay info
 | 
			
		||||
#   server = 1    # report local server info
 | 
			
		||||
#   counts = 0    # do not report server counts
 | 
			
		||||
#   unl = 1       # report server validator lists
 | 
			
		||||
#
 | 
			
		||||
#-------------------------------------------------------------------------------
 | 
			
		||||
#
 | 
			
		||||
# 9. Example Settings
 | 
			
		||||
 
 | 
			
		||||
@@ -122,6 +122,7 @@ class ValidatorList
 | 
			
		||||
        std::vector<PublicKey> list;
 | 
			
		||||
        std::size_t sequence;
 | 
			
		||||
        TimeKeeper::time_point expiration;
 | 
			
		||||
        std::string siteUri;
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    ManifestCache& validatorManifests_;
 | 
			
		||||
@@ -200,7 +201,8 @@ public:
 | 
			
		||||
        std::string const& manifest,
 | 
			
		||||
        std::string const& blob,
 | 
			
		||||
        std::string const& signature,
 | 
			
		||||
        std::uint32_t version);
 | 
			
		||||
        std::uint32_t version,
 | 
			
		||||
        std::string siteUri);
 | 
			
		||||
 | 
			
		||||
    /** Update trusted nodes
 | 
			
		||||
 | 
			
		||||
 
 | 
			
		||||
@@ -190,13 +190,13 @@ ValidatorList::load (
 | 
			
		||||
    return true;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
ListDisposition
 | 
			
		||||
ValidatorList::applyList (
 | 
			
		||||
    std::string const& manifest,
 | 
			
		||||
    std::string const& blob,
 | 
			
		||||
    std::string const& signature,
 | 
			
		||||
    std::uint32_t version)
 | 
			
		||||
    std::uint32_t version,
 | 
			
		||||
    std::string siteUri)
 | 
			
		||||
{
 | 
			
		||||
    if (version != requiredListVersion)
 | 
			
		||||
        return ListDisposition::unsupported_version;
 | 
			
		||||
@@ -215,6 +215,7 @@ ValidatorList::applyList (
 | 
			
		||||
    publisherLists_[pubKey].sequence = list["sequence"].asUInt ();
 | 
			
		||||
    publisherLists_[pubKey].expiration = TimeKeeper::time_point{
 | 
			
		||||
        TimeKeeper::duration{list["expiration"].asUInt()}};
 | 
			
		||||
    publisherLists_[pubKey].siteUri = std::move(siteUri);
 | 
			
		||||
    std::vector<PublicKey>& publisherList = publisherLists_[pubKey].list;
 | 
			
		||||
 | 
			
		||||
    std::vector<PublicKey> oldList = publisherList;
 | 
			
		||||
@@ -544,6 +545,7 @@ ValidatorList::getJson() const
 | 
			
		||||
        Json::Value& curr = jPublisherLists.append(Json::objectValue);
 | 
			
		||||
        curr[jss::pubkey_publisher] = strHex(p.first);
 | 
			
		||||
        curr[jss::available] = p.second.available;
 | 
			
		||||
        curr[jss::uri] = p.second.siteUri;
 | 
			
		||||
        if(p.second.expiration != TimeKeeper::time_point{})
 | 
			
		||||
        {
 | 
			
		||||
            curr[jss::seq] = static_cast<Json::UInt>(p.second.sequence);
 | 
			
		||||
 
 | 
			
		||||
@@ -320,7 +320,8 @@ ValidatorSite::parseJsonResponse (
 | 
			
		||||
        body["manifest"].asString (),
 | 
			
		||||
        body["blob"].asString (),
 | 
			
		||||
        body["signature"].asString(),
 | 
			
		||||
        body["version"].asUInt());
 | 
			
		||||
        body["version"].asUInt(),
 | 
			
		||||
        sites_[siteIdx].activeResource->uri);
 | 
			
		||||
 | 
			
		||||
    sites_[siteIdx].lastRefreshStatus.emplace(
 | 
			
		||||
        Site::Status{clock_type::now(), disp, ""});
 | 
			
		||||
 
 | 
			
		||||
@@ -70,6 +70,7 @@ public:
 | 
			
		||||
        bool expire = false;
 | 
			
		||||
        beast::IP::Address public_ip;
 | 
			
		||||
        int ipLimit = 0;
 | 
			
		||||
        std::uint32_t crawlOptions = 0;
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    using PeerSequence = std::vector <std::shared_ptr<Peer>>;
 | 
			
		||||
@@ -104,11 +105,6 @@ public:
 | 
			
		||||
    std::size_t
 | 
			
		||||
    size () = 0;
 | 
			
		||||
 | 
			
		||||
    /** Returns information reported to the crawl cgi command. */
 | 
			
		||||
    virtual
 | 
			
		||||
    Json::Value
 | 
			
		||||
    crawl() = 0;
 | 
			
		||||
 | 
			
		||||
    /** Return diagnostics on the status of all peers.
 | 
			
		||||
        @deprecated This is superceded by PropertyStream
 | 
			
		||||
    */
 | 
			
		||||
 
 | 
			
		||||
@@ -21,6 +21,7 @@
 | 
			
		||||
#include <ripple/app/misc/HashRouter.h>
 | 
			
		||||
#include <ripple/app/misc/NetworkOPs.h>
 | 
			
		||||
#include <ripple/app/misc/ValidatorList.h>
 | 
			
		||||
#include <ripple/app/misc/ValidatorSite.h>
 | 
			
		||||
#include <ripple/basics/base64.h>
 | 
			
		||||
#include <ripple/basics/make_SSLContext.h>
 | 
			
		||||
#include <ripple/beast/core/LexicalCast.h>
 | 
			
		||||
@@ -32,6 +33,7 @@
 | 
			
		||||
#include <ripple/overlay/impl/PeerImp.h>
 | 
			
		||||
#include <ripple/peerfinder/make_Manager.h>
 | 
			
		||||
#include <ripple/rpc/json_body.h>
 | 
			
		||||
#include <ripple/rpc/handlers/GetCounts.h>
 | 
			
		||||
#include <ripple/server/SimpleWriter.h>
 | 
			
		||||
 | 
			
		||||
#include <boost/utility/in_place_factory.hpp>
 | 
			
		||||
@@ -58,6 +60,18 @@ struct get_peer_json
 | 
			
		||||
    }
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
namespace CrawlOptions
 | 
			
		||||
{
 | 
			
		||||
    enum
 | 
			
		||||
    {
 | 
			
		||||
        Disabled     = 0,
 | 
			
		||||
        Overlay      = (1 << 0),
 | 
			
		||||
        ServerInfo   = (1 << 1),
 | 
			
		||||
        ServerCounts = (1 << 2),
 | 
			
		||||
        Unl          = (1 << 3)
 | 
			
		||||
    };
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
//------------------------------------------------------------------------------
 | 
			
		||||
 | 
			
		||||
OverlayImpl::Child::Child (OverlayImpl& overlay)
 | 
			
		||||
@@ -860,7 +874,7 @@ OverlayImpl::limit()
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Json::Value
 | 
			
		||||
OverlayImpl::crawl()
 | 
			
		||||
OverlayImpl::getOverlayInfo()
 | 
			
		||||
{
 | 
			
		||||
    using namespace std::chrono;
 | 
			
		||||
    Json::Value jv;
 | 
			
		||||
@@ -912,6 +926,65 @@ OverlayImpl::crawl()
 | 
			
		||||
    return jv;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Json::Value
 | 
			
		||||
OverlayImpl::getServerInfo()
 | 
			
		||||
{
 | 
			
		||||
    bool const humanReadable = false;
 | 
			
		||||
    bool const admin = false;
 | 
			
		||||
    bool const counters = false;
 | 
			
		||||
 | 
			
		||||
    Json::Value server_info = app_.getOPs().getServerInfo(humanReadable, admin, counters);
 | 
			
		||||
 | 
			
		||||
    // Filter out some information
 | 
			
		||||
    server_info.removeMember(jss::hostid);
 | 
			
		||||
    server_info.removeMember(jss::load_factor_fee_escalation);
 | 
			
		||||
    server_info.removeMember(jss::load_factor_fee_queue);
 | 
			
		||||
 | 
			
		||||
    if (server_info.isMember(jss::validated_ledger))
 | 
			
		||||
    {
 | 
			
		||||
        Json::Value& validated_ledger = server_info[jss::validated_ledger];
 | 
			
		||||
 | 
			
		||||
        validated_ledger.removeMember(jss::base_fee);
 | 
			
		||||
        validated_ledger.removeMember(jss::reserve_base_xrp);
 | 
			
		||||
        validated_ledger.removeMember(jss::reserve_inc_xrp);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    return server_info;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Json::Value
 | 
			
		||||
OverlayImpl::getServerCounts()
 | 
			
		||||
{
 | 
			
		||||
    return getCountsJson(app_, 10);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Json::Value
 | 
			
		||||
OverlayImpl::getUnlInfo()
 | 
			
		||||
{
 | 
			
		||||
    Json::Value validators = app_.validators().getJson();
 | 
			
		||||
 | 
			
		||||
    if (validators.isMember(jss::publisher_lists))
 | 
			
		||||
    {
 | 
			
		||||
        Json::Value& publisher_lists = validators[jss::publisher_lists];
 | 
			
		||||
 | 
			
		||||
        for (auto& publisher : publisher_lists)
 | 
			
		||||
        {
 | 
			
		||||
            publisher.removeMember(jss::list);
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    validators.removeMember(jss::signing_keys);
 | 
			
		||||
 | 
			
		||||
    Json::Value validatorSites = app_.validatorSites().getJson();
 | 
			
		||||
 | 
			
		||||
    if (validatorSites.isMember(jss::validator_sites))
 | 
			
		||||
    {
 | 
			
		||||
        validators[jss::validator_sites] = std::move(validatorSites[jss::validator_sites]);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    return validators;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// Returns information on verified peers.
 | 
			
		||||
Json::Value
 | 
			
		||||
OverlayImpl::json ()
 | 
			
		||||
@@ -923,7 +996,7 @@ bool
 | 
			
		||||
OverlayImpl::processRequest (http_request_type const& req,
 | 
			
		||||
    Handoff& handoff)
 | 
			
		||||
{
 | 
			
		||||
    if (req.target() != "/crawl")
 | 
			
		||||
    if (req.target() != "/crawl" || setup_.crawlOptions == CrawlOptions::Disabled)
 | 
			
		||||
        return false;
 | 
			
		||||
 | 
			
		||||
    boost::beast::http::response<json_body> msg;
 | 
			
		||||
@@ -932,7 +1005,25 @@ OverlayImpl::processRequest (http_request_type const& req,
 | 
			
		||||
    msg.insert("Server", BuildInfo::getFullVersionString());
 | 
			
		||||
    msg.insert("Content-Type", "application/json");
 | 
			
		||||
    msg.insert("Connection", "close");
 | 
			
		||||
    msg.body()["overlay"] = crawl();
 | 
			
		||||
    msg.body()["version"] = Json::Value(1u);
 | 
			
		||||
 | 
			
		||||
    if (setup_.crawlOptions & CrawlOptions::Overlay)
 | 
			
		||||
    {
 | 
			
		||||
        msg.body()["overlay"] = getOverlayInfo();
 | 
			
		||||
    }
 | 
			
		||||
    if (setup_.crawlOptions & CrawlOptions::ServerInfo)
 | 
			
		||||
    {
 | 
			
		||||
        msg.body()["server"] = getServerInfo();
 | 
			
		||||
    }
 | 
			
		||||
    if (setup_.crawlOptions & CrawlOptions::ServerCounts)
 | 
			
		||||
    {
 | 
			
		||||
        msg.body()["counts"] = getServerCounts();
 | 
			
		||||
    }
 | 
			
		||||
    if (setup_.crawlOptions & CrawlOptions::Unl)
 | 
			
		||||
    {
 | 
			
		||||
        msg.body()["unl"] = getUnlInfo();
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    msg.prepare_payload();
 | 
			
		||||
    handoff.response = std::make_shared<SimpleWriter>(msg);
 | 
			
		||||
    return true;
 | 
			
		||||
@@ -1145,23 +1236,72 @@ Overlay::Setup
 | 
			
		||||
setup_Overlay (BasicConfig const& config)
 | 
			
		||||
{
 | 
			
		||||
    Overlay::Setup setup;
 | 
			
		||||
    auto const& section = config.section("overlay");
 | 
			
		||||
    setup.context = make_SSLContext("");
 | 
			
		||||
    setup.expire = get<bool>(section, "expire", false);
 | 
			
		||||
 | 
			
		||||
    set (setup.ipLimit, "ip_limit", section);
 | 
			
		||||
    if (setup.ipLimit < 0)
 | 
			
		||||
        Throw<std::runtime_error> ("Configured IP limit is invalid");
 | 
			
		||||
 | 
			
		||||
    std::string ip;
 | 
			
		||||
    set (ip, "public_ip", section);
 | 
			
		||||
    if (! ip.empty ())
 | 
			
		||||
    {
 | 
			
		||||
        boost::system::error_code ec;
 | 
			
		||||
        setup.public_ip = beast::IP::Address::from_string (ip, ec);
 | 
			
		||||
        if (ec || beast::IP::is_private (setup.public_ip))
 | 
			
		||||
            Throw<std::runtime_error> ("Configured public IP is invalid");
 | 
			
		||||
        auto const& section = config.section("overlay");
 | 
			
		||||
        setup.context = make_SSLContext("");
 | 
			
		||||
        setup.expire = get<bool>(section, "expire", false);
 | 
			
		||||
 | 
			
		||||
        set(setup.ipLimit, "ip_limit", section);
 | 
			
		||||
        if (setup.ipLimit < 0)
 | 
			
		||||
            Throw<std::runtime_error>("Configured IP limit is invalid");
 | 
			
		||||
 | 
			
		||||
        std::string ip;
 | 
			
		||||
        set(ip, "public_ip", section);
 | 
			
		||||
        if (!ip.empty())
 | 
			
		||||
        {
 | 
			
		||||
            boost::system::error_code ec;
 | 
			
		||||
            setup.public_ip = beast::IP::Address::from_string(ip, ec);
 | 
			
		||||
            if (ec || beast::IP::is_private(setup.public_ip))
 | 
			
		||||
                Throw<std::runtime_error>("Configured public IP is invalid");
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
    {
 | 
			
		||||
        auto const& section = config.section("crawl");
 | 
			
		||||
        auto const& values = section.values();
 | 
			
		||||
 | 
			
		||||
        if (values.size() > 1)
 | 
			
		||||
        {
 | 
			
		||||
            Throw<std::runtime_error>(
 | 
			
		||||
                "Configured [crawl] section is invalid, too many values");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        bool crawlEnabled = true;
 | 
			
		||||
 | 
			
		||||
        // Only allow "0|1" as a value
 | 
			
		||||
        if (values.size() == 1)
 | 
			
		||||
        {
 | 
			
		||||
            try
 | 
			
		||||
            {
 | 
			
		||||
                crawlEnabled = boost::lexical_cast<bool>(values.front());
 | 
			
		||||
            }
 | 
			
		||||
            catch (boost::bad_lexical_cast const&)
 | 
			
		||||
            {
 | 
			
		||||
                Throw<std::runtime_error>(
 | 
			
		||||
                    "Configured [crawl] section has invalid value: " + values.front());
 | 
			
		||||
            }
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        if (crawlEnabled)
 | 
			
		||||
        {
 | 
			
		||||
            if (get<bool>(section, "overlay", true))
 | 
			
		||||
            {
 | 
			
		||||
                setup.crawlOptions |= CrawlOptions::Overlay;
 | 
			
		||||
            }
 | 
			
		||||
            if (get<bool>(section, "server", true))
 | 
			
		||||
            {
 | 
			
		||||
                setup.crawlOptions |= CrawlOptions::ServerInfo;
 | 
			
		||||
            }
 | 
			
		||||
            if (get<bool>(section, "counts", false))
 | 
			
		||||
            {
 | 
			
		||||
                setup.crawlOptions |= CrawlOptions::ServerCounts;
 | 
			
		||||
            }
 | 
			
		||||
            if (get<bool>(section, "unl", true))
 | 
			
		||||
            {
 | 
			
		||||
                setup.crawlOptions |= CrawlOptions::Unl;
 | 
			
		||||
            }
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    return setup;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 
 | 
			
		||||
@@ -172,6 +172,18 @@ public:
 | 
			
		||||
        http_request_type&& request,
 | 
			
		||||
            endpoint_type remote_endpoint) override;
 | 
			
		||||
 | 
			
		||||
    void
 | 
			
		||||
    connect(beast::IP::Endpoint const& remote_endpoint) override;
 | 
			
		||||
 | 
			
		||||
    int
 | 
			
		||||
    limit() override;
 | 
			
		||||
 | 
			
		||||
    std::size_t
 | 
			
		||||
    size() override;
 | 
			
		||||
 | 
			
		||||
    Json::Value
 | 
			
		||||
    json() override;
 | 
			
		||||
 | 
			
		||||
    PeerSequence
 | 
			
		||||
    getActivePeers() override;
 | 
			
		||||
 | 
			
		||||
@@ -374,24 +386,33 @@ private:
 | 
			
		||||
    processRequest (http_request_type const& req,
 | 
			
		||||
        Handoff& handoff);
 | 
			
		||||
 | 
			
		||||
    void
 | 
			
		||||
    connect (beast::IP::Endpoint const& remote_endpoint) override;
 | 
			
		||||
 | 
			
		||||
    /*  The number of active peers on the network
 | 
			
		||||
        Active peers are only those peers that have completed the handshake
 | 
			
		||||
        and are running the Ripple protocol.
 | 
			
		||||
    /** Returns information about peers on the overlay network.
 | 
			
		||||
        Reported through the /crawl API
 | 
			
		||||
        Controlled through the config section [crawl] overlay=[0|1]
 | 
			
		||||
    */
 | 
			
		||||
    std::size_t
 | 
			
		||||
    size() override;
 | 
			
		||||
 | 
			
		||||
    int
 | 
			
		||||
    limit () override;
 | 
			
		||||
 | 
			
		||||
    Json::Value
 | 
			
		||||
    crawl() override;
 | 
			
		||||
    getOverlayInfo();
 | 
			
		||||
 | 
			
		||||
    /** Returns information about the local server.
 | 
			
		||||
        Reported through the /crawl API
 | 
			
		||||
        Controlled through the config section [crawl] server=[0|1]
 | 
			
		||||
    */
 | 
			
		||||
    Json::Value
 | 
			
		||||
    json() override;
 | 
			
		||||
    getServerInfo();
 | 
			
		||||
 | 
			
		||||
    /** Returns information about the local server's performance counters.
 | 
			
		||||
        Reported through the /crawl API
 | 
			
		||||
        Controlled through the config section [crawl] counts=[0|1]
 | 
			
		||||
    */
 | 
			
		||||
    Json::Value
 | 
			
		||||
    getServerCounts();
 | 
			
		||||
 | 
			
		||||
    /** Returns information about the local server's UNL.
 | 
			
		||||
        Reported through the /crawl API
 | 
			
		||||
        Controlled through the config section [crawl] unl=[0|1]
 | 
			
		||||
    */
 | 
			
		||||
    Json::Value
 | 
			
		||||
    getUnlInfo();
 | 
			
		||||
 | 
			
		||||
    //--------------------------------------------------------------------------
 | 
			
		||||
 | 
			
		||||
 
 | 
			
		||||
@@ -485,6 +485,7 @@ JSS ( validated );                  // out: NetworkOPs, RPCHelpers, AccountTx*
 | 
			
		||||
                                    //      Tx
 | 
			
		||||
JSS ( validator_list_expires );     // out: NetworkOps, ValidatorList
 | 
			
		||||
JSS ( validator_list );             // out: NetworkOps, ValidatorList
 | 
			
		||||
JSS ( validators );
 | 
			
		||||
JSS ( validated_ledger );           // out: NetworkOPs
 | 
			
		||||
JSS ( validated_ledgers );          // out: NetworkOPs
 | 
			
		||||
JSS ( validation_key );             // out: ValidationCreate, ValidationSeed
 | 
			
		||||
 
 | 
			
		||||
@@ -58,58 +58,50 @@ textTime(std::string& text, UptimeClock::time_point& seconds,
 | 
			
		||||
        text += "s";
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// {
 | 
			
		||||
//   min_count: <number>  // optional, defaults to 10
 | 
			
		||||
// }
 | 
			
		||||
Json::Value doGetCounts (RPC::Context& context)
 | 
			
		||||
Json::Value getCountsJson(Application& app, int minObjectCount)
 | 
			
		||||
{
 | 
			
		||||
    int minCount = 10;
 | 
			
		||||
    auto objectCounts = CountedObjects::getInstance().getCounts(minObjectCount);
 | 
			
		||||
 | 
			
		||||
    if (context.params.isMember (jss::min_count))
 | 
			
		||||
        minCount = context.params[jss::min_count].asUInt ();
 | 
			
		||||
 | 
			
		||||
    auto objectCounts = CountedObjects::getInstance ().getCounts (minCount);
 | 
			
		||||
 | 
			
		||||
    Json::Value ret (Json::objectValue);
 | 
			
		||||
    Json::Value ret(Json::objectValue);
 | 
			
		||||
 | 
			
		||||
    for (auto const& it : objectCounts)
 | 
			
		||||
    {
 | 
			
		||||
        ret [it.first] = it.second;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    int dbKB = getKBUsedAll (context.app.getLedgerDB ().getSession ());
 | 
			
		||||
    int dbKB = getKBUsedAll (app.getLedgerDB ().getSession ());
 | 
			
		||||
 | 
			
		||||
    if (dbKB > 0)
 | 
			
		||||
        ret[jss::dbKBTotal] = dbKB;
 | 
			
		||||
 | 
			
		||||
    dbKB = getKBUsedDB (context.app.getLedgerDB ().getSession ());
 | 
			
		||||
    dbKB = getKBUsedDB (app.getLedgerDB ().getSession ());
 | 
			
		||||
 | 
			
		||||
    if (dbKB > 0)
 | 
			
		||||
        ret[jss::dbKBLedger] = dbKB;
 | 
			
		||||
 | 
			
		||||
    dbKB = getKBUsedDB (context.app.getTxnDB ().getSession ());
 | 
			
		||||
    dbKB = getKBUsedDB (app.getTxnDB ().getSession ());
 | 
			
		||||
 | 
			
		||||
    if (dbKB > 0)
 | 
			
		||||
        ret[jss::dbKBTransaction] = dbKB;
 | 
			
		||||
 | 
			
		||||
    {
 | 
			
		||||
        std::size_t c = context.app.getOPs().getLocalTxCount ();
 | 
			
		||||
        std::size_t c = app.getOPs().getLocalTxCount ();
 | 
			
		||||
        if (c > 0)
 | 
			
		||||
            ret[jss::local_txs] = static_cast<Json::UInt> (c);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    ret[jss::write_load] = context.app.getNodeStore ().getWriteLoad ();
 | 
			
		||||
    ret[jss::write_load] = app.getNodeStore ().getWriteLoad ();
 | 
			
		||||
 | 
			
		||||
    ret[jss::historical_perminute] = static_cast<int>(
 | 
			
		||||
        context.app.getInboundLedgers().fetchRate());
 | 
			
		||||
    ret[jss::SLE_hit_rate] = context.app.cachedSLEs().rate();
 | 
			
		||||
    ret[jss::node_hit_rate] = context.app.getNodeStore ().getCacheHitRate ();
 | 
			
		||||
    ret[jss::ledger_hit_rate] = context.app.getLedgerMaster ().getCacheHitRate ();
 | 
			
		||||
    ret[jss::AL_hit_rate] = context.app.getAcceptedLedgerCache ().getHitRate ();
 | 
			
		||||
        app.getInboundLedgers().fetchRate());
 | 
			
		||||
    ret[jss::SLE_hit_rate] = app.cachedSLEs().rate();
 | 
			
		||||
    ret[jss::node_hit_rate] = app.getNodeStore ().getCacheHitRate ();
 | 
			
		||||
    ret[jss::ledger_hit_rate] = app.getLedgerMaster ().getCacheHitRate ();
 | 
			
		||||
    ret[jss::AL_hit_rate] = app.getAcceptedLedgerCache ().getHitRate ();
 | 
			
		||||
 | 
			
		||||
    ret[jss::fullbelow_size] = static_cast<int>(context.app.family().fullbelow().size());
 | 
			
		||||
    ret[jss::treenode_cache_size] = context.app.family().treecache().getCacheSize();
 | 
			
		||||
    ret[jss::treenode_track_size] = context.app.family().treecache().getTrackSize();
 | 
			
		||||
    ret[jss::fullbelow_size] = static_cast<int>(app.family().fullbelow().size());
 | 
			
		||||
    ret[jss::treenode_cache_size] = app.family().treecache().getCacheSize();
 | 
			
		||||
    ret[jss::treenode_track_size] = app.family().treecache().getTrackSize();
 | 
			
		||||
 | 
			
		||||
    std::string uptime;
 | 
			
		||||
    auto s = UptimeClock::now();
 | 
			
		||||
@@ -121,21 +113,21 @@ Json::Value doGetCounts (RPC::Context& context)
 | 
			
		||||
    textTime (uptime, s, "second", 1s);
 | 
			
		||||
    ret[jss::uptime] = uptime;
 | 
			
		||||
 | 
			
		||||
    ret[jss::node_writes] = context.app.getNodeStore().getStoreCount();
 | 
			
		||||
    ret[jss::node_reads_total] = context.app.getNodeStore().getFetchTotalCount();
 | 
			
		||||
    ret[jss::node_reads_hit] = context.app.getNodeStore().getFetchHitCount();
 | 
			
		||||
    ret[jss::node_written_bytes] = context.app.getNodeStore().getStoreSize();
 | 
			
		||||
    ret[jss::node_read_bytes] = context.app.getNodeStore().getFetchSize();
 | 
			
		||||
    ret[jss::node_writes] = app.getNodeStore().getStoreCount();
 | 
			
		||||
    ret[jss::node_reads_total] = app.getNodeStore().getFetchTotalCount();
 | 
			
		||||
    ret[jss::node_reads_hit] = app.getNodeStore().getFetchHitCount();
 | 
			
		||||
    ret[jss::node_written_bytes] = app.getNodeStore().getStoreSize();
 | 
			
		||||
    ret[jss::node_read_bytes] = app.getNodeStore().getFetchSize();
 | 
			
		||||
 | 
			
		||||
    if (auto shardStore = context.app.getShardStore())
 | 
			
		||||
    if (auto shardStore = app.getShardStore())
 | 
			
		||||
    {
 | 
			
		||||
        Json::Value& jv = (ret[jss::shards] = Json::objectValue);
 | 
			
		||||
        jv[jss::fullbelow_size] =
 | 
			
		||||
            static_cast<int>(context.app.shardFamily()->fullbelow().size());
 | 
			
		||||
            static_cast<int>(app.shardFamily()->fullbelow().size());
 | 
			
		||||
        jv[jss::treenode_cache_size] =
 | 
			
		||||
            context.app.shardFamily()->treecache().getCacheSize();
 | 
			
		||||
            app.shardFamily()->treecache().getCacheSize();
 | 
			
		||||
        jv[jss::treenode_track_size] =
 | 
			
		||||
            context.app.shardFamily()->treecache().getTrackSize();
 | 
			
		||||
            app.shardFamily()->treecache().getTrackSize();
 | 
			
		||||
        ret[jss::write_load] = shardStore->getWriteLoad();
 | 
			
		||||
        ret[jss::node_hit_rate] = shardStore->getCacheHitRate();
 | 
			
		||||
        jv[jss::node_writes] = shardStore->getStoreCount();
 | 
			
		||||
@@ -148,4 +140,17 @@ Json::Value doGetCounts (RPC::Context& context)
 | 
			
		||||
    return ret;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// {
 | 
			
		||||
//   min_count: <number>  // optional, defaults to 10
 | 
			
		||||
// }
 | 
			
		||||
Json::Value doGetCounts (RPC::Context& context)
 | 
			
		||||
{
 | 
			
		||||
    int minCount = 10;
 | 
			
		||||
 | 
			
		||||
    if (context.params.isMember (jss::min_count))
 | 
			
		||||
        minCount = context.params[jss::min_count].asUInt ();
 | 
			
		||||
 | 
			
		||||
    return getCountsJson(context.app, minCount);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
} // ripple
 | 
			
		||||
 
 | 
			
		||||
							
								
								
									
										32
									
								
								src/ripple/rpc/handlers/GetCounts.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										32
									
								
								src/ripple/rpc/handlers/GetCounts.h
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,32 @@
 | 
			
		||||
//------------------------------------------------------------------------------
 | 
			
		||||
/*
 | 
			
		||||
    This file is part of rippled: https://github.com/ripple/rippled
 | 
			
		||||
    Copyright (c) 2012-2019 Ripple Labs Inc.
 | 
			
		||||
 | 
			
		||||
    Permission to use, copy, modify, and/or distribute this software for any
 | 
			
		||||
    purpose  with  or without fee is hereby granted, provided that the above
 | 
			
		||||
    copyright notice and this permission notice appear in all copies.
 | 
			
		||||
 | 
			
		||||
    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 | 
			
		||||
    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
 | 
			
		||||
    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 | 
			
		||||
    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 | 
			
		||||
    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
 | 
			
		||||
    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 | 
			
		||||
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 | 
			
		||||
*/
 | 
			
		||||
//==============================================================================
 | 
			
		||||
 | 
			
		||||
#ifndef RIPPLE_RPC_HANDLERS_GETCOUNTS_H_INCLUDED
 | 
			
		||||
#define RIPPLE_RPC_HANDLERS_GETCOUNTS_H_INCLUDED
 | 
			
		||||
 | 
			
		||||
#include <ripple/json/Object.h>
 | 
			
		||||
#include <ripple/app/main/Application.h>
 | 
			
		||||
 | 
			
		||||
namespace ripple {
 | 
			
		||||
 | 
			
		||||
Json::Value getCountsJson(Application& app, int minObjectCount);
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
#endif
 | 
			
		||||
@@ -396,6 +396,8 @@ private:
 | 
			
		||||
    {
 | 
			
		||||
        testcase ("Apply list");
 | 
			
		||||
 | 
			
		||||
        std::string const siteUri = "testApplyList.test";
 | 
			
		||||
 | 
			
		||||
        ManifestCache manifests;
 | 
			
		||||
        jtx::Env env (*this);
 | 
			
		||||
        auto trustedKeys = std::make_unique<ValidatorList> (
 | 
			
		||||
@@ -437,7 +439,7 @@ private:
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::stale ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, expiredblob, expiredSig, version));
 | 
			
		||||
                manifest1, expiredblob, expiredSig, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        // apply single list
 | 
			
		||||
        using namespace std::chrono_literals;
 | 
			
		||||
@@ -448,7 +450,7 @@ private:
 | 
			
		||||
        auto const sig1 = signList (blob1, pubSigningKeys1);
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList (
 | 
			
		||||
            manifest1, blob1, sig1, version));
 | 
			
		||||
            manifest1, blob1, sig1, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        for (auto const& val : list1)
 | 
			
		||||
        {
 | 
			
		||||
@@ -463,13 +465,13 @@ private:
 | 
			
		||||
                pubSigningKeys1.first, pubSigningKeys1.second, 1));
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::untrusted == trustedKeys->applyList (
 | 
			
		||||
            untrustedManifest, blob1, sig1, version));
 | 
			
		||||
            untrustedManifest, blob1, sig1, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        // do not use list with unhandled version
 | 
			
		||||
        auto const badVersion = 666;
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::unsupported_version ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, blob1, sig1, badVersion));
 | 
			
		||||
                manifest1, blob1, sig1, badVersion, siteUri));
 | 
			
		||||
 | 
			
		||||
        // apply list with highest sequence number
 | 
			
		||||
        auto const sequence2 = 2;
 | 
			
		||||
@@ -479,7 +481,7 @@ private:
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::accepted ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, blob2, sig2, version));
 | 
			
		||||
                manifest1, blob2, sig2, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        for (auto const& val : list1)
 | 
			
		||||
        {
 | 
			
		||||
@@ -496,11 +498,11 @@ private:
 | 
			
		||||
        // do not re-apply lists with past or current sequence numbers
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::stale ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, blob1, sig1, version));
 | 
			
		||||
                manifest1, blob1, sig1, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::same_sequence ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, blob2, sig2, version));
 | 
			
		||||
                manifest1, blob2, sig2, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        // apply list with new publisher key updated by manifest
 | 
			
		||||
        auto const pubSigningKeys2 = randomKeyPair(KeyType::secp256k1);
 | 
			
		||||
@@ -515,7 +517,7 @@ private:
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::accepted ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest2, blob3, sig3, version));
 | 
			
		||||
                manifest2, blob3, sig3, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        auto const sequence4 = 4;
 | 
			
		||||
        auto const blob4 = makeList (
 | 
			
		||||
@@ -523,7 +525,7 @@ private:
 | 
			
		||||
        auto const badSig = signList (blob4, pubSigningKeys1);
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::invalid ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                manifest1, blob4, badSig, version));
 | 
			
		||||
                manifest1, blob4, badSig, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        // do not apply list with revoked publisher key
 | 
			
		||||
        // applied list is removed due to revoked publisher key
 | 
			
		||||
@@ -540,7 +542,7 @@ private:
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(ListDisposition::untrusted ==
 | 
			
		||||
            trustedKeys->applyList (
 | 
			
		||||
                maxManifest, blob5, sig5, version));
 | 
			
		||||
                maxManifest, blob5, sig5, version, siteUri));
 | 
			
		||||
 | 
			
		||||
        BEAST_EXPECT(! trustedKeys->trustedPublisher(publisherPublic));
 | 
			
		||||
        for (auto const& val : list1)
 | 
			
		||||
@@ -555,6 +557,8 @@ private:
 | 
			
		||||
    {
 | 
			
		||||
        testcase ("Update trusted");
 | 
			
		||||
 | 
			
		||||
        std::string const siteUri = "testUpdateTrusted.test";
 | 
			
		||||
 | 
			
		||||
        PublicKey emptyLocalKey;
 | 
			
		||||
        ManifestCache manifests;
 | 
			
		||||
        jtx::Env env (*this);
 | 
			
		||||
@@ -805,7 +809,7 @@ private:
 | 
			
		||||
 | 
			
		||||
            BEAST_EXPECT(ListDisposition::accepted ==
 | 
			
		||||
                trustedKeys->applyList (
 | 
			
		||||
                    manifest, blob, sig, version));
 | 
			
		||||
                    manifest, blob, sig, version, siteUri));
 | 
			
		||||
 | 
			
		||||
            TrustChanges changes =
 | 
			
		||||
                trustedKeys->updateTrusted(activeValidators);
 | 
			
		||||
@@ -839,7 +843,7 @@ private:
 | 
			
		||||
 | 
			
		||||
            BEAST_EXPECT(ListDisposition::accepted ==
 | 
			
		||||
                trustedKeys->applyList (
 | 
			
		||||
                    manifest, blob2, sig2, version));
 | 
			
		||||
                    manifest, blob2, sig2, version, siteUri));
 | 
			
		||||
 | 
			
		||||
            changes = trustedKeys->updateTrusted (activeValidators);
 | 
			
		||||
            BEAST_EXPECT(changes.removed.empty());
 | 
			
		||||
@@ -942,7 +946,7 @@ private:
 | 
			
		||||
                    calcNodeID(valKeys.back().masterPublic));
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            auto addPublishedList = [this, &env, &trustedKeys, &valKeys]()
 | 
			
		||||
            auto addPublishedList = [this, &env, &trustedKeys, &valKeys, &siteUri]()
 | 
			
		||||
            {
 | 
			
		||||
                auto const publisherSecret = randomSecretKey();
 | 
			
		||||
                auto const publisherPublic =
 | 
			
		||||
@@ -970,7 +974,7 @@ private:
 | 
			
		||||
                auto const sig = signList (blob, pubSigningKeys);
 | 
			
		||||
 | 
			
		||||
                BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList (
 | 
			
		||||
                    manifest, blob, sig, version));
 | 
			
		||||
                    manifest, blob, sig, version, siteUri));
 | 
			
		||||
            };
 | 
			
		||||
 | 
			
		||||
            // Apply multiple published lists
 | 
			
		||||
@@ -999,6 +1003,8 @@ private:
 | 
			
		||||
    {
 | 
			
		||||
        testcase("Expires");
 | 
			
		||||
 | 
			
		||||
        std::string const siteUri = "testExpires.test";
 | 
			
		||||
 | 
			
		||||
        jtx::Env env(*this);
 | 
			
		||||
 | 
			
		||||
        auto toStr = [](PublicKey const& publicKey) {
 | 
			
		||||
@@ -1088,7 +1094,7 @@ private:
 | 
			
		||||
            // Apply first list
 | 
			
		||||
            BEAST_EXPECT(
 | 
			
		||||
                ListDisposition::accepted == trustedKeys->applyList(
 | 
			
		||||
                    prep1.manifest, prep1.blob, prep1.sig, prep1.version));
 | 
			
		||||
                    prep1.manifest, prep1.blob, prep1.sig, prep1.version, siteUri));
 | 
			
		||||
 | 
			
		||||
            // One list still hasn't published, so expiration is still unknown
 | 
			
		||||
            BEAST_EXPECT(trustedKeys->expires() == boost::none);
 | 
			
		||||
@@ -1096,7 +1102,7 @@ private:
 | 
			
		||||
            // Apply second list
 | 
			
		||||
            BEAST_EXPECT(
 | 
			
		||||
                ListDisposition::accepted == trustedKeys->applyList(
 | 
			
		||||
                    prep2.manifest, prep2.blob, prep2.sig, prep2.version));
 | 
			
		||||
                    prep2.manifest, prep2.blob, prep2.sig, prep2.version, siteUri));
 | 
			
		||||
 | 
			
		||||
            // We now have loaded both lists, so expiration is known
 | 
			
		||||
            BEAST_EXPECT(
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user