From 9c02cc1b171a027113582f9fb87061b045e23734 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 24 Jan 2015 07:08:03 -0800 Subject: [PATCH] Add /crawl cgi request feature to peer protocol (RIPD-729): This adds support for a cgi /crawl request, issued over HTTPS to the configured peer protocol port. The response to the request is a JSON object containing the node public key, type, and IP address of each directly connected neighbor. The IP address is suppressed unless the neighbor has requested its address to be revealed by adding "Crawl: public" to its HTTP headers. This field is currently set by the peer_private option in the rippled.cfg file. --- doc/rippled-example.cfg | 4 +- src/ripple/overlay/Overlay.h | 6 +++ src/ripple/overlay/README.md | 30 ++++++----- src/ripple/overlay/impl/ConnectAttempt.cpp | 13 ++--- src/ripple/overlay/impl/ConnectAttempt.h | 3 +- src/ripple/overlay/impl/OverlayImpl.cpp | 51 ++++++++++++++++++- src/ripple/overlay/impl/OverlayImpl.h | 7 +++ src/ripple/overlay/impl/PeerImp.cpp | 40 ++++++--------- src/ripple/overlay/impl/PeerImp.h | 23 ++------- src/ripple/peerfinder/Manager.h | 8 +++ src/ripple/peerfinder/impl/Logic.h | 7 +++ src/ripple/peerfinder/impl/Manager.cpp | 6 +++ .../peerfinder/impl/PeerfinderConfig.cpp | 2 +- src/ripple/server/impl/ServerHandlerImp.cpp | 1 + 14 files changed, 130 insertions(+), 71 deletions(-) diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg index 47b5516a3a..775840a872 100644 --- a/doc/rippled-example.cfg +++ b/doc/rippled-example.cfg @@ -688,9 +688,7 @@ # The 'temp_db' configures a look-aside cache for high volume storage # which doesn't necessarily persist between server launches. This # is an optional configuration parameter. If it is left out then -# no look-aside database is created or used. Use of temp_db may -# improve performance in some environments, but should be tested -# with and without to determine if beneficial. +# no look-aside database is created or used. # # The 'import_db' is used with the '--import' command line option to # migrate the specified database into the current database given diff --git a/src/ripple/overlay/Overlay.h b/src/ripple/overlay/Overlay.h index c54a8fa9f9..bb3be5b988 100644 --- a/src/ripple/overlay/Overlay.h +++ b/src/ripple/overlay/Overlay.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_OVERLAY_OVERLAY_H_INCLUDED #define RIPPLE_OVERLAY_OVERLAY_H_INCLUDED +#include #include #include #include @@ -101,6 +102,11 @@ public: std::size_t size () = 0; + /** Returns information reported to the crawl cgi command. */ + virtual + Json::Value + crawl() = 0; + /** Return diagnostics on the status of all peers. @deprecated This is superceded by PropertyStream */ diff --git a/src/ripple/overlay/README.md b/src/ripple/overlay/README.md index fef265257f..a4c831746a 100644 --- a/src/ripple/overlay/README.md +++ b/src/ripple/overlay/README.md @@ -51,9 +51,6 @@ operating in the Client Handler role accepts incoming connections from clients and services them through the JSON-RPC interface. A peer can operate in either the leaf or superpeer roles while also operating as a client handler. - - - ## Handshake To establish a protocol connection, a peer makes an outgoing TLS encrypted @@ -69,8 +66,8 @@ Upgrade: RTXP/1.2, RTXP/1.3 Connection: Upgrade Connect-As: Leaf, Peer Accept-Encoding: identity, zlib, snappy -Protocol-Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ -Protocol-Session-Cookie: 71ED064155FFADFA38782C5E0158CB26 +Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ +Session-Signature: 71ED064155FFADFA38782C5E0158CB26 ``` Upon receipt of a well-formed HTTP request the remote peer will send back a @@ -84,8 +81,8 @@ Upgrade: RTXP/1.2 Connection: Upgrade Connect-As: Leaf Transfer-Encoding: snappy -Protocol-Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ -Protocol-Session-Cookie: 71ED064155FFADFA38782C5E0158CB26 +Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ +Session-Signature: 71ED064155FFADFA38782C5E0158CB26 ``` If the remote peer has no available slots, the HTTP status code 503 (Service @@ -163,22 +160,29 @@ Content-Type: application/json of elements specified in the request. If a server does not recognize any of the connection types it must return a HTTP error response. -* `Protocol-Public-Key` +* `Public-Key` - This field value must be present, and contain a Base58 encoded value used + This field value must be present, and contain a base 64 encoded value used as a server public key identifier. -* `Protocol-Session-Proof` +* `Session-Signature` - This field must be present (TODO) + This field must be present. It contains a cryptographic token formed + from the SHA512 hash of the shared data exchanged during SSL handshaking. + For more details see the corresponding source code. -* _User Defined_ +* `Crawl` (optional) + + If present, and the value is "public" then neighbors will report the IP + address to crawler requests. If absent, neighbor's default behavior is to + not report IP addresses. + +* _User Defined_ (Unimplemented) The rippled operator may specify additional, optional fields and values through the configuration. These headers will be transmitted in the corresponding request or response messages. - --- [overlay_network]: http://en.wikipedia.org/wiki/Overlay_network diff --git a/src/ripple/overlay/impl/ConnectAttempt.cpp b/src/ripple/overlay/impl/ConnectAttempt.cpp index 9d9c17b65f..f3d53a0e26 100644 --- a/src/ripple/overlay/impl/ConnectAttempt.cpp +++ b/src/ripple/overlay/impl/ConnectAttempt.cpp @@ -218,7 +218,8 @@ ConnectAttempt::onHandshake (error_code ec) return close(); // makeSharedValue logs beast::http::message req = makeRequest( - remote_endpoint_.address()); + ! overlay_.peerFinder().config().peerPrivate, + remote_endpoint_.address()); auto const hello = buildHello (sharedValue, getApp()); appendHello (req, hello); @@ -509,7 +510,7 @@ ConnectAttempt::onReadBody (error_code ec, //-------------------------------------------------------------------------- beast::http::message -ConnectAttempt::makeRequest ( +ConnectAttempt::makeRequest (bool crawl, boost::asio::ip::address const& remote_address) { beast::http::message m; @@ -521,13 +522,7 @@ ConnectAttempt::makeRequest ( //std::string("RTXP/") + to_string (BuildInfo::getCurrentProtocol())); m.headers.append ("Connection", "Upgrade"); m.headers.append ("Connect-As", "Peer"); - //m.headers.append ("Connect-As", "Leaf, Peer"); - //m.headers.append ("Accept-Encoding", "identity"); - //m.headers.append ("Local-Address", stream_. - //m.headers.append ("X-Try-IPs", "192.168.0.1:51234"); - //m.headers.append ("X-Try-IPs", "208.239.114.74:51234"); - //m.headers.append ("A", "BC"); - //m.headers.append ("Content-Length", "0"); + m.headers.append ("Crawl", crawl ? "public" : "private"); return m; } diff --git a/src/ripple/overlay/impl/ConnectAttempt.h b/src/ripple/overlay/impl/ConnectAttempt.h index 4e39904c44..c88f42538a 100644 --- a/src/ripple/overlay/impl/ConnectAttempt.h +++ b/src/ripple/overlay/impl/ConnectAttempt.h @@ -106,7 +106,8 @@ private: static beast::http::message - makeRequest (boost::asio::ip::address const& remote_address); + makeRequest (bool crawl, + boost::asio::ip::address const& remote_address); template void processResponse (beast::http::message const& m, diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index f94f84d0c8..da1fadb427 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +194,8 @@ OverlayImpl::onHandoff (std::unique_ptr && ssl_bundle, beast::Journal journal (sink); Handoff handoff; + if (processRequest(request, handoff)) + return handoff; if (! isPeerUpgrade(request)) return handoff; @@ -461,8 +464,9 @@ OverlayImpl::onPrepare() auto const port = serverHandler_.setup().overlay.port; + config.peerPrivate = getConfig().PEER_PRIVATE; config.wantIncoming = - (! getConfig ().PEER_PRIVATE) && (port != 0); + (! config.peerPrivate) && (port != 0); // if it's a private peer or we are running as standalone // automatic connections would defeat the purpose. config.autoConnect = @@ -604,6 +608,34 @@ OverlayImpl::size() return m_publicKeyMap.size (); } +Json::Value +OverlayImpl::crawl() +{ + Json::Value jv; + auto& av = jv["active"] = Json::Value(Json::arrayValue); + std::lock_guard lock (mutex_); + for (auto const& e : m_publicKeyMap) + { + auto const sp = e.second.lock(); + if (sp) + { + auto& pv = av.append(Json::Value(Json::objectValue)); + pv["type"] = "peer"; + pv["public_key"] = beast::base64_encode( + sp->getNodePublic().getNodePublic().data(), + sp->getNodePublic().getNodePublic().size()); + if (sp->crawl()) + { + if (sp->slot()->inbound()) + pv["ip"] = sp->getRemoteAddress().address().to_string(); + else + pv["ip"] = sp->getRemoteAddress().to_string(); + } + } + } + return jv; +} + // Returns information on verified peers. Json::Value OverlayImpl::json () @@ -611,6 +643,23 @@ OverlayImpl::json () return foreach (get_peer_json()); } +bool +OverlayImpl::processRequest (beast::http::message const& req, + Handoff& handoff) +{ + if (req.url() != "/crawl") + return false; + + beast::http::message resp; + resp.request(false); + resp.status(200); + resp.reason("OK"); + Json::Value v; + v["overlay"] = crawl(); + handoff.response = HTTP::make_JsonWriter(resp, v); + return true; +} + Overlay::PeerSequence OverlayImpl::getActivePeers() { diff --git a/src/ripple/overlay/impl/OverlayImpl.h b/src/ripple/overlay/impl/OverlayImpl.h index df37ed3d9f..6db57fef46 100644 --- a/src/ripple/overlay/impl/OverlayImpl.h +++ b/src/ripple/overlay/impl/OverlayImpl.h @@ -207,9 +207,16 @@ private: std::size_t size() override; + Json::Value + crawl() override; + Json::Value json() override; + bool + processRequest (beast::http::message const& req, + Handoff& handoff); + //-------------------------------------------------------------------------- // diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index a435024695..b4ee59b98b 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -187,6 +187,15 @@ PeerImp::charge (Resource::Charge const& fee) //------------------------------------------------------------------------------ +bool +PeerImp::crawl() const +{ + auto const iter = http_message_.headers.find("Crawl"); + if (iter == http_message_.headers.end()) + return false; + return beast::ci_equal(iter->second, "public"); +} + Json::Value PeerImp::json() { @@ -407,28 +416,6 @@ PeerImp::makePrefix(id_t id) return ss.str(); } -beast::http::message -PeerImp::makeRequest(boost::asio::ip::address const& remote_address) -{ - beast::http::message m; - m.method (beast::http::method_t::http_get); - m.url ("/"); - m.version (1, 1); - m.headers.append ("User-Agent", BuildInfo::getFullVersionString()); - m.headers.append ("Upgrade", "RTXP/1.3"); - //std::string("RTXP/") + to_string (BuildInfo::getCurrentProtocol())); - m.headers.append ("Connection", "Upgrade"); - m.headers.append ("Connect-As", "Peer"); - //m.headers.append ("Connect-As", "Leaf, Peer"); - //m.headers.append ("Accept-Encoding", "identity"); - //m.headers.append ("Local-Address", stream_. - //m.headers.append ("X-Try-IPs", "192.168.0.1:51234"); - //m.headers.append ("X-Try-IPs", "208.239.114.74:51234"); - //m.headers.append ("A", "BC"); - //m.headers.append ("Content-Length", "0"); - return m; -} - void PeerImp::onTimer (error_code const& ec) { @@ -497,7 +484,9 @@ void PeerImp::doAccept() // TODO Apply headers to connection state. - auto resp = makeResponse(http_message_, sharedValue); + auto resp = makeResponse( + ! overlay_.peerFinder().config().peerPrivate, + http_message_, sharedValue); beast::http::write (write_buffer_, resp); auto const protocol = BuildInfo::make_protocol(hello_.protoversion()); @@ -536,8 +525,8 @@ void PeerImp::doAccept() } beast::http::message -PeerImp::makeResponse (beast::http::message const& req, - uint256 const& sharedValue) +PeerImp::makeResponse (bool crawl, + beast::http::message const& req, uint256 const& sharedValue) { beast::http::message resp; resp.request(false); @@ -548,6 +537,7 @@ PeerImp::makeResponse (beast::http::message const& req, resp.headers.append("Upgrade", "RTXP/1.2"); resp.headers.append("Connect-AS", "Peer"); resp.headers.append("Server", BuildInfo::getFullVersionString()); + resp.headers.append ("Crawl", crawl ? "public" : "private"); protocol::TMHello hello = buildHello(sharedValue, getApp()); appendHello(resp, hello); return resp; diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index 75eee1bccf..4616f1943e 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -109,49 +109,37 @@ private: // the current conditions as closely as possible. beast::IP::Endpoint remote_address_; - // These is up here to prevent warnings about order of initializations + // These are up here to prevent warnings about order of initializations // OverlayImpl& overlay_; bool m_inbound; - State state_; // Current state bool detaching_ = false; - // Node public key of peer. RippleAddress publicKey_; - std::string name_; - uint256 sharedValue_; // The indices of the smallest and largest ledgers this peer has available // LedgerIndex minLedger_ = 0; LedgerIndex maxLedger_ = 0; - uint256 closedLedgerHash_; uint256 previousLedgerHash_; - std::list recentLedgers_; std::list recentTxSets_; mutable std::mutex recentLock_; - protocol::TMStatusChange last_status_; protocol::TMHello hello_; - Resource::Consumer usage_; PeerFinder::Slot::ptr slot_; - beast::asio::streambuf read_buffer_; beast::http::message http_message_; - boost::optional http_parser_; beast::http::body http_body_; beast::asio::streambuf write_buffer_; std::queue send_queue_; bool gracefulClose_ = false; - std::unique_ptr load_event_; - std::unique_ptr validatorsConnection_; //-------------------------------------------------------------------------- @@ -234,6 +222,9 @@ public: return id_; } + bool + crawl() const; + bool cluster() const override { @@ -300,10 +291,6 @@ private: std::string makePrefix(id_t id); - static - beast::http::message - makeRequest (boost::asio::ip::address const& remote_address); - // Called when the timer wait completes void onTimer (boost::system::error_code const& ec); @@ -320,7 +307,7 @@ private: static beast::http::message - makeResponse (beast::http::message const& req, + makeResponse (bool crawl, beast::http::message const& req, uint256 const& sharedValue); void diff --git a/src/ripple/peerfinder/Manager.h b/src/ripple/peerfinder/Manager.h index 8d9cd62fd6..7fada8eac1 100644 --- a/src/ripple/peerfinder/Manager.h +++ b/src/ripple/peerfinder/Manager.h @@ -58,6 +58,9 @@ struct Config */ double outPeers; + /** `true` if we want our IP address kept private. */ + bool peerPrivate = true; + /** `true` if we want to accept incoming connections. */ bool wantIncoming; @@ -136,6 +139,11 @@ public: */ virtual void setConfig (Config const& config) = 0; + /** Returns the configuration for the manager. */ + virtual + Config + config() = 0; + /** Add a peer that should always be connected. This is useful for maintaining a private cluster of peers. The string is the name as specified in the configuration diff --git a/src/ripple/peerfinder/impl/Logic.h b/src/ripple/peerfinder/impl/Logic.h index faf20f50d9..841f51b790 100644 --- a/src/ripple/peerfinder/impl/Logic.h +++ b/src/ripple/peerfinder/impl/Logic.h @@ -180,6 +180,13 @@ public: state->counts.onConfig (state->config); } + Config + config() + { + typename SharedState::Access state (m_state); + return state->config; + } + void addFixedPeer (std::string const& name, beast::IP::Endpoint const& ep) diff --git a/src/ripple/peerfinder/impl/Manager.cpp b/src/ripple/peerfinder/impl/Manager.cpp index 3ec935b687..e1316ef623 100644 --- a/src/ripple/peerfinder/impl/Manager.cpp +++ b/src/ripple/peerfinder/impl/Manager.cpp @@ -94,6 +94,12 @@ public: m_logic.config (config); } + Config + config() override + { + return m_logic.config(); + } + void addFixedPeer (std::string const& name, std::vector const& addresses) override { diff --git a/src/ripple/peerfinder/impl/PeerfinderConfig.cpp b/src/ripple/peerfinder/impl/PeerfinderConfig.cpp index 4f10d14465..4b98c0ff46 100644 --- a/src/ripple/peerfinder/impl/PeerfinderConfig.cpp +++ b/src/ripple/peerfinder/impl/PeerfinderConfig.cpp @@ -24,7 +24,7 @@ namespace ripple { namespace PeerFinder { -Config::Config () +Config::Config() : maxPeers (Tuning::defaultMaxPeers) , outPeers (calcOutPeers ()) , wantIncoming (true) diff --git a/src/ripple/server/impl/ServerHandlerImp.cpp b/src/ripple/server/impl/ServerHandlerImp.cpp index 8753f943d9..592b111f59 100644 --- a/src/ripple/server/impl/ServerHandlerImp.cpp +++ b/src/ripple/server/impl/ServerHandlerImp.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include