Add /crawl cgi request feature to peer protocol (RIPD-729):

This adds support for a cgi /crawl request, issued over HTTPS to the configured
peer protocol port. The response to the request is a JSON object containing
the node public key, type, and IP address of each directly connected neighbor.
The IP address is suppressed unless the neighbor has requested its address
to be revealed by adding "Crawl: public" to its HTTP headers. This field is
currently set by the peer_private option in the rippled.cfg file.

doc/rippled-example.cfg

@@ -688,9 +688,7 @@
 #       The 'temp_db' configures a look-aside cache for high volume storage
 #           which doesn't necessarily persist between server launches. This
 #           is an optional configuration parameter. If it is left out then
-#           no look-aside database is created or used. Use of temp_db may
+#           no look-aside database is created or used.
-#           improve performance in some environments, but should be tested
-#           with and without to determine if beneficial.
 #
 #       The 'import_db' is used with the '--import' command line option to
 #           migrate the specified database into the current database given

src/ripple/overlay/Overlay.h

@@ -20,6 +20,7 @@
 #ifndef RIPPLE_OVERLAY_OVERLAY_H_INCLUDED
 #define RIPPLE_OVERLAY_OVERLAY_H_INCLUDED
 
+#include <ripple/json/json_value.h>
 #include <ripple/overlay/Peer.h>
 #include <ripple/server/Handoff.h>
 #include <beast/asio/ssl_bundle.h>
@@ -101,6 +102,11 @@ public:
     std::size_t
     size () = 0;
 
+    /** Returns information reported to the crawl cgi command. */
+    virtual
+    Json::Value
+    crawl() = 0;
+
     /** Return diagnostics on the status of all peers.
         @deprecated This is superceded by PropertyStream
     */

src/ripple/overlay/README.md

@@ -51,9 +51,6 @@ operating in the Client Handler role accepts incoming connections from clients
 and services them through the JSON-RPC interface. A peer can operate in either
 the leaf or superpeer roles while also operating as a client handler.
 
-
-
-
 ## Handshake
 
 To establish a protocol connection, a peer makes an outgoing TLS encrypted
@@ -69,8 +66,8 @@ Upgrade: RTXP/1.2, RTXP/1.3
 Connection: Upgrade
 Connect-As: Leaf, Peer
 Accept-Encoding: identity, zlib, snappy
-Protocol-Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ
+Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ
-Protocol-Session-Cookie: 71ED064155FFADFA38782C5E0158CB26
+Session-Signature: 71ED064155FFADFA38782C5E0158CB26
 ```
 
 Upon receipt of a well-formed HTTP request the remote peer will send back a
@@ -84,8 +81,8 @@ Upgrade: RTXP/1.2
 Connection: Upgrade
 Connect-As: Leaf
 Transfer-Encoding: snappy
-Protocol-Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ
+Public-Key: aBRoQibi2jpDofohooFuzZi9nEzKw9Zdfc4ExVNmuXHaJpSPh8uJ
-Protocol-Session-Cookie: 71ED064155FFADFA38782C5E0158CB26
+Session-Signature: 71ED064155FFADFA38782C5E0158CB26
 ```
 
 If the remote peer has no available slots, the HTTP status code 503 (Service
@@ -163,22 +160,29 @@ Content-Type: application/json
     of elements specified in the request. If a server does not recognize any
     of the connection types it must return a HTTP error response.
 
-* `Protocol-Public-Key`
+* `Public-Key`
 
-    This field value must be present, and contain a Base58 encoded value used
+    This field value must be present, and contain a base 64 encoded value used
     as a server public key identifier.
 
-* `Protocol-Session-Proof`
+* `Session-Signature`
 
-    This field must be present (TODO)
+    This field must be present. It contains a cryptographic token formed
+    from the SHA512 hash of the shared data exchanged during SSL handshaking.
+    For more details see the corresponding source code.
 
-* _User Defined_
+* `Crawl` (optional)
+
+    If present, and the value is "public" then neighbors will report the IP
+    address to crawler requests. If absent, neighbor's default behavior is to
+    not report IP addresses.
+
+* _User Defined_ (Unimplemented)
 
     The rippled operator may specify additional, optional fields and values
     through the configuration. These headers will be transmitted in the
     corresponding request or response messages.
 
-
 ---
 
 [overlay_network]: http://en.wikipedia.org/wiki/Overlay_network

src/ripple/overlay/impl/ConnectAttempt.cpp

@@ -218,7 +218,8 @@ ConnectAttempt::onHandshake (error_code ec)
         return close(); // makeSharedValue logs
 
     beast::http::message req = makeRequest(
-        remote_endpoint_.address());
+        ! overlay_.peerFinder().config().peerPrivate,
+            remote_endpoint_.address());
     auto const hello = buildHello (sharedValue, getApp());
     appendHello (req, hello);
 
@@ -509,7 +510,7 @@ ConnectAttempt::onReadBody (error_code ec,
 //--------------------------------------------------------------------------
 
 beast::http::message
-ConnectAttempt::makeRequest (
+ConnectAttempt::makeRequest (bool crawl,
     boost::asio::ip::address const& remote_address)
 {
     beast::http::message m;
@@ -521,13 +522,7 @@ ConnectAttempt::makeRequest (
         //std::string("RTXP/") + to_string (BuildInfo::getCurrentProtocol()));
     m.headers.append ("Connection", "Upgrade");
     m.headers.append ("Connect-As", "Peer");
-    //m.headers.append ("Connect-As", "Leaf, Peer");
+    m.headers.append ("Crawl", crawl ? "public" : "private");
-    //m.headers.append ("Accept-Encoding", "identity");
-    //m.headers.append ("Local-Address", stream_.
-    //m.headers.append ("X-Try-IPs", "192.168.0.1:51234");
-    //m.headers.append ("X-Try-IPs", "208.239.114.74:51234");
-    //m.headers.append ("A", "BC");
-    //m.headers.append ("Content-Length", "0");
     return m;
 }
 

src/ripple/overlay/impl/ConnectAttempt.h

@@ -106,7 +106,8 @@ private:
 
     static
     beast::http::message
-    makeRequest (boost::asio::ip::address const& remote_address);
+    makeRequest (bool crawl,
+        boost::asio::ip::address const& remote_address);
 
     template <class Streambuf>
     void processResponse (beast::http::message const& m,

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -27,6 +27,7 @@
 #include <ripple/overlay/impl/TMHello.h>
 #include <ripple/peerfinder/make_Manager.h>
 #include <beast/ByteOrder.h>
+#include <beast/crypto/base64.h>
 #include <beast/http/rfc2616.h>
 #include <beast/utility/ci_char_traits.h>
 #include <beast/utility/WrappedSink.h>
@@ -193,6 +194,8 @@ OverlayImpl::onHandoff (std::unique_ptr <beast::asio::ssl_bundle>&& ssl_bundle,
     beast::Journal journal (sink);
 
     Handoff handoff;
+    if (processRequest(request, handoff))
+        return handoff;
     if (! isPeerUpgrade(request))
         return handoff;
 
@@ -461,8 +464,9 @@ OverlayImpl::onPrepare()
 
     auto const port = serverHandler_.setup().overlay.port;
 
+    config.peerPrivate = getConfig().PEER_PRIVATE;
     config.wantIncoming =
-        (! getConfig ().PEER_PRIVATE) && (port != 0);
+        (! config.peerPrivate) && (port != 0);
     // if it's a private peer or we are running as standalone
     // automatic connections would defeat the purpose.
     config.autoConnect =
@@ -604,6 +608,34 @@ OverlayImpl::size()
     return m_publicKeyMap.size ();
 }
 
+Json::Value
+OverlayImpl::crawl()
+{
+    Json::Value jv;
+    auto& av = jv["active"] = Json::Value(Json::arrayValue);
+    std::lock_guard <decltype(mutex_)> lock (mutex_);
+    for (auto const& e : m_publicKeyMap)
+    {
+        auto const sp = e.second.lock();
+        if (sp)
+        {
+            auto& pv = av.append(Json::Value(Json::objectValue));
+            pv["type"] = "peer";
+            pv["public_key"] = beast::base64_encode(
+                sp->getNodePublic().getNodePublic().data(),
+                    sp->getNodePublic().getNodePublic().size());
+            if (sp->crawl())
+            {
+                if (sp->slot()->inbound())
+                    pv["ip"] = sp->getRemoteAddress().address().to_string();
+                else
+                    pv["ip"] = sp->getRemoteAddress().to_string();
+            }
+        }
+    }
+    return jv;
+}
+
 // Returns information on verified peers.
 Json::Value
 OverlayImpl::json ()
@@ -611,6 +643,23 @@ OverlayImpl::json ()
     return foreach (get_peer_json());
 }
 
+bool
+OverlayImpl::processRequest (beast::http::message const& req,
+    Handoff& handoff)
+{
+    if (req.url() != "/crawl")
+        return false;
+
+    beast::http::message resp;
+    resp.request(false);
+    resp.status(200);
+    resp.reason("OK");
+    Json::Value v;
+    v["overlay"] = crawl();
+    handoff.response = HTTP::make_JsonWriter(resp, v);
+    return true;
+}
+
 Overlay::PeerSequence
 OverlayImpl::getActivePeers()
 {

src/ripple/overlay/impl/OverlayImpl.h

@@ -207,9 +207,16 @@ private:
     std::size_t
     size() override;
 
+    Json::Value
+    crawl() override;
+
     Json::Value
     json() override;
 
+    bool
+    processRequest (beast::http::message const& req,
+        Handoff& handoff);
+
     //--------------------------------------------------------------------------
 
     //

src/ripple/overlay/impl/PeerImp.cpp

@@ -187,6 +187,15 @@ PeerImp::charge (Resource::Charge const& fee)
 
 //------------------------------------------------------------------------------
 
+bool
+PeerImp::crawl() const
+{
+    auto const iter = http_message_.headers.find("Crawl");
+    if (iter == http_message_.headers.end())
+        return false;
+    return beast::ci_equal(iter->second, "public");
+}
+
 Json::Value
 PeerImp::json()
 {
@@ -407,28 +416,6 @@ PeerImp::makePrefix(id_t id)
     return ss.str();
 }
 
-beast::http::message
-PeerImp::makeRequest(boost::asio::ip::address const& remote_address)
-{
-    beast::http::message m;
-    m.method (beast::http::method_t::http_get);
-    m.url ("/");
-    m.version (1, 1);
-    m.headers.append ("User-Agent", BuildInfo::getFullVersionString());
-    m.headers.append ("Upgrade", "RTXP/1.3");
-        //std::string("RTXP/") + to_string (BuildInfo::getCurrentProtocol()));
-    m.headers.append ("Connection", "Upgrade");
-    m.headers.append ("Connect-As", "Peer");
-    //m.headers.append ("Connect-As", "Leaf, Peer");
-    //m.headers.append ("Accept-Encoding", "identity");
-    //m.headers.append ("Local-Address", stream_.
-    //m.headers.append ("X-Try-IPs", "192.168.0.1:51234");
-    //m.headers.append ("X-Try-IPs", "208.239.114.74:51234");
-    //m.headers.append ("A", "BC");
-    //m.headers.append ("Content-Length", "0");
-    return m;
-}
-
 void
 PeerImp::onTimer (error_code const& ec)
 {
@@ -497,7 +484,9 @@ void PeerImp::doAccept()
 
     // TODO Apply headers to connection state.
 
-    auto resp = makeResponse(http_message_, sharedValue);
+    auto resp = makeResponse(
+        ! overlay_.peerFinder().config().peerPrivate,
+            http_message_, sharedValue);
     beast::http::write (write_buffer_, resp);
 
     auto const protocol = BuildInfo::make_protocol(hello_.protoversion());
@@ -536,8 +525,8 @@ void PeerImp::doAccept()
 }
 
 beast::http::message
-PeerImp::makeResponse (beast::http::message const& req,
+PeerImp::makeResponse (bool crawl,
-    uint256 const& sharedValue)
+    beast::http::message const& req, uint256 const& sharedValue)
 {
     beast::http::message resp;
     resp.request(false);
@@ -548,6 +537,7 @@ PeerImp::makeResponse (beast::http::message const& req,
     resp.headers.append("Upgrade", "RTXP/1.2");
     resp.headers.append("Connect-AS", "Peer");
     resp.headers.append("Server", BuildInfo::getFullVersionString());
+    resp.headers.append ("Crawl", crawl ? "public" : "private");
     protocol::TMHello hello = buildHello(sharedValue, getApp());
     appendHello(resp, hello);
     return resp;

src/ripple/overlay/impl/PeerImp.h

@@ -109,49 +109,37 @@ private:
     // the current conditions as closely as possible.
     beast::IP::Endpoint remote_address_;
 
-    // These is up here to prevent warnings about order of initializations
+    // These are up here to prevent warnings about order of initializations
     //
     OverlayImpl& overlay_;
     bool m_inbound;
-
     State state_;          // Current state
     bool detaching_ = false;
-
     // Node public key of peer.
     RippleAddress publicKey_;
-
     std::string name_;
-
     uint256 sharedValue_;
 
     // The indices of the smallest and largest ledgers this peer has available
     //
     LedgerIndex minLedger_ = 0;
     LedgerIndex maxLedger_ = 0;
-
     uint256 closedLedgerHash_;
     uint256 previousLedgerHash_;
-
     std::list<uint256> recentLedgers_;
     std::list<uint256> recentTxSets_;
     mutable std::mutex recentLock_;
-
     protocol::TMStatusChange last_status_;
     protocol::TMHello hello_;
-
     Resource::Consumer usage_;
     PeerFinder::Slot::ptr slot_;
-
     beast::asio::streambuf read_buffer_;
     beast::http::message http_message_;
-    boost::optional <beast::http::parser> http_parser_;
     beast::http::body http_body_;
     beast::asio::streambuf write_buffer_;
     std::queue<Message::pointer> send_queue_;
     bool gracefulClose_ = false;
-
     std::unique_ptr <LoadEvent> load_event_;
-
     std::unique_ptr<Validators::Connection> validatorsConnection_;
 
     //--------------------------------------------------------------------------
@@ -234,6 +222,9 @@ public:
         return id_;
     }
 
+    bool
+    crawl() const;
+
     bool
     cluster() const override
     {
@@ -300,10 +291,6 @@ private:
     std::string
     makePrefix(id_t id);
 
-    static
-    beast::http::message
-    makeRequest (boost::asio::ip::address const& remote_address);
-
     // Called when the timer wait completes
     void
     onTimer (boost::system::error_code const& ec);
@@ -320,7 +307,7 @@ private:
 
     static
     beast::http::message
-    makeResponse (beast::http::message const& req,
+    makeResponse (bool crawl, beast::http::message const& req,
         uint256 const& sharedValue);
 
     void

src/ripple/peerfinder/Manager.h

@@ -58,6 +58,9 @@ struct Config
     */
     double outPeers;
 
+    /** `true` if we want our IP address kept private. */
+    bool peerPrivate = true;
+
     /** `true` if we want to accept incoming connections. */
     bool wantIncoming;
 
@@ -136,6 +139,11 @@ public:
     */
     virtual void setConfig (Config const& config) = 0;
 
+    /** Returns the configuration for the manager. */
+    virtual
+    Config
+    config() = 0;
+
     /** Add a peer that should always be connected.
         This is useful for maintaining a private cluster of peers.
         The string is the name as specified in the configuration

src/ripple/peerfinder/impl/Logic.h

@@ -180,6 +180,13 @@ public:
         state->counts.onConfig (state->config);
     }
 
+    Config
+    config()
+    {
+        typename SharedState::Access state (m_state);
+        return state->config;
+    }
+
     void
     addFixedPeer (std::string const& name,
         beast::IP::Endpoint const& ep)

src/ripple/peerfinder/impl/Manager.cpp

@@ -94,6 +94,12 @@ public:
         m_logic.config (config);
     }
 
+    Config
+    config() override
+    {
+        return m_logic.config();
+    }
+
     void addFixedPeer (std::string const& name,
         std::vector <beast::IP::Endpoint> const& addresses) override
     {

src/ripple/peerfinder/impl/PeerfinderConfig.cpp

@@ -24,7 +24,7 @@
 namespace ripple {
 namespace PeerFinder {
 
-Config::Config ()
+Config::Config()
     : maxPeers (Tuning::defaultMaxPeers)
     , outPeers (calcOutPeers ())
     , wantIncoming (true)

src/ripple/server/impl/ServerHandlerImp.cpp

@@ -20,6 +20,7 @@
 #include <BeastConfig.h>
 #include <ripple/app/main/Application.h>
 #include <ripple/json/json_reader.h>
+#include <ripple/server/JsonWriter.h>
 #include <ripple/server/make_ServerHandler.h>
 #include <ripple/server/impl/JSONRPCUtil.h>
 #include <ripple/server/impl/ServerHandlerImp.h>