Merge master (1.2.4) into develop (1.3.0-b2)

src/ripple/app/misc/ValidatorSite.h

@@ -58,7 +58,8 @@ namespace ripple {
 
     @li @c "version": 1
 
-    @li @c "refreshInterval" (optional)
+    @li @c "refreshInterval" (optional, integer minutes).
+        This value is clamped internally to [1,1440] (1 min - 1 day)
 */
 class ValidatorSite
 {
@@ -125,11 +126,15 @@ private:
     // The configured list of URIs for fetching lists
     std::vector<Site> sites_;
 
+    // time to allow for requests to complete
+    const std::chrono::seconds requestTimeout_;
+
 public:
     ValidatorSite (
         boost::asio::io_service& ios,
         ValidatorList& validators,
-        beast::Journal j);
+        beast::Journal j,
+        std::chrono::seconds timeout = std::chrono::seconds{20});
     ~ValidatorSite ();
 
     /** Load configured site URIs.
@@ -184,8 +189,15 @@ public:
 
 private:
     /// Queue next site to be fetched
+    /// lock over state_mutex_ required
     void
-    setTimer ();
+    setTimer (std::lock_guard<std::mutex>&);
+
+    /// request took too long
+    void
+    onRequestTimeout (
+        std::size_t siteIdx,
+        error_code const& ec);
 
     /// Fetch site whose time has come
     void

src/ripple/app/misc/impl/ValidatorSite.cpp

@@ -26,15 +26,15 @@
 #include <ripple/basics/Slice.h>
 #include <ripple/json/json_reader.h>
 #include <ripple/protocol/JsonFields.h>
+#include <boost/algorithm/clamp.hpp>
 #include <boost/regex.hpp>
 #include <algorithm>
 
 namespace ripple {
 
-// default site query frequency - 5 minutes
-auto constexpr DEFAULT_REFRESH_INTERVAL = std::chrono::minutes{5};
-auto constexpr ERROR_RETRY_INTERVAL = std::chrono::seconds{30};
-unsigned short constexpr MAX_REDIRECTS = 3;
+auto           constexpr default_refresh_interval = std::chrono::minutes{5};
+auto           constexpr error_retry_interval     = std::chrono::seconds{30};
+unsigned short constexpr max_redirects            = 3;
 
 ValidatorSite::Site::Resource::Resource (std::string uri_)
     : uri {std::move(uri_)}
@@ -82,7 +82,7 @@ ValidatorSite::Site::Site (std::string uri)
     : loadedResource {std::make_shared<Resource>(std::move(uri))}
     , startingResource {loadedResource}
     , redirCount {0}
-    , refreshInterval {DEFAULT_REFRESH_INTERVAL}
+    , refreshInterval {default_refresh_interval}
     , nextRefresh {clock_type::now()}
 {
 }
@@ -90,7 +90,8 @@ ValidatorSite::Site::Site (std::string uri)
 ValidatorSite::ValidatorSite (
     boost::asio::io_service& ios,
     ValidatorList& validators,
-    beast::Journal j)
+    beast::Journal j,
+    std::chrono::seconds timeout)
     : ios_ (ios)
     , validators_ (validators)
     , j_ (j)
@@ -98,6 +99,7 @@ ValidatorSite::ValidatorSite (
     , fetching_ (false)
     , pending_ (false)
     , stopping_ (false)
+    , requestTimeout_ (timeout)
 {
 }
 
@@ -153,7 +155,7 @@ ValidatorSite::start ()
 {
     std::lock_guard <std::mutex> lock{state_mutex_};
     if (timer_.expires_at() == clock_type::time_point{})
-        setTimer ();
+        setTimer (lock);
 }
 
 void
@@ -168,20 +170,29 @@ ValidatorSite::stop()
 {
     std::unique_lock<std::mutex> lock{state_mutex_};
     stopping_ = true;
-    cv_.wait(lock, [&]{ return ! fetching_; });
-
+    // work::cancel() must be called before the
+    // cv wait in order to kick any asio async operations
+    // that might be pending.
     if(auto sp = work_.lock())
         sp->cancel();
+    cv_.wait(lock, [&]{ return ! fetching_; });
 
-    error_code ec;
-    timer_.cancel(ec);
+    // docs indicate cancel() can throw, but this should be
+    // reconsidered if it changes to noexcept
+    try
+    {
+        timer_.cancel();
+    }
+    catch (boost::system::system_error const&)
+    {
+    }
     stopping_ = false;
     pending_ = false;
     cv_.notify_all();
 }
 
 void
-ValidatorSite::setTimer ()
+ValidatorSite::setTimer (std::lock_guard<std::mutex>& state_lock)
 {
     std::lock_guard <std::mutex> lock{sites_mutex_};
 
@@ -196,8 +207,11 @@ ValidatorSite::setTimer ()
         pending_ = next->nextRefresh <= clock_type::now();
         cv_.notify_all();
         timer_.expires_at (next->nextRefresh);
-        timer_.async_wait (std::bind (&ValidatorSite::onTimer, this,
-            std::distance (sites_.begin (), next), std::placeholders::_1));
+        auto idx = std::distance (sites_.begin (), next);
+        timer_.async_wait ([this, idx] (boost::system::error_code const& ec)
+        {
+            this->onTimer (idx, ec);
+        });
     }
 }
 
@@ -205,22 +219,42 @@ void
 ValidatorSite::makeRequest (
     std::shared_ptr<Site::Resource> resource,
     std::size_t siteIdx,
-    std::lock_guard<std::mutex>& lock)
+    std::lock_guard<std::mutex>& sites_lock)
 {
     fetching_ = true;
     sites_[siteIdx].activeResource = resource;
     std::shared_ptr<detail::Work> sp;
-    auto onFetch =
-        [this, siteIdx] (error_code const& err, detail::response_type&& resp)
+    auto timeoutCancel =
+        [this] ()
         {
+            std::lock_guard <std::mutex> lock_state{state_mutex_};
+            // docs indicate cancel_one() can throw, but this
+            // should be reconsidered if it changes to noexcept
+            try
+            {
+                timer_.cancel_one();
+            }
+            catch (boost::system::system_error const&)
+            {
+            }
+        };
+    auto onFetch =
+        [this, siteIdx, timeoutCancel] (
+            error_code const& err, detail::response_type&& resp)
+        {
+            timeoutCancel ();
             onSiteFetch (err, std::move(resp), siteIdx);
         };
 
     auto onFetchFile =
-        [this, siteIdx] (error_code const& err, std::string const& resp)
-    {
-        onTextFetch (err, resp, siteIdx);
-    };
+        [this, siteIdx, timeoutCancel] (
+            error_code const& err, std::string const& resp)
+        {
+            timeoutCancel ();
+            onTextFetch (err, resp, siteIdx);
+        };
+
+    JLOG (j_.debug()) << "Starting request for " << resource->uri;
 
     if (resource->pUrl.scheme == "https")
     {
@@ -252,6 +286,34 @@ ValidatorSite::makeRequest (
 
     work_ = sp;
     sp->run ();
+    // start a timer for the request, which shouldn't take more
+    // than requestTimeout_ to complete
+    std::lock_guard <std::mutex> lock_state{state_mutex_};
+    timer_.expires_after (requestTimeout_);
+    timer_.async_wait ([this, siteIdx] (boost::system::error_code const& ec)
+        {
+            this->onRequestTimeout (siteIdx, ec);
+        });
+}
+
+void
+ValidatorSite::onRequestTimeout (
+    std::size_t siteIdx,
+    error_code const& ec)
+{
+    if (ec)
+        return;
+
+    {
+        std::lock_guard <std::mutex> lock_site{sites_mutex_};
+        JLOG (j_.warn()) <<
+            "Request for " << sites_[siteIdx].activeResource->uri <<
+            " took too long";
+    }
+
+    std::lock_guard<std::mutex> lock_state{state_mutex_};
+    if(auto sp = work_.lock())
+        sp->cancel();
 }
 
 void
@@ -268,14 +330,12 @@ ValidatorSite::onTimer (
         return;
     }
 
-    std::lock_guard <std::mutex> lock{sites_mutex_};
-    sites_[siteIdx].nextRefresh =
-        clock_type::now() + sites_[siteIdx].refreshInterval;
-
-    assert(! fetching_);
-    sites_[siteIdx].redirCount = 0;
     try
     {
+        std::lock_guard <std::mutex> lock{sites_mutex_};
+        sites_[siteIdx].nextRefresh =
+            clock_type::now() + sites_[siteIdx].refreshInterval;
+        sites_[siteIdx].redirCount = 0;
         // the WorkSSL client can throw if SSL init fails
         makeRequest(sites_[siteIdx].startingResource, siteIdx, lock);
     }
@@ -292,7 +352,7 @@ void
 ValidatorSite::parseJsonResponse (
     std::string const& res,
     std::size_t siteIdx,
-    std::lock_guard<std::mutex>& lock)
+    std::lock_guard<std::mutex>& sites_lock)
 {
     Json::Reader r;
     Json::Value body;
@@ -370,10 +430,15 @@ ValidatorSite::parseJsonResponse (
     if (body.isMember ("refresh_interval") &&
         body["refresh_interval"].isNumeric ())
     {
-        // TODO: should we sanity check/clamp this value
-        // to something reasonable?
-        sites_[siteIdx].refreshInterval =
-            std::chrono::minutes{body["refresh_interval"].asUInt ()};
+        using namespace std::chrono_literals;
+        std::chrono::minutes const refresh =
+            boost::algorithm::clamp(
+                std::chrono::minutes {body["refresh_interval"].asUInt ()},
+                1min,
+                24h);
+        sites_[siteIdx].refreshInterval = refresh;
+        sites_[siteIdx].nextRefresh =
+            clock_type::now() + sites_[siteIdx].refreshInterval;
     }
 }
 
@@ -381,7 +446,7 @@ std::shared_ptr<ValidatorSite::Site::Resource>
 ValidatorSite::processRedirect (
     detail::response_type& res,
     std::size_t siteIdx,
-    std::lock_guard<std::mutex>& lock)
+    std::lock_guard<std::mutex>& sites_lock)
 {
     using namespace boost::beast::http;
     std::shared_ptr<Site::Resource> newLocation;
@@ -395,7 +460,7 @@ ValidatorSite::processRedirect (
         throw std::runtime_error{"missing location"};
     }
 
-    if (sites_[siteIdx].redirCount == MAX_REDIRECTS)
+    if (sites_[siteIdx].redirCount == max_redirects)
     {
         JLOG (j_.warn()) <<
             "Exceeded max redirects for validator list at " <<
@@ -435,6 +500,8 @@ ValidatorSite::onSiteFetch(
 {
     {
         std::lock_guard <std::mutex> lock_sites{sites_mutex_};
+        JLOG (j_.debug()) << "Got completion for "
+            << sites_[siteIdx].activeResource->uri;
         auto onError = [&](std::string const& errMsg, bool retry)
         {
             sites_[siteIdx].lastRefreshStatus.emplace(
@@ -443,7 +510,7 @@ ValidatorSite::onSiteFetch(
                             errMsg});
             if (retry)
                 sites_[siteIdx].nextRefresh =
-                        clock_type::now() + ERROR_RETRY_INTERVAL;
+                        clock_type::now() + error_retry_interval;
         };
         if (ec)
         {
@@ -506,7 +573,7 @@ ValidatorSite::onSiteFetch(
     std::lock_guard <std::mutex> lock_state{state_mutex_};
     fetching_ = false;
     if (! stopping_)
-        setTimer ();
+        setTimer (lock_state);
     cv_.notify_all();
 }
 
@@ -547,7 +614,7 @@ ValidatorSite::onTextFetch(
     std::lock_guard <std::mutex> lock_state{state_mutex_};
     fetching_ = false;
     if (! stopping_)
-        setTimer ();
+        setTimer (lock_state);
     cv_.notify_all();
 }
 

src/ripple/ledger/impl/CachedView.cpp

@@ -31,38 +31,38 @@ CachedViewImpl::exists (Keylet const& k) const
 }
 
 std::shared_ptr<SLE const>
-CachedViewImpl::read (Keylet const& k) const
+CachedViewImpl::read(Keylet const& k) const
 {
     {
-        std::lock_guard<
-            std::mutex> lock(mutex_);
+        std::lock_guard<std::mutex> lock(mutex_);
         auto const iter = map_.find(k.key);
         if (iter != map_.end())
         {
-            if (! k.check(*iter->second))
+            if (!iter->second || !k.check(*iter->second))
                 return nullptr;
             return iter->second;
         }
     }
-    auto const digest =
-        base_.digest(k.key);
-    if (! digest)
+    auto const digest = base_.digest(k.key);
+    if (!digest)
         return nullptr;
-    auto sle = cache_.fetch(*digest,
-        [&]() { return base_.read(k); });
-    std::lock_guard<
-        std::mutex> lock(mutex_);
-    auto const iter =
-        map_.find(k.key);
-    if (iter == map_.end())
+    auto sle = cache_.fetch(*digest, [&]() { return base_.read(k); });
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto const er = map_.emplace(k.key, sle);
+    auto const& iter = er.first;
+    bool const inserted = er.second;
+    if (iter->second && !k.check(*iter->second))
     {
-        map_.emplace(k.key, sle);
-        return sle;
+        if (!inserted)
+        {
+            // On entry, this function did not find this key in map_. Now something
+            // (another thread?) has inserted the sle into the map and it has
+            // the wrong type.
+            LogicError("CachedView::read: wrong type");
+        }
+        return nullptr;
     }
-    if (! k.check(*iter->second))
-        LogicError("CachedView::read: wrong type");
     return iter->second;
-
 }
 
 } // detail

src/ripple/nodestore/impl/DatabaseShardImp.cpp

@@ -524,12 +524,12 @@ DatabaseShardImp::setStored(std::shared_ptr<Ledger const> const& ledger)
         updateStats(lock);
 
         // Update peers with new shard index
-        protocol::TMShardInfo message;
+        protocol::TMPeerShardInfo message;
         PublicKey const& publicKey {app_.nodeIdentity().first};
         message.set_nodepubkey(publicKey.data(), publicKey.size());
         message.set_shardindexes(std::to_string(shardIndex));
         app_.overlay().foreach(send_always(
-            std::make_shared<Message>(message, protocol::mtSHARD_INFO)));
+            std::make_shared<Message>(message, protocol::mtPEER_SHARD_INFO)));
     }
 }
 

src/ripple/overlay/Overlay.h

@@ -137,6 +137,11 @@ public:
     std::shared_ptr<Peer>
     findPeerByShortID (Peer::id_t const& id) = 0;
 
+    /** Returns the peer with the matching public key, or null. */
+    virtual
+    std::shared_ptr<Peer>
+    findPeerByPublicKey (PublicKey const& pubKey) = 0;
+
     /** Broadcast a proposal. */
     virtual
     void

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -769,10 +769,10 @@ OverlayImpl::crawlShards(bool pubKey, std::uint32_t hops)
                 }
 
                 // Relay request to active peers
-                protocol::TMGetShardInfo tmGS;
-                tmGS.set_hops(hops);
+                protocol::TMGetPeerShardInfo tmGPS;
+                tmGPS.set_hops(hops);
                 foreach(send_always(std::make_shared<Message>(
-                    tmGS, protocol::mtGET_SHARD_INFO)));
+                    tmGPS, protocol::mtGET_PEER_SHARD_INFO)));
 
                 if (csCV_.wait_for(l, timeout) == std::cv_status::timeout)
                 {
@@ -1082,6 +1082,23 @@ OverlayImpl::findPeerByShortID (Peer::id_t const& id)
     return {};
 }
 
+// A public key hash map was not used due to the peer connect/disconnect
+// update overhead outweighing the performance of a small set linear search.
+std::shared_ptr<Peer>
+OverlayImpl::findPeerByPublicKey (PublicKey const& pubKey)
+{
+    std::lock_guard <decltype(mutex_)> lock(mutex_);
+    for (auto const& e : ids_)
+    {
+        if (auto peer = e.second.lock())
+        {
+            if (peer->getNodePublic() == pubKey)
+                return peer;
+        }
+    }
+    return {};
+}
+
 void
 OverlayImpl::send (protocol::TMProposeSet& m)
 {

src/ripple/overlay/impl/OverlayImpl.h

@@ -194,6 +194,9 @@ public:
     std::shared_ptr<Peer>
     findPeerByShortID (Peer::id_t const& id) override;
 
+    std::shared_ptr<Peer>
+    findPeerByPublicKey (PublicKey const& pubKey) override;
+
     void
     send (protocol::TMProposeSet& m) override;
 

src/ripple/overlay/impl/PeerImp.cpp

@@ -134,9 +134,9 @@ PeerImp::run()
     }
 
     // Request shard info from peer
-    protocol::TMGetShardInfo tmGS;
-    tmGS.set_hops(0);
-    send(std::make_shared<Message>(tmGS, protocol::mtGET_SHARD_INFO));
+    protocol::TMGetPeerShardInfo tmGPS;
+    tmGPS.set_hops(0);
+    send(std::make_shared<Message>(tmGPS, protocol::mtGET_PEER_SHARD_INFO));
 
     setTimer();
 }
@@ -1000,17 +1000,29 @@ PeerImp::onMessage (std::shared_ptr <protocol::TMCluster> const& m)
 }
 
 void
-PeerImp::onMessage (std::shared_ptr <protocol::TMGetShardInfo> const& m)
+PeerImp::onMessage(std::shared_ptr <protocol::TMGetShardInfo> const& m)
 {
-    if (m->hops() > csHopLimit || m->peerchain_size() > csHopLimit)
-    {
-        fee_ = Resource::feeInvalidRequest;
-        JLOG(p_journal_.warn()) <<
-            (m->hops() > csHopLimit ?
-            "Hops (" + std::to_string(m->hops()) + ") exceed limit" :
-            "Invalid Peerchain");
-        return;
-    }
+    // DEPRECATED
+}
+
+void
+PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
+{
+    // DEPRECATED
+}
+
+void
+PeerImp::onMessage (std::shared_ptr <protocol::TMGetPeerShardInfo> const& m)
+{
+    auto badData = [&](std::string msg) {
+        fee_ = Resource::feeBadData;
+        JLOG(p_journal_.warn()) << msg;
+    };
+
+    if (m->hops() > csHopLimit)
+        return badData("Invalid hops: " + std::to_string(m->hops()));
+    if (m->peerchain_size() > csHopLimit)
+        return badData("Invalid peer chain");
 
     // Reply with shard info we may have
     if (auto shardStore = app_.getShardStore())
@@ -1019,7 +1031,7 @@ PeerImp::onMessage (std::shared_ptr <protocol::TMGetShardInfo> const& m)
         auto shards {shardStore->getCompleteShards()};
         if (!shards.empty())
         {
-            protocol::TMShardInfo reply;
+            protocol::TMPeerShardInfo reply;
             reply.set_shardindexes(shards);
 
             if (m->has_lastlink())
@@ -1028,7 +1040,8 @@ PeerImp::onMessage (std::shared_ptr <protocol::TMGetShardInfo> const& m)
             if (m->peerchain_size() > 0)
                 *reply.mutable_peerchain() = m->peerchain();
 
-            send(std::make_shared<Message>(reply, protocol::mtSHARD_INFO));
+            send(std::make_shared<Message>(
+                reply, protocol::mtPEER_SHARD_INFO));
 
             JLOG(p_journal_.trace()) <<
                 "Sent shard indexes " << shards;
@@ -1044,31 +1057,41 @@ PeerImp::onMessage (std::shared_ptr <protocol::TMGetShardInfo> const& m)
         if (m->hops() == 0)
             m->set_lastlink(true);
 
-        m->add_peerchain(id());
+        m->add_peerchain()->set_nodepubkey(
+            publicKey_.data(), publicKey_.size());
+
         overlay_.foreach(send_if_not(
-            std::make_shared<Message>(*m, protocol::mtGET_SHARD_INFO),
+            std::make_shared<Message>(*m, protocol::mtGET_PEER_SHARD_INFO),
             match_peer(this)));
     }
 }
 
 void
-PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
+PeerImp::onMessage(std::shared_ptr <protocol::TMPeerShardInfo> const& m)
 {
-    if (m->shardindexes().empty() || m->peerchain_size() > csHopLimit)
-    {
+    auto badData = [&](std::string msg) {
         fee_ = Resource::feeBadData;
-        JLOG(p_journal_.warn()) <<
-            (m->shardindexes().empty() ?
-            "Missing shard indexes" :
-            "Invalid Peerchain");
-        return;
-    }
+        JLOG(p_journal_.warn()) << msg;
+    };
+
+    if (m->shardindexes().empty())
+        return badData("Missing shard indexes");
+    if (m->peerchain_size() > csHopLimit)
+        return badData("Invalid peer chain");
+    if (m->has_nodepubkey() && !publicKeyType(makeSlice(m->nodepubkey())))
+        return badData("Invalid public key");
 
     // Check if the message should be forwarded to another peer
     if (m->peerchain_size() > 0)
     {
-        auto const peerId {m->peerchain(m->peerchain_size() - 1)};
-        if (auto peer = overlay_.findPeerByShortID(peerId))
+        // Get the Public key of the last link in the peer chain
+        auto const s {makeSlice(m->peerchain(
+            m->peerchain_size() - 1).nodepubkey())};
+        if (!publicKeyType(s))
+            return badData("Invalid pubKey");
+        PublicKey peerPubKey(s);
+
+        if (auto peer = overlay_.findPeerByPublicKey(peerPubKey))
         {
             if (!m->has_nodepubkey())
                 m->set_nodepubkey(publicKey_.data(), publicKey_.size());
@@ -1083,10 +1106,11 @@ PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
             }
 
             m->mutable_peerchain()->RemoveLast();
-            peer->send(std::make_shared<Message>(*m, protocol::mtSHARD_INFO));
+            peer->send(std::make_shared<Message>(
+                *m, protocol::mtPEER_SHARD_INFO));
 
             JLOG(p_journal_.trace()) <<
-                "Relayed TMShardInfo to peer with IP " <<
+                "Relayed TMPeerShardInfo to peer with IP " <<
                 remote_address_.address().to_string();
         }
         else
@@ -1172,19 +1196,11 @@ PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
                 break;
             }
             default:
-                fee_ = Resource::feeBadData;
-                return;
+                return badData("Invalid shard indexes");
             }
         }
     }
 
-    // Get the Public key of the node reporting the shard info
-    PublicKey publicKey;
-    if (m->has_nodepubkey())
-        publicKey = PublicKey(makeSlice(m->nodepubkey()));
-    else
-        publicKey = publicKey_;
-
     // Get the IP of the node reporting the shard info
     beast::IP::Endpoint endpoint;
     if (m->has_endpoint())
@@ -1194,18 +1210,21 @@ PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
             auto result {
                 beast::IP::Endpoint::from_string_checked(m->endpoint())};
             if (!result.second)
-            {
-                fee_ = Resource::feeBadData;
-                JLOG(p_journal_.warn()) <<
-                    "failed to parse incoming endpoint: {" <<
-                    m->endpoint() << "}";
-                return;
-            }
+                return badData("Invalid incoming endpoint: " + m->endpoint());
             endpoint = std::move(result.first);
         }
     }
     else if (crawl()) // Check if peer will share IP publicly
+    {
         endpoint = remote_address_;
+    }
+
+    // Get the Public key of the node reporting the shard info
+    PublicKey publicKey;
+    if (m->has_nodepubkey())
+        publicKey = PublicKey(makeSlice(m->nodepubkey()));
+    else
+        publicKey = publicKey_;
 
     {
         std::lock_guard<std::mutex> l {shardInfoMutex_};
@@ -1229,7 +1248,7 @@ PeerImp::onMessage(std::shared_ptr <protocol::TMShardInfo> const& m)
     }
 
     JLOG(p_journal_.trace()) <<
-        "Consumed TMShardInfo originating from public key " <<
+        "Consumed TMPeerShardInfo originating from public key " <<
         toBase58(TokenType::NodePublic, publicKey) <<
         " shard indexes " << m->shardindexes();
 

src/ripple/overlay/impl/PeerImp.h

@@ -423,6 +423,8 @@ public:
     void onMessage (std::shared_ptr <protocol::TMCluster> const& m);
     void onMessage (std::shared_ptr <protocol::TMGetShardInfo> const& m);
     void onMessage (std::shared_ptr <protocol::TMShardInfo> const& m);
+    void onMessage (std::shared_ptr <protocol::TMGetPeerShardInfo> const& m);
+    void onMessage (std::shared_ptr <protocol::TMPeerShardInfo> const& m);
     void onMessage (std::shared_ptr <protocol::TMGetPeers> const& m);
     void onMessage (std::shared_ptr <protocol::TMPeers> const& m);
     void onMessage (std::shared_ptr <protocol::TMEndpoints> const& m);

src/ripple/overlay/impl/ProtocolMessage.h

@@ -41,24 +41,26 @@ protocolMessageName (int type)
 {
     switch (type)
     {
-    case protocol::mtHELLO:             return "hello";
-    case protocol::mtMANIFESTS:         return "manifests";
-    case protocol::mtPING:              return "ping";
-    case protocol::mtPROOFOFWORK:       return "proof_of_work";
-    case protocol::mtCLUSTER:           return "cluster";
-    case protocol::mtGET_SHARD_INFO:    return "get_shard_info";
-    case protocol::mtSHARD_INFO:        return "shard_info";
-    case protocol::mtGET_PEERS:         return "get_peers";
-    case protocol::mtPEERS:             return "peers";
-    case protocol::mtENDPOINTS:         return "endpoints";
-    case protocol::mtTRANSACTION:       return "tx";
-    case protocol::mtGET_LEDGER:        return "get_ledger";
-    case protocol::mtLEDGER_DATA:       return "ledger_data";
-    case protocol::mtPROPOSE_LEDGER:    return "propose";
-    case protocol::mtSTATUS_CHANGE:     return "status";
-    case protocol::mtHAVE_SET:          return "have_set";
-    case protocol::mtVALIDATION:        return "validation";
-    case protocol::mtGET_OBJECTS:       return "get_objects";
+    case protocol::mtHELLO:                 return "hello";
+    case protocol::mtMANIFESTS:             return "manifests";
+    case protocol::mtPING:                  return "ping";
+    case protocol::mtPROOFOFWORK:           return "proof_of_work";
+    case protocol::mtCLUSTER:               return "cluster";
+    case protocol::mtGET_SHARD_INFO:        return "get_shard_info";
+    case protocol::mtSHARD_INFO:            return "shard_info";
+    case protocol::mtGET_PEER_SHARD_INFO:   return "get_peer_shard_info";
+    case protocol::mtPEER_SHARD_INFO:       return "peer_shard_info";
+    case protocol::mtGET_PEERS:             return "get_peers";
+    case protocol::mtPEERS:                 return "peers";
+    case protocol::mtENDPOINTS:             return "endpoints";
+    case protocol::mtTRANSACTION:           return "tx";
+    case protocol::mtGET_LEDGER:            return "get_ledger";
+    case protocol::mtLEDGER_DATA:           return "ledger_data";
+    case protocol::mtPROPOSE_LEDGER:        return "propose";
+    case protocol::mtSTATUS_CHANGE:         return "status";
+    case protocol::mtHAVE_SET:              return "have_set";
+    case protocol::mtVALIDATION:            return "validation";
+    case protocol::mtGET_OBJECTS:           return "get_objects";
     default:
         break;
     };
@@ -128,23 +130,25 @@ invokeProtocolMessage (Buffers const& buffers, Handler& handler)
 
     switch (type)
     {
-    case protocol::mtHELLO:         ec = detail::invoke<protocol::TMHello> (type, buffers, handler); break;
-    case protocol::mtMANIFESTS:     ec = detail::invoke<protocol::TMManifests> (type, buffers, handler); break;
-    case protocol::mtPING:          ec = detail::invoke<protocol::TMPing> (type, buffers, handler); break;
-    case protocol::mtCLUSTER:       ec = detail::invoke<protocol::TMCluster> (type, buffers, handler); break;
-    case protocol::mtGET_SHARD_INFO:ec = detail::invoke<protocol::TMGetShardInfo> (type, buffers, handler); break;
-    case protocol::mtSHARD_INFO:    ec = detail::invoke<protocol::TMShardInfo>(type, buffers, handler); break;
-    case protocol::mtGET_PEERS:     ec = detail::invoke<protocol::TMGetPeers> (type, buffers, handler); break;
-    case protocol::mtPEERS:         ec = detail::invoke<protocol::TMPeers> (type, buffers, handler); break;
-    case protocol::mtENDPOINTS:     ec = detail::invoke<protocol::TMEndpoints> (type, buffers, handler); break;
-    case protocol::mtTRANSACTION:   ec = detail::invoke<protocol::TMTransaction> (type, buffers, handler); break;
-    case protocol::mtGET_LEDGER:    ec = detail::invoke<protocol::TMGetLedger> (type, buffers, handler); break;
-    case protocol::mtLEDGER_DATA:   ec = detail::invoke<protocol::TMLedgerData> (type, buffers, handler); break;
-    case protocol::mtPROPOSE_LEDGER:ec = detail::invoke<protocol::TMProposeSet> (type, buffers, handler); break;
-    case protocol::mtSTATUS_CHANGE: ec = detail::invoke<protocol::TMStatusChange> (type, buffers, handler); break;
-    case protocol::mtHAVE_SET:      ec = detail::invoke<protocol::TMHaveTransactionSet> (type, buffers, handler); break;
-    case protocol::mtVALIDATION:    ec = detail::invoke<protocol::TMValidation> (type, buffers, handler); break;
-    case protocol::mtGET_OBJECTS:   ec = detail::invoke<protocol::TMGetObjectByHash> (type, buffers, handler); break;
+    case protocol::mtHELLO:                 ec = detail::invoke<protocol::TMHello> (type, buffers, handler); break;
+    case protocol::mtMANIFESTS:             ec = detail::invoke<protocol::TMManifests> (type, buffers, handler); break;
+    case protocol::mtPING:                  ec = detail::invoke<protocol::TMPing> (type, buffers, handler); break;
+    case protocol::mtCLUSTER:               ec = detail::invoke<protocol::TMCluster> (type, buffers, handler); break;
+    case protocol::mtGET_SHARD_INFO:        ec = detail::invoke<protocol::TMGetShardInfo> (type, buffers, handler); break;
+    case protocol::mtSHARD_INFO:            ec = detail::invoke<protocol::TMShardInfo>(type, buffers, handler); break;
+    case protocol::mtGET_PEER_SHARD_INFO:   ec = detail::invoke<protocol::TMGetPeerShardInfo> (type, buffers, handler); break;
+    case protocol::mtPEER_SHARD_INFO:       ec = detail::invoke<protocol::TMPeerShardInfo>(type, buffers, handler); break;
+    case protocol::mtGET_PEERS:             ec = detail::invoke<protocol::TMGetPeers> (type, buffers, handler); break;
+    case protocol::mtPEERS:                 ec = detail::invoke<protocol::TMPeers> (type, buffers, handler); break;
+    case protocol::mtENDPOINTS:             ec = detail::invoke<protocol::TMEndpoints> (type, buffers, handler); break;
+    case protocol::mtTRANSACTION:           ec = detail::invoke<protocol::TMTransaction> (type, buffers, handler); break;
+    case protocol::mtGET_LEDGER:            ec = detail::invoke<protocol::TMGetLedger> (type, buffers, handler); break;
+    case protocol::mtLEDGER_DATA:           ec = detail::invoke<protocol::TMLedgerData> (type, buffers, handler); break;
+    case protocol::mtPROPOSE_LEDGER:        ec = detail::invoke<protocol::TMProposeSet> (type, buffers, handler); break;
+    case protocol::mtSTATUS_CHANGE:         ec = detail::invoke<protocol::TMStatusChange> (type, buffers, handler); break;
+    case protocol::mtHAVE_SET:              ec = detail::invoke<protocol::TMHaveTransactionSet> (type, buffers, handler); break;
+    case protocol::mtVALIDATION:            ec = detail::invoke<protocol::TMValidation> (type, buffers, handler); break;
+    case protocol::mtGET_OBJECTS:           ec = detail::invoke<protocol::TMGetObjectByHash> (type, buffers, handler); break;
     default:
         ec = handler.onMessageUnknown (type);
         break;

src/ripple/overlay/impl/TrafficCount.cpp

@@ -66,6 +66,8 @@ TrafficCount::category TrafficCount::categorize (
             (type == protocol::mtENDPOINTS) ||
             (type == protocol::mtGET_SHARD_INFO) ||
             (type == protocol::mtSHARD_INFO) ||
+            (type == protocol::mtGET_PEER_SHARD_INFO) ||
+            (type == protocol::mtPEER_SHARD_INFO) ||
             (type == protocol::mtPEERS) ||
             (type == protocol::mtGET_PEERS))
         return TrafficCount::category::CT_overlay;

src/ripple/proto/ripple.proto

@@ -21,6 +21,8 @@ enum MessageType
     mtGET_OBJECTS           = 42;
     mtGET_SHARD_INFO        = 50;
     mtSHARD_INFO            = 51;
+    mtGET_PEER_SHARD_INFO   = 52;
+    mtPEER_SHARD_INFO       = 53;
 
     // <available>          = 10;
     // <available>          = 11;
@@ -132,19 +134,43 @@ message TMCluster
 // Request info on shards held
 message TMGetShardInfo
 {
-    required uint32 hops            = 1; // number of hops to travel
-    optional bool lastLink          = 2; // true if last link in the peer chain
-    repeated uint32 peerchain       = 3;
+    required uint32 hops            = 1 [deprecated=true]; // number of hops to travel
+    optional bool lastLink          = 2 [deprecated=true]; // true if last link in the peer chain
+    repeated uint32 peerchain       = 3 [deprecated=true]; // IDs used to route messages
 }
 
 // Info about shards held
 message TMShardInfo
+{
+    required string shardIndexes    = 1 [deprecated=true]; // rangeSet of shard indexes
+    optional bytes nodePubKey       = 2 [deprecated=true]; // The node's public key
+    optional string endpoint        = 3 [deprecated=true]; // ipv6 or ipv4 address
+    optional bool lastLink          = 4 [deprecated=true]; // true if last link in the peer chain
+    repeated uint32 peerchain       = 5 [deprecated=true]; // IDs used to route messages
+}
+
+// Node public key
+message TMLink
+{
+    required bytes nodePubKey       = 1; // node public key
+}
+
+// Request info on shards held
+message TMGetPeerShardInfo
+{
+    required uint32 hops            = 1; // number of hops to travel
+    optional bool lastLink          = 2; // true if last link in the peer chain
+    repeated TMLink peerChain       = 3; // public keys used to route messages
+}
+
+// Info about shards held
+message TMPeerShardInfo
 {
     required string shardIndexes    = 1; // rangeSet of shard indexes
-    optional bytes nodePubKey       = 2; // The node's public key
+    optional bytes nodePubKey       = 2; // node public key
     optional string endpoint        = 3; // ipv6 or ipv4 address
     optional bool lastLink          = 4; // true if last link in the peer chain
-    repeated uint32 peerchain       = 5; // List of IDs used to route messages
+    repeated TMLink peerChain       = 5; // public keys used to route messages
 }
 
 // A transaction can have only one input and one output.

src/test/app/ValidatorSite_test.cpp

@@ -30,8 +30,10 @@
 #include <test/jtx.h>
 #include <test/jtx/TrustedPublisherServer.h>
 #include <test/unit_test/FileDirGuard.h>
+#include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/asio.hpp>
+#include <boost/range/adaptor/transformed.hpp>
 #include <chrono>
 
 namespace ripple {
@@ -48,6 +50,8 @@ constexpr const char* realValidatorContents()
 }
 )vl";
 }
+
+auto constexpr default_expires = std::chrono::seconds{3600};
 }
 
 class ValidatorSite_test : public beast::unit_test::suite
@@ -185,13 +189,24 @@ private:
         }
     };
 
-    void
-    testFetchList (
-        std::vector<std::pair<std::string, std::string>> const& paths)
+    struct FetchListConfig
     {
-        testcase << "Fetch list - " << paths[0].first <<
-            (paths.size() > 1 ? ", " + paths[1].first : "");
-
+        std::string path;
+        std::string msg;
+        bool failFetch = false;
+        bool failApply = false;
+        int serverVersion = 1;
+        std::chrono::seconds expiresFromNow = detail::default_expires;
+        int expectedRefreshMin = 0;
+    };
+    void
+    testFetchList (std::vector<FetchListConfig> const& paths)
+    {
+        testcase << "Fetch list - " <<
+            boost::algorithm::join (paths |
+                boost::adaptors::transformed(
+                    [](FetchListConfig const& cfg){ return cfg.path; }),
+                ", ");
         using namespace jtx;
 
         Env env (*this);
@@ -204,20 +219,16 @@ private:
         std::vector<std::string> emptyCfgKeys;
         struct publisher
         {
+            publisher(FetchListConfig const& c) : cfg{c} {}
             std::unique_ptr<TrustedPublisherServer> server;
             std::vector<Validator> list;
             std::string uri;
-            std::string expectMsg;
-            bool shouldFail;
+            FetchListConfig const& cfg;
             bool isRetry;
         };
         std::vector<publisher> servers;
 
         auto const sequence = 1;
-        auto const version = 1;
-        using namespace std::chrono_literals;
-        NetClock::time_point const expiration =
-            env.timeKeeper().now() + 3600s;
         auto constexpr listSize = 20;
         std::vector<std::string> cfgPublishers;
 
@@ -233,11 +244,9 @@ private:
                 publisherPublic, publisherSecret,
                 pubSigningKeys.first, pubSigningKeys.second, 1);
 
-            servers.push_back({});
+            servers.push_back(cfg);
             auto& item = servers.back();
-            item.shouldFail = ! cfg.second.empty();
-            item.isRetry = cfg.first == "/bad-resource";
-            item.expectMsg = cfg.second;
+            item.isRetry = cfg.path == "/bad-resource";
             item.list.reserve (listSize);
             while (item.list.size () < listSize)
                 item.list.push_back (randomValidator());
@@ -247,20 +256,24 @@ private:
                 pubSigningKeys,
                 manifest,
                 sequence,
-                expiration,
-                version,
+                env.timeKeeper().now() + cfg.expiresFromNow,
+                cfg.serverVersion,
                 item.list);
 
             std::stringstream uri;
-            uri << "http://" << item.server->local_endpoint() << cfg.first;
+            uri << "http://" << item.server->local_endpoint() << cfg.path;
             item.uri = uri.str();
         }
 
         BEAST_EXPECT(trustedKeys.load (
             emptyLocalKey, emptyCfgKeys, cfgPublishers));
 
+        using namespace std::chrono_literals;
         auto sites = std::make_unique<ValidatorSite> (
-            env.app().getIOService(), env.app().validators(), journal);
+            env.app().getIOService(),
+            env.app().validators(),
+            journal,
+            2s);
 
         std::vector<std::string> uris;
         for (auto const& u : servers)
@@ -269,30 +282,45 @@ private:
         sites->start();
         sites->join();
 
+        auto const jv = sites->getJson();
         for (auto const& u : servers)
         {
             for (auto const& val : u.list)
             {
                 BEAST_EXPECT(
-                    trustedKeys.listed (val.masterPublic) != u.shouldFail);
+                    trustedKeys.listed (val.masterPublic) != u.cfg.failApply);
                 BEAST_EXPECT(
-                    trustedKeys.listed (val.signingPublic) != u.shouldFail);
+                    trustedKeys.listed (val.signingPublic) != u.cfg.failApply);
             }
 
-            auto const jv = sites->getJson();
             Json::Value myStatus;
             for (auto const& vs : jv[jss::validator_sites])
                 if (vs[jss::uri].asString().find(u.uri) != std::string::npos)
                     myStatus = vs;
             BEAST_EXPECTS(
                 myStatus[jss::last_refresh_message].asString().empty()
-                    != u.shouldFail, to_string(myStatus));
-            if (u.shouldFail)
+                    != u.cfg.failFetch,
+                to_string(myStatus) + "\n" + sink.strm_.str());
+
+            if (! u.cfg.msg.empty())
+            {
+                BEAST_EXPECTS(
+                    sink.strm_.str().find(u.cfg.msg) != std::string::npos,
+                    sink.strm_.str());
+            }
+
+
+            if (u.cfg.expectedRefreshMin)
+            {
+                BEAST_EXPECTS(
+                    myStatus[jss::refresh_interval_min].asInt()
+                        == u.cfg.expectedRefreshMin,
+                    to_string(myStatus));
+            }
+
+            if (u.cfg.failFetch)
             {
                 using namespace std::chrono;
-                BEAST_EXPECTS(
-                    sink.strm_.str().find(u.expectMsg) != std::string::npos,
-                    sink.strm_.str());
                 log << " -- Msg: " <<
                     myStatus[jss::last_refresh_message].asString() << std::endl;
                 std::stringstream nextRefreshStr
@@ -412,40 +440,99 @@ public:
         testConfigLoad ();
 
         // fetch single site
-        testFetchList ({{"/validators",""}});
+        testFetchList ({{"/validators", ""}});
         // fetch multiple sites
-        testFetchList ({{"/validators",""}, {"/validators",""}});
+        testFetchList ({{"/validators", ""}, {"/validators", ""}});
         // fetch single site with single redirects
-        testFetchList ({{"/redirect_once/301",""}});
-        testFetchList ({{"/redirect_once/302",""}});
-        testFetchList ({{"/redirect_once/307",""}});
-        testFetchList ({{"/redirect_once/308",""}});
+        testFetchList ({{"/redirect_once/301", ""}});
+        testFetchList ({{"/redirect_once/302", ""}});
+        testFetchList ({{"/redirect_once/307", ""}});
+        testFetchList ({{"/redirect_once/308", ""}});
         // one redirect, one not
-        testFetchList ({{"/validators",""}, {"/redirect_once/302",""}});
+        testFetchList ({{"/validators", ""}, {"/redirect_once/302", ""}});
         // fetch single site with undending redirect (fails to load)
-        testFetchList ({{"/redirect_forever/301", "Exceeded max redirects"}});
+        testFetchList ({
+            {"/redirect_forever/301", "Exceeded max redirects", true, true}});
         // two that redirect forever
         testFetchList ({
-            {"/redirect_forever/307","Exceeded max redirects"},
-            {"/redirect_forever/308","Exceeded max redirects"}});
+            {"/redirect_forever/307", "Exceeded max redirects", true, true},
+            {"/redirect_forever/308", "Exceeded max redirects", true, true}});
         // one undending redirect, one not
         testFetchList (
-            {{"/validators",""},
-            {"/redirect_forever/302","Exceeded max redirects"}});
+            {{"/validators", ""},
+            {"/redirect_forever/302", "Exceeded max redirects", true, true}});
         // invalid redir Location
         testFetchList ({
             {"/redirect_to/ftp://invalid-url/302",
-                "Invalid redirect location"}});
+             "Invalid redirect location",
+             true,
+             true}});
+        testFetchList ({
+            {"/redirect_to/file://invalid-url/302",
+             "Invalid redirect location",
+             true,
+             true}});
         // invalid json
-        testFetchList ({{"/validators/bad", "Unable to parse JSON response"}});
+        testFetchList ({
+            {"/validators/bad", "Unable to parse JSON response", true, true}});
         // error status returned
-        testFetchList ({{"/bad-resource", "returned bad status"}});
+        testFetchList ({
+            {"/bad-resource", "returned bad status", true, true}});
         // location field missing
         testFetchList ({
-            {"/redirect_nolo/308", "returned a redirect with no Location"}});
+            {"/redirect_nolo/308",
+            "returned a redirect with no Location",
+            true,
+            true}});
         // json fields missing
         testFetchList ({
-            {"/validators/missing", "Missing fields in JSON response"}});
+            {"/validators/missing",
+            "Missing fields in JSON response",
+            true,
+            true}});
+        // timeout
+        testFetchList ({
+            {"/sleep/3", "took too long", true, true}});
+        // bad manifest version
+        testFetchList ({
+            {"/validators", "Unsupported version", false, true, 4}});
+        using namespace std::chrono_literals;
+        // get old validator list
+        testFetchList ({
+            {"/validators", "Stale validator list", false, true, 1, 0s}});
+        // force an out-of-range expiration value
+        testFetchList ({
+            {"/validators",
+            "Invalid validator list",
+            false,
+            true,
+            1,
+            std::chrono::seconds{Json::Value::maxInt + 1}}});
+        // verify refresh intervals are properly clamped
+        testFetchList ({
+            {"/validators/refresh/0",
+            "",
+            false,
+            false,
+            1,
+            detail::default_expires,
+            1}}); // minimum of 1 minute
+        testFetchList ({
+            {"/validators/refresh/10",
+            "",
+            false,
+            false,
+            1,
+            detail::default_expires,
+            10}}); // 10 minutes is fine
+        testFetchList ({
+            {"/validators/refresh/2000",
+            "",
+            false,
+            false,
+            1,
+            detail::default_expires,
+            60*24}}); // max of 24 hours
         testFileURLs();
     }
 };

src/test/jtx/TrustedPublisherServer.h

@@ -28,6 +28,8 @@
 #include <boost/asio.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/beast/http.hpp>
+#include <boost/lexical_cast.hpp>
+#include <thread>
 
 namespace ripple {
 namespace test {
@@ -44,8 +46,7 @@ class TrustedPublisherServer
 
     socket_type sock_;
     boost::asio::ip::tcp::acceptor acceptor_;
-
-    std::string list_;
+    std::function<std::string(int)> getList_;
 
 public:
 
@@ -82,14 +83,16 @@ public:
         data.pop_back();
         data += "]}";
         std::string blob = base64_encode(data);
-
-        list_ = "{\"blob\":\"" + blob + "\"";
-
         auto const sig = sign(keys.first, keys.second, makeSlice(data));
-
-        list_ += ",\"signature\":\"" + strHex(sig) + "\"";
-        list_ += ",\"manifest\":\"" + manifest + "\"";
-        list_ += ",\"version\":" + std::to_string(version) + '}';
+        getList_ = [blob, sig, manifest, version](int interval) {
+            std::stringstream l;
+            l << "{\"blob\":\"" << blob << "\"" <<
+                ",\"signature\":\"" << strHex(sig) << "\"" <<
+                ",\"manifest\":\"" << manifest << "\"" <<
+                ",\"refresh_interval\": " << interval <<
+                ",\"version\":" << version  << '}';
+            return l.str();
+        };
 
         acceptor_.open(ep.protocol());
         error_code ec;
@@ -163,61 +166,75 @@ private:
         error_code ec;
         for (;;)
         {
-            req_type req;
-            http::read(sock, sb, req, ec);
-            if (ec)
-                break;
-            auto path = req.target().to_string();
             resp_type res;
-            res.insert("Server", "TrustedPublisherServer");
-            res.version(req.version());
-
-            if (boost::starts_with(path, "/validators"))
-            {
-                res.result(http::status::ok);
-                res.insert("Content-Type", "application/json");
-                if (path == "/validators/bad")
-                    res.body() = "{ 'bad': \"1']" ;
-                else if (path == "/validators/missing")
-                    res.body() = "{\"version\": 1}";
-                else
-                    res.body() = list_;
-            }
-            else if (boost::starts_with(path, "/redirect"))
-            {
-                if (boost::ends_with(path, "/301"))
-                    res.result(http::status::moved_permanently);
-                else if (boost::ends_with(path, "/302"))
-                    res.result(http::status::found);
-                else if (boost::ends_with(path, "/307"))
-                    res.result(http::status::temporary_redirect);
-                else if (boost::ends_with(path, "/308"))
-                    res.result(http::status::permanent_redirect);
-
-                std::stringstream location;
-                if (boost::starts_with(path, "/redirect_to/"))
-                {
-                    location << path.substr(13);
-                }
-                else if (! boost::starts_with(path, "/redirect_nolo"))
-                {
-                    location << "http://" << local_endpoint() <<
-                        (boost::starts_with(path, "/redirect_forever/") ?
-                            path : "/validators");
-                }
-                if (! location.str().empty())
-                    res.insert("Location", location.str());
-            }
-            else
-            {
-                // unknown request
-                res.result(boost::beast::http::status::not_found);
-                res.insert("Content-Type", "text/html");
-                res.body() = "The file '" + path + "' was not found";
-            }
-
+            req_type req;
             try
             {
+                http::read(sock, sb, req, ec);
+                if (ec)
+                    break;
+                auto path = req.target().to_string();
+                res.insert("Server", "TrustedPublisherServer");
+                res.version(req.version());
+
+                if (boost::starts_with(path, "/validators"))
+                {
+                    res.result(http::status::ok);
+                    res.insert("Content-Type", "application/json");
+                    if (path == "/validators/bad")
+                        res.body() = "{ 'bad': \"1']" ;
+                    else if (path == "/validators/missing")
+                        res.body() = "{\"version\": 1}";
+                    else
+                    {
+                        int refresh = 5;
+                        if (boost::starts_with(path, "/validators/refresh"))
+                            refresh =
+                                boost::lexical_cast<unsigned int>(
+                                    path.substr(20));
+                        res.body() = getList_(refresh);
+                    }
+                }
+                else if (boost::starts_with(path, "/sleep/"))
+                {
+                    auto const sleep_sec =
+                        boost::lexical_cast<unsigned int>(path.substr(7));
+                    std::this_thread::sleep_for(
+                        std::chrono::seconds{sleep_sec});
+                }
+                else if (boost::starts_with(path, "/redirect"))
+                {
+                    if (boost::ends_with(path, "/301"))
+                        res.result(http::status::moved_permanently);
+                    else if (boost::ends_with(path, "/302"))
+                        res.result(http::status::found);
+                    else if (boost::ends_with(path, "/307"))
+                        res.result(http::status::temporary_redirect);
+                    else if (boost::ends_with(path, "/308"))
+                        res.result(http::status::permanent_redirect);
+
+                    std::stringstream location;
+                    if (boost::starts_with(path, "/redirect_to/"))
+                    {
+                        location << path.substr(13);
+                    }
+                    else if (! boost::starts_with(path, "/redirect_nolo"))
+                    {
+                        location << "http://" << local_endpoint() <<
+                            (boost::starts_with(path, "/redirect_forever/") ?
+                                path : "/validators");
+                    }
+                    if (! location.str().empty())
+                        res.insert("Location", location.str());
+                }
+                else
+                {
+                    // unknown request
+                    res.result(boost::beast::http::status::not_found);
+                    res.insert("Content-Type", "text/html");
+                    res.body() = "The file '" + path + "' was not found";
+                }
+
                 res.prepare_payload();
             }
             catch (std::exception const& e)