Load validator list from file:

* Adds local file:// URL support to the [validator_list_sites] stanza.
  The file:// URL must not contain a hostname. Allows a rippled node
  operator to "sideload" a new list if their node is unable to reach
  a validator list's web site before an old list expires. Lists
  loaded from a file will be validated in the same way a downloaded
  list is validated.
* Generalize file/dir "guards" from Config test so they can be reused
  in other tests.
* Check for error when reading validators.txt. Saves some parsing and
  checking of an empty string, and will give a more meaningful error.
* Completes RIPD-1674.

src/ripple/app/misc/ValidatorSite.h

@@ -200,6 +200,13 @@ private:
         detail::response_type&& res,
         std::size_t siteIdx);
 
+    /// Store latest list fetched from anywhere
+    void
+    onTextFetch(
+        boost::system::error_code const& ec,
+        std::string const& res,
+        std::size_t siteIdx);
+
     /// Initiate request to given resource.
     /// lock over sites_mutex_ required
     void
@@ -212,7 +219,7 @@ private:
     /// lock over sites_mutex_ required
     void
     parseJsonResponse (
-        detail::response_type& res,
+        std::string const& res,
         std::size_t siteIdx,
         std::lock_guard<std::mutex>& lock);
 

src/ripple/app/misc/detail/WorkFile.h

@@ -0,0 +1,107 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2018 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
+#define RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
+
+#include <ripple/app/misc/detail/Work.h>
+#include <ripple/basics/ByteUtilities.h>
+#include <ripple/basics/FileUtilities.h>
+#include <cassert>
+#include <cerrno>
+
+namespace ripple {
+
+namespace detail {
+
+// Work with files
+class WorkFile: public Work
+    , public std::enable_shared_from_this<WorkFile>
+{
+protected:
+    using error_code = boost::system::error_code;
+    // Override the definition in Work.h
+    using response_type = std::string;
+
+public:
+    using callback_type =
+        std::function<void(error_code const&, response_type const&)>;
+public:
+    WorkFile(
+        std::string const& path,
+        boost::asio::io_service& ios, callback_type cb);
+    ~WorkFile();
+
+    void run() override;
+
+    void cancel() override;
+
+private:
+    std::string path_;
+    callback_type cb_;
+    boost::asio::io_service& ios_;
+    boost::asio::io_service::strand strand_;
+
+};
+
+//------------------------------------------------------------------------------
+
+WorkFile::WorkFile(
+    std::string const& path,
+    boost::asio::io_service& ios, callback_type cb)
+    : path_(path)
+    , cb_(std::move(cb))
+    , ios_(ios)
+    , strand_(ios)
+{
+}
+
+WorkFile::~WorkFile()
+{
+    if (cb_)
+        cb_ (make_error_code(boost::system::errc::interrupted), {});
+}
+
+void
+WorkFile::run()
+{
+    if (! strand_.running_in_this_thread())
+        return ios_.post(strand_.wrap (std::bind(
+            &WorkFile::run, shared_from_this())));
+
+    error_code ec;
+    auto const fileContents = getFileContents(ec, path_, megabytes(1));
+
+    assert(cb_);
+    cb_(ec, fileContents);
+    cb_ = nullptr;
+}
+
+void
+WorkFile::cancel()
+{
+    // Nothing to do. Either it finished in run, or it didn't start.
+}
+
+
+} // detail
+
+} // ripple
+
+#endif

src/ripple/app/misc/impl/ValidatorSite.cpp

@@ -19,6 +19,7 @@
 
 #include <ripple/app/misc/ValidatorList.h>
 #include <ripple/app/misc/ValidatorSite.h>
+#include <ripple/app/misc/detail/WorkFile.h>
 #include <ripple/app/misc/detail/WorkPlain.h>
 #include <ripple/app/misc/detail/WorkSSL.h>
 #include <ripple/basics/base64.h>
@@ -38,14 +39,43 @@ unsigned short constexpr MAX_REDIRECTS = 3;
 ValidatorSite::Site::Resource::Resource (std::string uri_)
     : uri {std::move(uri_)}
 {
-    if (! parseUrl (pUrl, uri) ||
-        (pUrl.scheme != "http" && pUrl.scheme != "https"))
-    {
-        throw std::runtime_error {"invalid url"};
-    }
+    if (! parseUrl (pUrl, uri))
+        throw std::runtime_error("URI '" + uri + "' cannot be parsed");
 
-    if (! pUrl.port)
-        pUrl.port = (pUrl.scheme == "https") ? 443 : 80;
+    if  (pUrl.scheme == "file")
+    {
+        if (!pUrl.domain.empty())
+            throw std::runtime_error("file URI cannot contain a hostname");
+
+#if _MSC_VER    // MSVC: Windows paths need the leading / removed
+        {
+            if (pUrl.path[0] == '/')
+                pUrl.path = pUrl.path.substr(1);
+
+        }
+#endif
+
+        if (pUrl.path.empty())
+            throw std::runtime_error("file URI must contain a path");
+    }
+    else if (pUrl.scheme == "http")
+    {
+        if (pUrl.domain.empty())
+            throw std::runtime_error("http URI must contain a hostname");
+
+        if (!pUrl.port)
+            pUrl.port = 80;
+    }
+    else if (pUrl.scheme == "https")
+    {
+        if (pUrl.domain.empty())
+            throw std::runtime_error("https URI must contain a hostname");
+
+        if (!pUrl.port)
+            pUrl.port = 443;
+    }
+    else
+        throw std::runtime_error ("Unsupported scheme: '" + pUrl.scheme + "'");
 }
 
 ValidatorSite::Site::Site (std::string uri)
@@ -103,10 +133,11 @@ ValidatorSite::load (
         {
             sites_.emplace_back (uri);
         }
-        catch (std::exception &)
+        catch (std::exception const& e)
         {
             JLOG (j_.error()) <<
-                "Invalid validator site uri: " << uri;
+                "Invalid validator site uri: " << uri <<
+                ": " << e.what();
             return false;
         }
     }
@@ -185,6 +216,12 @@ ValidatorSite::makeRequest (
             onSiteFetch (err, std::move(resp), siteIdx);
         };
 
+    auto onFetchFile =
+        [this, siteIdx] (error_code const& err, std::string const& resp)
+    {
+        onTextFetch (err, resp, siteIdx);
+    };
+
     if (resource->pUrl.scheme == "https")
     {
         sp = std::make_shared<detail::WorkSSL>(
@@ -195,7 +232,7 @@ ValidatorSite::makeRequest (
             j_,
             onFetch);
     }
-    else
+    else if(resource->pUrl.scheme == "http")
     {
         sp = std::make_shared<detail::WorkPlain>(
             resource->pUrl.domain,
@@ -204,6 +241,14 @@ ValidatorSite::makeRequest (
             ios_,
             onFetch);
     }
+    else
+    {
+        BOOST_ASSERT(resource->pUrl.scheme == "file");
+        sp = std::make_shared<detail::WorkFile>(
+            resource->pUrl.path,
+            ios_,
+            onFetchFile);
+    }
 
     work_ = sp;
     sp->run ();
@@ -245,13 +290,13 @@ ValidatorSite::onTimer (
 
 void
 ValidatorSite::parseJsonResponse (
-    detail::response_type& res,
+    std::string const& res,
     std::size_t siteIdx,
     std::lock_guard<std::mutex>& lock)
 {
     Json::Reader r;
     Json::Value body;
-    if (! r.parse(res.body().data(), body))
+    if (! r.parse(res.data(), body))
     {
         JLOG (j_.warn()) <<
             "Unable to parse JSON response from  " <<
@@ -367,6 +412,10 @@ ValidatorSite::processRedirect (
         newLocation = std::make_shared<Site::Resource>(
             std::string(res[field::location]));
         ++sites_[siteIdx].redirCount;
+        if (newLocation->pUrl.scheme != "http" &&
+            newLocation->pUrl.scheme != "https")
+            throw std::runtime_error("invalid scheme in redirect " +
+                newLocation->pUrl.scheme);
     }
     catch (std::exception &)
     {
@@ -414,7 +463,7 @@ ValidatorSite::onSiteFetch(
                 switch (res.result())
                 {
                 case status::ok:
-                    parseJsonResponse(res, siteIdx, lock_sites);
+                    parseJsonResponse(res.body(), siteIdx, lock_sites);
                     break;
                 case status::moved_permanently :
                 case status::permanent_redirect :
@@ -460,6 +509,47 @@ ValidatorSite::onSiteFetch(
     cv_.notify_all();
 }
 
+void
+ValidatorSite::onTextFetch(
+    boost::system::error_code const& ec,
+    std::string const& res,
+    std::size_t siteIdx)
+{
+    {
+        std::lock_guard <std::mutex> lock_sites{sites_mutex_};
+        try
+        {
+            if (ec)
+            {
+                JLOG (j_.warn()) <<
+                    "Problem retrieving from " <<
+                    sites_[siteIdx].activeResource->uri <<
+                    " " <<
+                    ec.value() <<
+                    ":" <<
+                    ec.message();
+                throw std::runtime_error{"fetch error"};
+            }
+
+            parseJsonResponse(res, siteIdx, lock_sites);
+        }
+        catch (std::exception& ex)
+        {
+            sites_[siteIdx].lastRefreshStatus.emplace(
+                Site::Status{clock_type::now(),
+                ListDisposition::invalid,
+                ex.what()});
+        }
+        sites_[siteIdx].activeResource.reset();
+    }
+
+    std::lock_guard <std::mutex> lock_state{state_mutex_};
+    fetching_ = false;
+    if (! stopping_)
+        setTimer ();
+    cv_.notify_all();
+}
+
 Json::Value
 ValidatorSite::getJson() const
 {