Load validator list from file:

* Adds local file:// URL support to the [validator_list_sites] stanza.
  The file:// URL must not contain a hostname. Allows a rippled node
  operator to "sideload" a new list if their node is unable to reach
  a validator list's web site before an old list expires. Lists
  loaded from a file will be validated in the same way a downloaded
  list is validated.
* Generalize file/dir "guards" from Config test so they can be reused
  in other tests.
* Check for error when reading validators.txt. Saves some parsing and
  checking of an empty string, and will give a more meaningful error.
* Completes RIPD-1674.

CMakeLists.txt

@@ -1376,6 +1376,7 @@ else ()
     src/ripple/basics/impl/base64.cpp
     src/ripple/basics/impl/contract.cpp
     src/ripple/basics/impl/CountedObject.cpp
+    src/ripple/basics/impl/FileUtilities.cpp
     src/ripple/basics/impl/Log.cpp
     src/ripple/basics/impl/strHex.cpp
     src/ripple/basics/impl/StringUtilities.cpp
@@ -1474,6 +1475,7 @@ install (
     src/ripple/basics/Blob.h
     src/ripple/basics/Buffer.h
     src/ripple/basics/CountedObject.h
+    src/ripple/basics/FileUtilities.h
     src/ripple/basics/LocalValue.h
     src/ripple/basics/Log.h
     src/ripple/basics/Slice.h
@@ -2074,6 +2076,7 @@ else ()
     #]===============================]
     src/test/basics/Buffer_test.cpp
     src/test/basics/DetectCrash_test.cpp
+    src/test/basics/FileUtilities_test.cpp
     src/test/basics/KeyCache_test.cpp
     src/test/basics/PerfLog_test.cpp
     src/test/basics/RangeSet_test.cpp

cfg/validators-example.txt

@@ -14,10 +14,8 @@
 #
 #   List of the validation public keys of nodes to always accept as validators.
 #
-#   The latest list of recommended validators can be obtained from
+#   Manually listing validator keys is not recommended for production networks.
-#   https://ripple.com/ripple.txt
+#   See validator_list_sites and validator_list_keys below.
-#
-#   See also https://wiki.ripple.com/Ripple.txt
 #
 #   Examples:
 #    n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5
@@ -27,9 +25,13 @@
 #
 #   List of URIs serving lists of recommended validators.
 #
+#   The latest list of recommended validator sites can be
+#   obtained from https://ripple.com/ripple.txt
+#
 #   Examples:
 #    https://vl.ripple.com
 #    http://127.0.0.1:8000
+#    file:///etc/opt/ripple/vl.txt
 #
 # [validator_list_keys]
 #
@@ -39,6 +41,9 @@
 #   publisher key.
 #   Validator list keys should be hex-encoded.
 #
+#   The latest list of recommended validator keys can be
+#   obtained from https://ripple.com/ripple.txt
+#
 #   Examples:
 #    ed499d732bded01504a7407c224412ef550cc1ade638a4de4eb88af7c36cb8b282
 #    0202d3f36a801349f3be534e3f64cfa77dede6e1b6310a0b48f40f20f955cec945

src/ripple/app/misc/ValidatorSite.h

@@ -200,6 +200,13 @@ private:
         detail::response_type&& res,
         std::size_t siteIdx);
 
+    /// Store latest list fetched from anywhere
+    void
+    onTextFetch(
+        boost::system::error_code const& ec,
+        std::string const& res,
+        std::size_t siteIdx);
+
     /// Initiate request to given resource.
     /// lock over sites_mutex_ required
     void
@@ -212,7 +219,7 @@ private:
     /// lock over sites_mutex_ required
     void
     parseJsonResponse (
-        detail::response_type& res,
+        std::string const& res,
         std::size_t siteIdx,
         std::lock_guard<std::mutex>& lock);
 

src/ripple/app/misc/detail/WorkFile.h

@@ -0,0 +1,107 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2018 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
+#define RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
+
+#include <ripple/app/misc/detail/Work.h>
+#include <ripple/basics/ByteUtilities.h>
+#include <ripple/basics/FileUtilities.h>
+#include <cassert>
+#include <cerrno>
+
+namespace ripple {
+
+namespace detail {
+
+// Work with files
+class WorkFile: public Work
+    , public std::enable_shared_from_this<WorkFile>
+{
+protected:
+    using error_code = boost::system::error_code;
+    // Override the definition in Work.h
+    using response_type = std::string;
+
+public:
+    using callback_type =
+        std::function<void(error_code const&, response_type const&)>;
+public:
+    WorkFile(
+        std::string const& path,
+        boost::asio::io_service& ios, callback_type cb);
+    ~WorkFile();
+
+    void run() override;
+
+    void cancel() override;
+
+private:
+    std::string path_;
+    callback_type cb_;
+    boost::asio::io_service& ios_;
+    boost::asio::io_service::strand strand_;
+
+};
+
+//------------------------------------------------------------------------------
+
+WorkFile::WorkFile(
+    std::string const& path,
+    boost::asio::io_service& ios, callback_type cb)
+    : path_(path)
+    , cb_(std::move(cb))
+    , ios_(ios)
+    , strand_(ios)
+{
+}
+
+WorkFile::~WorkFile()
+{
+    if (cb_)
+        cb_ (make_error_code(boost::system::errc::interrupted), {});
+}
+
+void
+WorkFile::run()
+{
+    if (! strand_.running_in_this_thread())
+        return ios_.post(strand_.wrap (std::bind(
+            &WorkFile::run, shared_from_this())));
+
+    error_code ec;
+    auto const fileContents = getFileContents(ec, path_, megabytes(1));
+
+    assert(cb_);
+    cb_(ec, fileContents);
+    cb_ = nullptr;
+}
+
+void
+WorkFile::cancel()
+{
+    // Nothing to do. Either it finished in run, or it didn't start.
+}
+
+
+} // detail
+
+} // ripple
+
+#endif

src/ripple/app/misc/impl/ValidatorSite.cpp

@@ -19,6 +19,7 @@
 
 #include <ripple/app/misc/ValidatorList.h>
 #include <ripple/app/misc/ValidatorSite.h>
+#include <ripple/app/misc/detail/WorkFile.h>
 #include <ripple/app/misc/detail/WorkPlain.h>
 #include <ripple/app/misc/detail/WorkSSL.h>
 #include <ripple/basics/base64.h>
@@ -38,14 +39,43 @@ unsigned short constexpr MAX_REDIRECTS = 3;
 ValidatorSite::Site::Resource::Resource (std::string uri_)
     : uri {std::move(uri_)}
 {
-    if (! parseUrl (pUrl, uri) ||
+    if (! parseUrl (pUrl, uri))
-        (pUrl.scheme != "http" && pUrl.scheme != "https"))
+        throw std::runtime_error("URI '" + uri + "' cannot be parsed");
+
+    if  (pUrl.scheme == "file")
     {
-        throw std::runtime_error {"invalid url"};
+        if (!pUrl.domain.empty())
+            throw std::runtime_error("file URI cannot contain a hostname");
+
+#if _MSC_VER    // MSVC: Windows paths need the leading / removed
+        {
+            if (pUrl.path[0] == '/')
+                pUrl.path = pUrl.path.substr(1);
+
         }
+#endif
+
+        if (pUrl.path.empty())
+            throw std::runtime_error("file URI must contain a path");
+    }
+    else if (pUrl.scheme == "http")
+    {
+        if (pUrl.domain.empty())
+            throw std::runtime_error("http URI must contain a hostname");
 
         if (!pUrl.port)
-        pUrl.port = (pUrl.scheme == "https") ? 443 : 80;
+            pUrl.port = 80;
+    }
+    else if (pUrl.scheme == "https")
+    {
+        if (pUrl.domain.empty())
+            throw std::runtime_error("https URI must contain a hostname");
+
+        if (!pUrl.port)
+            pUrl.port = 443;
+    }
+    else
+        throw std::runtime_error ("Unsupported scheme: '" + pUrl.scheme + "'");
 }
 
 ValidatorSite::Site::Site (std::string uri)
@@ -103,10 +133,11 @@ ValidatorSite::load (
         {
             sites_.emplace_back (uri);
         }
-        catch (std::exception &)
+        catch (std::exception const& e)
         {
             JLOG (j_.error()) <<
-                "Invalid validator site uri: " << uri;
+                "Invalid validator site uri: " << uri <<
+                ": " << e.what();
             return false;
         }
     }
@@ -185,6 +216,12 @@ ValidatorSite::makeRequest (
             onSiteFetch (err, std::move(resp), siteIdx);
         };
 
+    auto onFetchFile =
+        [this, siteIdx] (error_code const& err, std::string const& resp)
+    {
+        onTextFetch (err, resp, siteIdx);
+    };
+
     if (resource->pUrl.scheme == "https")
     {
         sp = std::make_shared<detail::WorkSSL>(
@@ -195,7 +232,7 @@ ValidatorSite::makeRequest (
             j_,
             onFetch);
     }
-    else
+    else if(resource->pUrl.scheme == "http")
     {
         sp = std::make_shared<detail::WorkPlain>(
             resource->pUrl.domain,
@@ -204,6 +241,14 @@ ValidatorSite::makeRequest (
             ios_,
             onFetch);
     }
+    else
+    {
+        BOOST_ASSERT(resource->pUrl.scheme == "file");
+        sp = std::make_shared<detail::WorkFile>(
+            resource->pUrl.path,
+            ios_,
+            onFetchFile);
+    }
 
     work_ = sp;
     sp->run ();
@@ -245,13 +290,13 @@ ValidatorSite::onTimer (
 
 void
 ValidatorSite::parseJsonResponse (
-    detail::response_type& res,
+    std::string const& res,
     std::size_t siteIdx,
     std::lock_guard<std::mutex>& lock)
 {
     Json::Reader r;
     Json::Value body;
-    if (! r.parse(res.body().data(), body))
+    if (! r.parse(res.data(), body))
     {
         JLOG (j_.warn()) <<
             "Unable to parse JSON response from  " <<
@@ -367,6 +412,10 @@ ValidatorSite::processRedirect (
         newLocation = std::make_shared<Site::Resource>(
             std::string(res[field::location]));
         ++sites_[siteIdx].redirCount;
+        if (newLocation->pUrl.scheme != "http" &&
+            newLocation->pUrl.scheme != "https")
+            throw std::runtime_error("invalid scheme in redirect " +
+                newLocation->pUrl.scheme);
     }
     catch (std::exception &)
     {
@@ -414,7 +463,7 @@ ValidatorSite::onSiteFetch(
                 switch (res.result())
                 {
                 case status::ok:
-                    parseJsonResponse(res, siteIdx, lock_sites);
+                    parseJsonResponse(res.body(), siteIdx, lock_sites);
                     break;
                 case status::moved_permanently :
                 case status::permanent_redirect :
@@ -460,6 +509,47 @@ ValidatorSite::onSiteFetch(
     cv_.notify_all();
 }
 
+void
+ValidatorSite::onTextFetch(
+    boost::system::error_code const& ec,
+    std::string const& res,
+    std::size_t siteIdx)
+{
+    {
+        std::lock_guard <std::mutex> lock_sites{sites_mutex_};
+        try
+        {
+            if (ec)
+            {
+                JLOG (j_.warn()) <<
+                    "Problem retrieving from " <<
+                    sites_[siteIdx].activeResource->uri <<
+                    " " <<
+                    ec.value() <<
+                    ":" <<
+                    ec.message();
+                throw std::runtime_error{"fetch error"};
+            }
+
+            parseJsonResponse(res, siteIdx, lock_sites);
+        }
+        catch (std::exception& ex)
+        {
+            sites_[siteIdx].lastRefreshStatus.emplace(
+                Site::Status{clock_type::now(),
+                ListDisposition::invalid,
+                ex.what()});
+        }
+        sites_[siteIdx].activeResource.reset();
+    }
+
+    std::lock_guard <std::mutex> lock_state{state_mutex_};
+    fetching_ = false;
+    if (! stopping_)
+        setTimer ();
+    cv_.notify_all();
+}
+
 Json::Value
 ValidatorSite::getJson() const
 {

src/ripple/basics/ByteUtilities.h

@@ -31,8 +31,11 @@ namespace ripple {
     template<class T>
     constexpr auto megabytes(T value) noexcept
     {
-        return kilobytes(1) * kilobytes(1) * value;
+        return kilobytes(kilobytes(value));
     }
+
+    static_assert(kilobytes(2) == 2048, "kilobytes(2) == 2048");
+    static_assert(megabytes(3) == 3145728, "megabytes(3) == 3145728");
 }
 
 #endif

src/ripple/basics/FileUtilities.h

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2018 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_BASICS_FILEUTILITIES_H_INCLUDED
+#define RIPPLE_BASICS_FILEUTILITIES_H_INCLUDED
+
+#include <boost/system/error_code.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/optional.hpp>
+
+namespace ripple
+{
+
+// TODO: Should this one function have its own file, or can it
+// be absorbed somewhere else?
+
+std::string getFileContents(boost::system::error_code& ec,
+    boost::filesystem::path const& sourcePath,
+    boost::optional<std::size_t> maxSize = boost::none);
+
+}
+
+#endif

src/ripple/basics/impl/FileUtilities.cpp

@@ -0,0 +1,63 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2018 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/basics/FileUtilities.h>
+
+namespace ripple
+{
+
+std::string getFileContents(boost::system::error_code& ec,
+    boost::filesystem::path const& sourcePath,
+    boost::optional<std::size_t> maxSize)
+{
+    using namespace boost::filesystem;
+    using namespace boost::system::errc;
+
+    path fullPath{ canonical(sourcePath, ec) };
+    if (ec)
+        return {};
+
+    if (maxSize && (file_size(fullPath, ec) > *maxSize || ec))
+    {
+        if (!ec)
+            ec = make_error_code(file_too_large);
+        return {};
+    }
+
+    ifstream fileStream(fullPath, std::ios::in);
+
+    if (!fileStream)
+    {
+        ec = make_error_code(static_cast<errc_t>(errno));
+        return {};
+    }
+
+    const std::string result{ std::istreambuf_iterator<char>{fileStream},
+        std::istreambuf_iterator<char>{} };
+
+    if (fileStream.bad ())
+    {
+        ec = make_error_code(static_cast<errc_t>(errno));
+        return {};
+    }
+
+    return result;
+}
+
+}

src/ripple/basics/impl/StringUtilities.cpp

@@ -89,11 +89,15 @@ uint64_t uintFromHex (std::string const& strSrc)
     return uValue;
 }
 
-// TODO Callers should be using beast::URL and beast::parse_URL instead.
 bool parseUrl (parsedURL& pUrl, std::string const& strUrl)
 {
     // scheme://username:password@hostname:port/rest
-    static boost::regex reUrl ("(?i)\\`\\s*([[:alpha:]][-+.[:alpha:][:digit:]]*)://([^/]+)(/.*)?\\s*?\\'");
+    static boost::regex reUrl (
+        "(?i)\\`\\s*"
+        "([[:alpha:]][-+.[:alpha:][:digit:]]*):"    //scheme
+        "//([^/]+)?"                                 // hostname
+        "(/.*)?"                                    // path and parameters
+        "\\s*?\\'");
     boost::smatch smMatch;
 
     bool bMatch = boost::regex_match (strUrl, smMatch, reUrl); // Match status code.

src/ripple/core/impl/Config.cpp

@@ -20,6 +20,7 @@
 #include <ripple/core/Config.h>
 #include <ripple/core/ConfigSections.h>
 #include <ripple/basics/contract.h>
+#include <ripple/basics/FileUtilities.h>
 #include <ripple/basics/Log.h>
 #include <ripple/json/json_reader.h>
 #include <ripple/protocol/Feature.h>
@@ -30,6 +31,7 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/format.hpp>
 #include <boost/regex.hpp>
+#include <boost/system/error_code.hpp>
 #include <fstream>
 #include <iostream>
 #include <iterator>
@@ -268,21 +270,13 @@ void Config::load ()
     if (!QUIET)
         std::cerr << "Loading: " << CONFIG_FILE << "\n";
 
-    std::ifstream ifsConfig (CONFIG_FILE.c_str (), std::ios::in);
+    boost::system::error_code ec;
+    auto const fileContents = getFileContents(ec, CONFIG_FILE);
 
-    if (!ifsConfig)
+    if (ec)
     {
-        std::cerr << "Failed to open '" << CONFIG_FILE << "'." << std::endl;
+        std::cerr << "Failed to read '" << CONFIG_FILE << "'." <<
-        return;
+            ec.value() << ": " << ec.message() << std::endl;
-    }
-
-    std::string fileContents;
-    fileContents.assign ((std::istreambuf_iterator<char>(ifsConfig)),
-                          std::istreambuf_iterator<char>());
-
-    if (ifsConfig.bad ())
-    {
-        std::cerr << "Failed to read '" << CONFIG_FILE << "'." << std::endl;
         return;
     }
 
@@ -472,13 +466,14 @@ void Config::loadFromString (std::string const& fileContents)
                 (boost::filesystem::is_regular_file (validatorsFile) ||
                 boost::filesystem::is_symlink (validatorsFile)))
         {
-            std::ifstream ifsDefault (validatorsFile.native().c_str());
+            boost::system::error_code ec;
-
+            auto const data = getFileContents(ec, validatorsFile);
-            std::string data;
+            if (ec)
-
+            {
-            data.assign (
+                Throw<std::runtime_error>("Failed to read '" +
-                std::istreambuf_iterator<char>(ifsDefault),
+                    validatorsFile.string() + "'." +
-                std::istreambuf_iterator<char>());
+                    std::to_string(ec.value()) + ": " + ec.message());
+            }
 
             auto iniFile = parseIniFile (data, true);
 

src/ripple/unity/basics1.cpp

@@ -21,6 +21,7 @@
 #include <ripple/basics/impl/base64.cpp>
 #include <ripple/basics/impl/contract.cpp>
 #include <ripple/basics/impl/CountedObject.cpp>
+#include <ripple/basics/impl/FileUtilities.cpp>
 #include <ripple/basics/impl/Log.cpp>
 #include <ripple/basics/impl/strHex.cpp>
 #include <ripple/basics/impl/StringUtilities.cpp>

src/test/basics/FileUtilities_test.cpp

@@ -0,0 +1,76 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2018 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/basics/ByteUtilities.h>
+#include <ripple/basics/FileUtilities.h>
+#include <ripple/beast/unit_test.h>
+#include <test/unit_test/FileDirGuard.h>
+
+namespace ripple {
+
+class FileUtilities_test : public beast::unit_test::suite
+{
+public:
+    void testGetFileContents()
+    {
+        using namespace ripple::test::detail;
+        using namespace boost::system;
+
+        constexpr const char* expectedContents =
+            "This file is very short. That's all we need.";
+
+        FileDirGuard file(*this, "test_file", "test.txt",
+            expectedContents);
+
+        error_code ec;
+        auto const path = file.file();
+
+        {
+            // Test with no max
+            auto const good = getFileContents(ec, path);
+            BEAST_EXPECT(!ec);
+            BEAST_EXPECT(good == expectedContents);
+        }
+
+        {
+            // Test with large max
+            auto const good = getFileContents(ec, path, kilobytes(1));
+            BEAST_EXPECT(!ec);
+            BEAST_EXPECT(good == expectedContents);
+        }
+
+        {
+            // Test with small max
+            auto const bad = getFileContents(ec, path, 16);
+            BEAST_EXPECT(ec && ec.value() ==
+                boost::system::errc::file_too_large);
+            BEAST_EXPECT(bad.empty());
+        }
+
+    }
+
+    void run () override
+    {
+        testGetFileContents();
+    }
+};
+
+BEAST_DEFINE_TESTSUITE(FileUtilities, ripple_basics, ripple);
+
+} // ripple

src/test/basics/StringUtilities_test.cpp

@@ -80,6 +80,10 @@ public:
         BEAST_EXPECT(parseUrl (pUrl, "scheme://[::1]:123/path"));
         BEAST_EXPECT(*pUrl.port == 123);
         BEAST_EXPECT(pUrl.domain == "::1");
+        BEAST_EXPECT(parseUrl(pUrl, "nodomain:///path/path/path"));
+        BEAST_EXPECT(pUrl.scheme == "nodomain");
+        BEAST_EXPECT(pUrl.domain.empty());
+        BEAST_EXPECT(pUrl.path == "/path/path/path");
     }
 
     void testToString ()

src/test/core/Config_test.cpp

@@ -22,6 +22,7 @@
 #include <ripple/core/ConfigSections.h>
 #include <ripple/server/Port.h>
 #include <test/jtx/TestSuite.h>
+#include <test/unit_test/FileDirGuard.h>
 #include <boost/filesystem.hpp>
 #include <boost/format.hpp>
 #include <fstream>
@@ -121,88 +122,12 @@ backend=sqlite
         configContentsTemplate % dbPathSection % valFileSection);
 }
 
-/**
-   Write a config file and remove when done.
- */
-class ConfigGuard
-{
-private:
-    bool rmSubDir_{false};
-
-protected:
-    using path = boost::filesystem::path;
-    path subDir_;
-    beast::unit_test::suite& test_;
-
-    auto rmDir (path const& toRm)
-    {
-        if (is_directory (toRm) && is_empty (toRm))
-            remove (toRm);
-        else
-            test_.log << "Expected " << toRm.string ()
-                      << " to be an empty existing directory." << std::endl;
-    }
-
-public:
-    ConfigGuard (beast::unit_test::suite& test, path subDir,
-            bool useCounter = true)
-        : subDir_ (std::move (subDir))
-        , test_ (test)
-    {
-        using namespace boost::filesystem;
-        {
-            static auto subDirCounter = 0;
-            if (useCounter)
-                subDir_ += std::to_string(++subDirCounter);
-            if (!exists (subDir_))
-            {
-                create_directory (subDir_);
-                rmSubDir_ = true;
-            }
-            else if (is_directory (subDir_))
-                rmSubDir_ = false;
-            else
-            {
-                // Cannot run the test. Someone created a file where we want to
-                // put our directory
-                Throw<std::runtime_error> (
-                    "Cannot create directory: " + subDir_.string ());
-            }
-        }
-    }
-
-    ~ConfigGuard ()
-    {
-        try
-        {
-            using namespace boost::filesystem;
-
-            if (rmSubDir_)
-                rmDir (subDir_);
-            else
-                test_.log << "Skipping rm dir: "
-                          << subDir_.string () << std::endl;
-        }
-        catch (std::exception& e)
-        {
-            // if we throw here, just let it die.
-            test_.log << "Error in ~ConfigGuard: " << e.what () << std::endl;
-        };
-    }
-
-    path const& subdir() const
-    {
-        return subDir_;
-    }
-};
-
 /**
    Write a rippled config file and remove when done.
  */
-class RippledCfgGuard : public ConfigGuard
+class RippledCfgGuard : public ripple::test::detail::FileDirGuard
 {
 private:
-    path configFile_;
     path dataDir_;
 
     bool rmDataDir_{false};
@@ -214,28 +139,17 @@ public:
         path subDir, path const& dbPath,
             path const& validatorsFile,
                 bool useCounter = true)
-        : ConfigGuard (test, std::move (subDir), useCounter)
+        : FileDirGuard(test, std::move (subDir),
+            path (Config::configFileName),
+            configContents (dbPath.string (), validatorsFile.string ()),
+            useCounter)
         , dataDir_ (dbPath)
     {
         if (dbPath.empty ())
-            dataDir_ = subDir_ / path (Config::databaseDirName);
+            dataDir_ = subdir() / path (Config::databaseDirName);
-
-        configFile_ = subDir_ / path (Config::configFileName);
-
-        if (!exists (configFile_))
-        {
-            std::ofstream o (configFile_.string ());
-            o << configContents (dbPath.string (), validatorsFile.string ());
-        }
-        else
-        {
-            Throw<std::runtime_error> (
-                "Refusing to overwrite existing config file: " +
-                    configFile_.string ());
-        }
 
         rmDataDir_ = !exists (dataDir_);
-        config_.setup (configFile_.string (), /*bQuiet*/ true,
+        config_.setup (file_.string (), /*bQuiet*/ true,
             /* bSilent */ false, /* bStandalone */ false);
     }
 
@@ -246,7 +160,7 @@ public:
 
     std::string configFile() const
     {
-        return configFile_.string();
+        return file().string();
     }
 
     bool dataDirExists () const
@@ -256,7 +170,7 @@ public:
 
     bool configFileExists () const
     {
-        return boost::filesystem::exists (configFile_);
+        return fileExists();
     }
 
     ~RippledCfgGuard ()
@@ -264,12 +178,6 @@ public:
         try
         {
             using namespace boost::filesystem;
-            if (!boost::filesystem::exists (configFile_))
-                test_.log << "Expected " << configFile_.string ()
-                          << " to be an existing file." << std::endl;
-            else
-                remove (configFile_);
-
             if (rmDataDir_)
                 rmDir (dataDir_);
             else
@@ -314,65 +222,37 @@ moreripplevalidators.net
 /**
    Write a validators.txt file and remove when done.
  */
-class ValidatorsTxtGuard : public ConfigGuard
+class ValidatorsTxtGuard : public test::detail::FileDirGuard
 {
-private:
-    path validatorsFile_;
-
 public:
     ValidatorsTxtGuard (beast::unit_test::suite& test,
         path subDir, path const& validatorsFileName,
             bool useCounter = true)
-        : ConfigGuard (test, std::move (subDir), useCounter)
+        : FileDirGuard (test, std::move (subDir),
-    {
+            path (
-        using namespace boost::filesystem;
-        validatorsFile_ = current_path () / subDir_ / path (
                 validatorsFileName.empty () ? Config::validatorsFileName :
-            validatorsFileName);
+                validatorsFileName),
-
+            valFileContents (),
-        if (!exists (validatorsFile_))
+            useCounter)
     {
-            std::ofstream o (validatorsFile_.string ());
-            o << valFileContents ();
-        }
-        else
-        {
-            Throw<std::runtime_error> (
-                "Refusing to overwrite existing config file: " +
-                    validatorsFile_.string ());
-        }
     }
 
     bool validatorsFileExists () const
     {
-        return boost::filesystem::exists (validatorsFile_);
+        return fileExists();
     }
 
     std::string validatorsFile () const
     {
-        return validatorsFile_.string ();
+        return absolute(file()).string();
     }
 
     ~ValidatorsTxtGuard ()
     {
-        try
-        {
-            using namespace boost::filesystem;
-            if (!boost::filesystem::exists (validatorsFile_))
-                test_.log << "Expected " << validatorsFile_.string ()
-                          << " to be an existing file." << std::endl;
-            else
-                remove (validatorsFile_);
-        }
-        catch (std::exception& e)
-        {
-            // if we throw here, just let it die.
-            test_.log << "Error in ~ValidatorsTxtGuard: "
-                      << e.what () << std::endl;
-        };
     }
 };
 }  // detail
+
 class Config_test final : public TestSuite
 {
 private:
@@ -440,7 +320,7 @@ port_wss_admin
         {
             // read from file absolute path
             auto const cwd = current_path ();
-            detail::ConfigGuard const g0(*this, "test_db");
+            ripple::test::detail::DirGuard const g0(*this, "test_db");
             path const dataDirRel ("test_data_dir");
             path const dataDirAbs(cwd / g0.subdir () / dataDirRel);
             detail::RippledCfgGuard const g (*this, g0.subdir(),
@@ -767,9 +647,9 @@ trustthesevalidators.gov
     {
         detail::RippledCfgGuard const cfg(*this, "testSetup",
             explicitPath ? "test_db" : "", "");
-        /* ConfigGuard has a Config object that gets loaded on construction,
+        /* RippledCfgGuard has a Config object that gets loaded on
-            but Config::setup is not reentrant, so we need a fresh config
+            construction, but Config::setup is not reentrant, so we
-            for every test case, so ignore it.
+            need a fresh config for every test case, so ignore it.
         */
         {
             Config config;

src/test/unit_test/FileDirGuard.h

@@ -0,0 +1,169 @@
+//------------------------------------------------------------------------------
+/*
+This file is part of rippled: https://github.com/ripple/rippled
+Copyright (c) 2018 Ripple Labs Inc.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose  with  or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef TEST_UNIT_TEST_DIRGUARD_H
+#define TEST_UNIT_TEST_DIRGUARD_H
+
+#include <ripple/basics/contract.h>
+#include <test/jtx/TestSuite.h>
+#include <boost/filesystem.hpp>
+
+namespace ripple {
+namespace test {
+namespace detail {
+
+/**
+    Create a directory and remove it when it's done
+*/
+class DirGuard
+{
+protected:
+    using path = boost::filesystem::path;
+
+private:
+    path subDir_;
+    bool rmSubDir_{false};
+
+protected:
+    beast::unit_test::suite& test_;
+
+    auto rmDir (path const& toRm)
+    {
+        if (is_directory (toRm) && is_empty (toRm))
+            remove (toRm);
+        else
+            test_.log << "Expected " << toRm.string ()
+            << " to be an empty existing directory." << std::endl;
+    }
+
+public:
+    DirGuard (beast::unit_test::suite& test, path subDir,
+        bool useCounter = true)
+        : subDir_ (std::move (subDir))
+        , test_ (test)
+    {
+        using namespace boost::filesystem;
+
+        static auto subDirCounter = 0;
+        if (useCounter)
+            subDir_ += std::to_string(++subDirCounter);
+        if (!exists (subDir_))
+        {
+            create_directory (subDir_);
+            rmSubDir_ = true;
+        }
+        else if (is_directory (subDir_))
+            rmSubDir_ = false;
+        else
+        {
+            // Cannot run the test. Someone created a file where we want to
+            // put our directory
+            Throw<std::runtime_error> (
+                "Cannot create directory: " + subDir_.string ());
+        }
+    }
+
+    ~DirGuard ()
+    {
+        try
+        {
+            using namespace boost::filesystem;
+
+            if (rmSubDir_)
+                rmDir (subDir_);
+            else
+                test_.log << "Skipping rm dir: "
+                << subDir_.string () << std::endl;
+        }
+        catch (std::exception& e)
+        {
+            // if we throw here, just let it die.
+            test_.log << "Error in ~DirGuard: " << e.what () << std::endl;
+        };
+    }
+
+    path const& subdir() const
+    {
+        return subDir_;
+    }
+};
+
+/**
+    Write a file in a directory and remove when done
+*/
+class FileDirGuard : public DirGuard
+{
+protected:
+    path const file_;
+
+public:
+    FileDirGuard(beast::unit_test::suite& test,
+        path subDir, path file, std::string const& contents,
+        bool useCounter = true)
+        : DirGuard(test, subDir, useCounter)
+        , file_(file.is_absolute()  ? file : subdir() / file)
+    {
+        if (!exists (file_))
+        {
+            std::ofstream o (file_.string ());
+            o << contents;
+        }
+        else
+        {
+            Throw<std::runtime_error> (
+                "Refusing to overwrite existing file: " +
+                file_.string ());
+        }
+    }
+
+    ~FileDirGuard ()
+    {
+        try
+        {
+            using namespace boost::filesystem;
+            if (!exists (file_))
+                test_.log << "Expected " << file_.string ()
+                << " to be an existing file." << std::endl;
+            else
+                remove (file_);
+        }
+        catch (std::exception& e)
+        {
+            // if we throw here, just let it die.
+            test_.log << "Error in ~FileGuard: "
+                << e.what () << std::endl;
+        };
+    }
+
+    path const& file() const
+    {
+        return file_;
+    }
+
+    bool fileExists () const
+    {
+        return boost::filesystem::exists (file_);
+    }
+};
+
+}
+}
+}
+
+#endif // TEST_UNIT_TEST_DIRGUARD_H

src/test/unity/basics_test_unity.cpp

@@ -23,6 +23,7 @@
 #include <test/basics/Buffer_test.cpp>
 #include <test/basics/contract_test.cpp>
 #include <test/basics/DetectCrash_test.cpp>
+#include <test/basics/FileUtilities_test.cpp>
 #include <test/basics/hardened_hash_test.cpp>
 #include <test/basics/KeyCache_test.cpp>
 #include <test/basics/mulDiv_test.cpp>