ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Load validator list from file:

Browse Source 
* Adds local file:// URL support to the [validator_list_sites] stanza.
  The file:// URL must not contain a hostname. Allows a rippled node
  operator to "sideload" a new list if their node is unable to reach
  a validator list's web site before an old list expires. Lists
  loaded from a file will be validated in the same way a downloaded
  list is validated.
* Generalize file/dir "guards" from Config test so they can be reused
  in other tests.
* Check for error when reading validators.txt. Saves some parsing and
  checking of an empty string, and will give a more meaningful error.
* Completes RIPD-1674.
This commit is contained in:
Edward Hennis
2018-10-23 18:08:57 -04:00
committed by Nik Bougalis
parent e7a69cce65
commit c1a02440dc
17 changed files with 774 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/ripple/app/misc/ValidatorSite.h
View File

@@ -200,6 +200,13 @@ private:
detail::response_type&& res,
std::size_t siteIdx);
/// Store latest list fetched from anywhere
void
onTextFetch(
boost::system::error_code const& ec,
std::string const& res,
std::size_t siteIdx);
/// Initiate request to given resource.
/// lock over sites_mutex_ required
void
@@ -212,7 +219,7 @@ private:
/// lock over sites_mutex_ required
void
parseJsonResponse (
detail::response_type& res,
std::string const& res,
std::size_t siteIdx,
std::lock_guard<std::mutex>& lock);

107
src/ripple/app/misc/detail/WorkFile.h Normal file
View File

@@ -0,0 +1,107 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2018 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
#define RIPPLE_APP_MISC_DETAIL_WORKFILE_H_INCLUDED
#include <ripple/app/misc/detail/Work.h>
#include <ripple/basics/ByteUtilities.h>
#include <ripple/basics/FileUtilities.h>
#include <cassert>
#include <cerrno>
namespace ripple {
namespace detail {
// Work with files
class WorkFile: public Work
, public std::enable_shared_from_this<WorkFile>
{
protected:
using error_code = boost::system::error_code;
// Override the definition in Work.h
using response_type = std::string;
public:
using callback_type =
std::function<void(error_code const&, response_type const&)>;
public:
WorkFile(
std::string const& path,
boost::asio::io_service& ios, callback_type cb);
~WorkFile();
void run() override;
void cancel() override;
private:
std::string path_;
callback_type cb_;
boost::asio::io_service& ios_;
boost::asio::io_service::strand strand_;
};
//------------------------------------------------------------------------------
WorkFile::WorkFile(
std::string const& path,
boost::asio::io_service& ios, callback_type cb)
: path_(path)
, cb_(std::move(cb))
, ios_(ios)
, strand_(ios)
{
}
WorkFile::~WorkFile()
{
if (cb_)
cb_ (make_error_code(boost::system::errc::interrupted), {});
}
void
WorkFile::run()
{
if (! strand_.running_in_this_thread())
return ios_.post(strand_.wrap (std::bind(
&WorkFile::run, shared_from_this())));
error_code ec;
auto const fileContents = getFileContents(ec, path_, megabytes(1));
assert(cb_);
cb_(ec, fileContents);
cb_ = nullptr;
}
void
WorkFile::cancel()
{
// Nothing to do. Either it finished in run, or it didn't start.
}
} // detail
} // ripple
#endif

116
src/ripple/app/misc/impl/ValidatorSite.cpp
View File

@@ -19,6 +19,7 @@
#include <ripple/app/misc/ValidatorList.h>
#include <ripple/app/misc/ValidatorSite.h>
#include <ripple/app/misc/detail/WorkFile.h>
#include <ripple/app/misc/detail/WorkPlain.h>
#include <ripple/app/misc/detail/WorkSSL.h>
#include <ripple/basics/base64.h>
@@ -38,14 +39,43 @@ unsigned short constexpr MAX_REDIRECTS = 3;
ValidatorSite::Site::Resource::Resource (std::string uri_)
: uri {std::move(uri_)}
{
if (! parseUrl (pUrl, uri) ||
(pUrl.scheme != "http" && pUrl.scheme != "https"))
{
throw std::runtime_error {"invalid url"};
}
if (! parseUrl (pUrl, uri))
throw std::runtime_error("URI '" + uri + "' cannot be parsed");
if (! pUrl.port)
pUrl.port = (pUrl.scheme == "https") ? 443 : 80;
if (pUrl.scheme == "file")
{
if (!pUrl.domain.empty())
throw std::runtime_error("file URI cannot contain a hostname");
#if _MSC_VER // MSVC: Windows paths need the leading / removed
{
if (pUrl.path[0] == '/')
pUrl.path = pUrl.path.substr(1);
}
#endif
if (pUrl.path.empty())
throw std::runtime_error("file URI must contain a path");
}
else if (pUrl.scheme == "http")
{
if (pUrl.domain.empty())
throw std::runtime_error("http URI must contain a hostname");
if (!pUrl.port)
pUrl.port = 80;
}
else if (pUrl.scheme == "https")
{
if (pUrl.domain.empty())
throw std::runtime_error("https URI must contain a hostname");
if (!pUrl.port)
pUrl.port = 443;
}
else
throw std::runtime_error ("Unsupported scheme: '" + pUrl.scheme + "'");
}
ValidatorSite::Site::Site (std::string uri)
@@ -103,10 +133,11 @@ ValidatorSite::load (
{
sites_.emplace_back (uri);
}
catch (std::exception &)
catch (std::exception const& e)
{
JLOG (j_.error()) <<
"Invalid validator site uri: " << uri;
"Invalid validator site uri: " << uri <<
": " << e.what();
return false;
}
}
@@ -185,6 +216,12 @@ ValidatorSite::makeRequest (
onSiteFetch (err, std::move(resp), siteIdx);
};
auto onFetchFile =
[this, siteIdx] (error_code const& err, std::string const& resp)
{
onTextFetch (err, resp, siteIdx);
};
if (resource->pUrl.scheme == "https")
{
sp = std::make_shared<detail::WorkSSL>(
@@ -195,7 +232,7 @@ ValidatorSite::makeRequest (
j_,
onFetch);
}
else
else if(resource->pUrl.scheme == "http")
{
sp = std::make_shared<detail::WorkPlain>(
resource->pUrl.domain,
@@ -204,6 +241,14 @@ ValidatorSite::makeRequest (
ios_,
onFetch);
}
else
{
BOOST_ASSERT(resource->pUrl.scheme == "file");
sp = std::make_shared<detail::WorkFile>(
resource->pUrl.path,
ios_,
onFetchFile);
}
work_ = sp;
sp->run ();
@@ -245,13 +290,13 @@ ValidatorSite::onTimer (
void
ValidatorSite::parseJsonResponse (
detail::response_type& res,
std::string const& res,
std::size_t siteIdx,
std::lock_guard<std::mutex>& lock)
{
Json::Reader r;
Json::Value body;
if (! r.parse(res.body().data(), body))
if (! r.parse(res.data(), body))
{
JLOG (j_.warn()) <<
"Unable to parse JSON response from " <<
@@ -367,6 +412,10 @@ ValidatorSite::processRedirect (
newLocation = std::make_shared<Site::Resource>(
std::string(res[field::location]));
++sites_[siteIdx].redirCount;
if (newLocation->pUrl.scheme != "http" &&
newLocation->pUrl.scheme != "https")
throw std::runtime_error("invalid scheme in redirect " +
newLocation->pUrl.scheme);
}
catch (std::exception &)
{
@@ -414,7 +463,7 @@ ValidatorSite::onSiteFetch(
switch (res.result())
{
case status::ok:
parseJsonResponse(res, siteIdx, lock_sites);
parseJsonResponse(res.body(), siteIdx, lock_sites);
break;
case status::moved_permanently :
case status::permanent_redirect :
@@ -460,6 +509,47 @@ ValidatorSite::onSiteFetch(
cv_.notify_all();
}
void
ValidatorSite::onTextFetch(
boost::system::error_code const& ec,
std::string const& res,
std::size_t siteIdx)
{
{
std::lock_guard <std::mutex> lock_sites{sites_mutex_};
try
{
if (ec)
{
JLOG (j_.warn()) <<
"Problem retrieving from " <<
sites_[siteIdx].activeResource->uri <<
" " <<
ec.value() <<
":" <<
ec.message();
throw std::runtime_error{"fetch error"};
}
parseJsonResponse(res, siteIdx, lock_sites);
}
catch (std::exception& ex)
{
sites_[siteIdx].lastRefreshStatus.emplace(
Site::Status{clock_type::now(),
ListDisposition::invalid,
ex.what()});
}
sites_[siteIdx].activeResource.reset();
}
std::lock_guard <std::mutex> lock_state{state_mutex_};
fetching_ = false;
if (! stopping_)
setTimer ();
cv_.notify_all();
}
Json::Value
ValidatorSite::getJson() const
{

5
src/ripple/basics/ByteUtilities.h
View File

@@ -31,8 +31,11 @@ namespace ripple {
template<class T>
constexpr auto megabytes(T value) noexcept
{
return kilobytes(1) * kilobytes(1) * value;
return kilobytes(kilobytes(value));
}
static_assert(kilobytes(2) == 2048, "kilobytes(2) == 2048");
static_assert(megabytes(3) == 3145728, "megabytes(3) == 3145728");
}
#endif

39
src/ripple/basics/FileUtilities.h Normal file
View File

@@ -0,0 +1,39 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2018 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_BASICS_FILEUTILITIES_H_INCLUDED
#define RIPPLE_BASICS_FILEUTILITIES_H_INCLUDED
#include <boost/system/error_code.hpp>
#include <boost/filesystem.hpp>
#include <boost/optional.hpp>
namespace ripple
{
// TODO: Should this one function have its own file, or can it
// be absorbed somewhere else?
std::string getFileContents(boost::system::error_code& ec,
boost::filesystem::path const& sourcePath,
boost::optional<std::size_t> maxSize = boost::none);
}
#endif

63
src/ripple/basics/impl/FileUtilities.cpp Normal file
View File

@@ -0,0 +1,63 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2018 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/FileUtilities.h>
namespace ripple
{
std::string getFileContents(boost::system::error_code& ec,
boost::filesystem::path const& sourcePath,
boost::optional<std::size_t> maxSize)
{
using namespace boost::filesystem;
using namespace boost::system::errc;
path fullPath{ canonical(sourcePath, ec) };
if (ec)
return {};
if (maxSize && (file_size(fullPath, ec) > *maxSize || ec))
{
if (!ec)
ec = make_error_code(file_too_large);
return {};
}
ifstream fileStream(fullPath, std::ios::in);
if (!fileStream)
{
ec = make_error_code(static_cast<errc_t>(errno));
return {};
}
const std::string result{ std::istreambuf_iterator<char>{fileStream},
std::istreambuf_iterator<char>{} };
if (fileStream.bad ())
{
ec = make_error_code(static_cast<errc_t>(errno));
return {};
}
return result;
}
}

8
src/ripple/basics/impl/StringUtilities.cpp
View File

@@ -89,11 +89,15 @@ uint64_t uintFromHex (std::string const& strSrc)
return uValue;
}
// TODO Callers should be using beast::URL and beast::parse_URL instead.
bool parseUrl (parsedURL& pUrl, std::string const& strUrl)
{
// scheme://username:password@hostname:port/rest
static boost::regex reUrl ("(?i)\\`\\s*([[:alpha:]][-+.[:alpha:][:digit:]]*)://([^/]+)(/.*)?\\s*?\\'");
static boost::regex reUrl (
"(?i)\\`\\s*"
"([[:alpha:]][-+.[:alpha:][:digit:]]*):" //scheme
"//([^/]+)?" // hostname
"(/.*)?" // path and parameters
"\\s*?\\'");
boost::smatch smMatch;
bool bMatch = boost::regex_match (strUrl, smMatch, reUrl); // Match status code.

35
src/ripple/core/impl/Config.cpp
View File

@@ -20,6 +20,7 @@
#include <ripple/core/Config.h>
#include <ripple/core/ConfigSections.h>
#include <ripple/basics/contract.h>
#include <ripple/basics/FileUtilities.h>
#include <ripple/basics/Log.h>
#include <ripple/json/json_reader.h>
#include <ripple/protocol/Feature.h>
@@ -30,6 +31,7 @@
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
#include <boost/regex.hpp>
#include <boost/system/error_code.hpp>
#include <fstream>
#include <iostream>
#include <iterator>
@@ -268,21 +270,13 @@ void Config::load ()
if (!QUIET)
std::cerr << "Loading: " << CONFIG_FILE << "\n";
std::ifstream ifsConfig (CONFIG_FILE.c_str (), std::ios::in);
boost::system::error_code ec;
auto const fileContents = getFileContents(ec, CONFIG_FILE);
if (!ifsConfig)
if (ec)
{
std::cerr << "Failed to open '" << CONFIG_FILE << "'." << std::endl;
return;
}
std::string fileContents;
fileContents.assign ((std::istreambuf_iterator<char>(ifsConfig)),
std::istreambuf_iterator<char>());
if (ifsConfig.bad ())
{
std::cerr << "Failed to read '" << CONFIG_FILE << "'." << std::endl;
std::cerr << "Failed to read '" << CONFIG_FILE << "'." <<
ec.value() << ": " << ec.message() << std::endl;
return;
}
@@ -472,13 +466,14 @@ void Config::loadFromString (std::string const& fileContents)
(boost::filesystem::is_regular_file (validatorsFile) ||
boost::filesystem::is_symlink (validatorsFile)))
{
std::ifstream ifsDefault (validatorsFile.native().c_str());
std::string data;
data.assign (
std::istreambuf_iterator<char>(ifsDefault),
std::istreambuf_iterator<char>());
boost::system::error_code ec;
auto const data = getFileContents(ec, validatorsFile);
if (ec)
{
Throw<std::runtime_error>("Failed to read '" +
validatorsFile.string() + "'." +
std::to_string(ec.value()) + ": " + ec.message());
}
auto iniFile = parseIniFile (data, true);

1
src/ripple/unity/basics1.cpp
View File

@@ -21,6 +21,7 @@
#include <ripple/basics/impl/base64.cpp>
#include <ripple/basics/impl/contract.cpp>
#include <ripple/basics/impl/CountedObject.cpp>
#include <ripple/basics/impl/FileUtilities.cpp>
#include <ripple/basics/impl/Log.cpp>
#include <ripple/basics/impl/strHex.cpp>
#include <ripple/basics/impl/StringUtilities.cpp>