ValidatorList.h

//------------------------------------------------------------------------------
/*
    This file is part of rippled: https://github.com/ripple/rippled
    Copyright (c) 2015 Ripple Labs Inc.
 
    Permission to use, copy, modify, and/or distribute this software for any
    purpose  with  or without fee is hereby granted, provided that the above
    copyright notice and this permission notice appear in all copies.
 
    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
 
#ifndef RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED
#define RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED
 
#include <ripple/app/misc/Manifest.h>
#include <ripple/basics/Log.h>
#include <ripple/basics/UnorderedContainers.h>
#include <ripple/core/TimeKeeper.h>
#include <ripple/crypto/csprng.h>
#include <ripple/json/json_value.h>
#include <ripple/overlay/Message.h>
#include <ripple/protocol/PublicKey.h>
#include <boost/iterator/counting_iterator.hpp>
#include <boost/range/adaptors.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <mutex>
#include <numeric>
#include <shared_mutex>
 
namespace protocol {
class TMValidatorList;
class TMValidatorListCollection;
}  // namespace protocol
 
namespace ripple {
 
class Overlay;
class HashRouter;
class Message;
class NetworkOPs;
class Peer;
class STValidation;
 
/* Entries in this enum are ordered by "desirability".
   The "better" dispositions have lower values than the
   "worse" dispositions */
enum class ListDisposition {
    accepted = 0,
 
    expired,
 
    pending,
 
    same_sequence,
 
    known_sequence,
 
    stale,
 
    untrusted,
 
    unsupported_version,
 
    invalid
};
 
/* Entries in this enum are ordered by "desirability".
   The "better" dispositions have lower values than the
   "worse" dispositions */
enum class PublisherStatus {
    // Publisher has provided a valid file
    available = 0,
 
    // Current list is expired without replacement
    expired,
 
    // No file seen yet
    unavailable,
 
    // Publisher has revoked their manifest key
    revoked,
 
};
 
std::string
to_string(ListDisposition disposition);
 
struct TrustChanges
{
    explicit TrustChanges() = default;
 
    hash_set<NodeID> added;
    hash_set<NodeID> removed;
};
 
struct ValidatorBlobInfo
{
    // base-64 encoded JSON containing the validator list.
    std::string blob;
    // hex-encoded signature of the blob using the publisher's signing key
    std::string signature;
    // base-64 or hex-encoded manifest containing the publisher's master and
    // signing public keys
    std::optional<std::string> manifest;
};
 
class ValidatorList
{
    struct PublisherList
    {
        explicit PublisherList() = default;
 
        std::vector<PublicKey> list;
        std::vector<std::string> manifests;
        std::size_t sequence;
        TimeKeeper::time_point validFrom;
        TimeKeeper::time_point validUntil;
        std::string siteUri;
        // base-64 encoded JSON containing the validator list.
        std::string rawBlob;
        // hex-encoded signature of the blob using the publisher's signing key
        std::string rawSignature;
        // base-64 or hex-encoded manifest containing the publisher's master and
        // signing public keys
        std::optional<std::string> rawManifest;
        uint256 hash;
    };
 
    struct PublisherListCollection
    {
        PublisherStatus status;
        /*
        The `current` VL is the one which
         1. Has the largest sequence number that
         2. Has ever been effective (the effective date is absent or in the
            past).
        If this VL has expired, all VLs with previous sequence numbers
        will also be considered expired, and thus there will be no valid VL
        until one with a larger sequence number becomes effective. This is to
        prevent allowing old VLs to reactivate.
        */
        PublisherList current;
        /*
        The `remaining` list holds any relevant VLs which have a larger sequence
        number than current. By definition they will all have an effective date
        in the future. Relevancy will be determined by sorting the VLs by
        sequence number, then iterating over the list and removing any VLs for
        which the following VL (ignoring gaps) has the same or earlier effective
        date.
        */
        std::map<std::size_t, PublisherList> remaining;
        std::optional<std::size_t> maxSequence;
        // The hash of the full set if sent in a single message
        uint256 fullHash;
        std::string rawManifest;
        std::uint32_t rawVersion = 0;
    };
 
    ManifestCache& validatorManifests_;
    ManifestCache& publisherManifests_;
    TimeKeeper& timeKeeper_;
    boost::filesystem::path const dataPath_;
    beast::Journal const j_;
    boost::shared_mutex mutable mutex_;
    using lock_guard = std::lock_guard<decltype(mutex_)>;
    using shared_lock = std::shared_lock<decltype(mutex_)>;
 
    std::atomic<std::size_t> quorum_;
    std::optional<std::size_t> minimumQuorum_;
 
    // Published lists stored by publisher master public key
    hash_map<PublicKey, PublisherListCollection> publisherLists_;
 
    // Listed master public keys with the number of lists they appear on
    hash_map<PublicKey, std::size_t> keyListings_;
 
    // The current list of trusted master keys
    hash_set<PublicKey> trustedMasterKeys_;
 
    // The current list of trusted signing keys. For those validators using
    // a manifest, the signing key is the ephemeral key. For the ones using
    // a seed, the signing key is the same as the master key.
    hash_set<PublicKey> trustedSigningKeys_;
 
    std::optional<PublicKey> localPubKey_;
 
    // The below variable contains the Publisher list specified in the local
    // config file under the title of SECTION_VALIDATORS or [validators].
    // This list is not associated with the masterKey of any publisher.
 
    // Appropos PublisherListCollection fields, localPublisherList does not
    // have any "remaining" manifests. It is assumed to be perennially
    // "available". The "validUntil" field is set to the highest possible
    // value of the field, hence this list is always valid.
    PublisherList localPublisherList;
 
    // The master public keys of the current negative UNL
    hash_set<PublicKey> negativeUNL_;
 
    // Currently supported versions of publisher list format
    static constexpr std::uint32_t supportedListVersions[]{1, 2};
    // In the initial release, to prevent potential abuse and attacks, any VL
    // collection with more than 5 entries will be considered malformed.
    static constexpr std::size_t maxSupportedBlobs = 5;
    // Prefix of the file name used to store cache files.
    static const std::string filePrefix_;
 
public:
    ValidatorList(
        ManifestCache& validatorManifests,
        ManifestCache& publisherManifests,
        TimeKeeper& timeKeeper,
        std::string const& databasePath,
        beast::Journal j,
        std::optional<std::size_t> minimumQuorum = std::nullopt);
    ~ValidatorList() = default;
 
    struct PublisherListStats
    {
        explicit PublisherListStats() = default;
        explicit PublisherListStats(ListDisposition d);
        PublisherListStats(
            ListDisposition d,
            PublicKey key,
            PublisherStatus stat,
            std::size_t seq);
 
        ListDisposition
        bestDisposition() const;
        ListDisposition
        worstDisposition() const;
        void
        mergeDispositions(PublisherListStats const& src);
 
        // Tracks the dispositions of each processed list and how many times it
        // occurred
        std::map<ListDisposition, std::size_t> dispositions;
        std::optional<PublicKey> publisherKey;
        PublisherStatus status = PublisherStatus::unavailable;
        std::size_t sequence = 0;
    };
 
    struct MessageWithHash
    {
        explicit MessageWithHash() = default;
        explicit MessageWithHash(
            std::shared_ptr<Message> const& message_,
            uint256 hash_,
            std::size_t num_);
        std::shared_ptr<Message> message;
        uint256 hash;
        std::size_t numVLs = 0;
    };
 
    bool
    load(
        std::optional<PublicKey> const& localSigningKey,
        std::vector<std::string> const& configKeys,
        std::vector<std::string> const& publisherKeys);
 
    static std::vector<ValidatorBlobInfo>
    parseBlobs(std::uint32_t version, Json::Value const& body);
 
    static std::vector<ValidatorBlobInfo>
    parseBlobs(protocol::TMValidatorList const& body);
 
    static std::vector<ValidatorBlobInfo>
    parseBlobs(protocol::TMValidatorListCollection const& body);
 
    static void
    sendValidatorList(
        Peer& peer,
        std::uint64_t peerSequence,
        PublicKey const& publisherKey,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        HashRouter& hashRouter,
        beast::Journal j);
 
    [[nodiscard]] static std::pair<std::size_t, std::size_t>
    buildValidatorListMessages(
        std::size_t messageVersion,
        std::uint64_t peerSequence,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        std::vector<MessageWithHash>& messages,
        std::size_t maxSize = maximiumMessageSize);
 
    PublisherListStats
    applyListsAndBroadcast(
        std::string const& manifest,
        std::uint32_t version,
        std::vector<ValidatorBlobInfo> const& blobs,
        std::string siteUri,
        uint256 const& hash,
        Overlay& overlay,
        HashRouter& hashRouter,
        NetworkOPs& networkOPs);
 
    PublisherListStats
    applyLists(
        std::string const& manifest,
        std::uint32_t version,
        std::vector<ValidatorBlobInfo> const& blobs,
        std::string siteUri,
        std::optional<uint256> const& hash = {});
 
    /* Attempt to read previously stored list files. Expected to only be
       called when loading from URL fails.
 
       @return A list of valid file:// URLs, if any.
 
       @par Thread Safety
 
       May be called concurrently
    */
    std::vector<std::string>
    loadLists();
 
    TrustChanges
    updateTrusted(
        hash_set<NodeID> const& seenValidators,
        NetClock::time_point closeTime,
        NetworkOPs& ops,
        Overlay& overlay,
        HashRouter& hashRouter);
 
    std::size_t
    quorum() const
    {
        return quorum_;
    }
 
    bool
    trusted(PublicKey const& identity) const;
 
    bool
    listed(PublicKey const& identity) const;
 
    std::optional<PublicKey>
    getTrustedKey(PublicKey const& identity) const;
 
    std::optional<PublicKey>
    getListedKey(PublicKey const& identity) const;
 
    bool
    trustedPublisher(PublicKey const& identity) const;
 
    std::optional<PublicKey>
    localPublicKey() const;
 
    void
    for_each_listed(std::function<void(PublicKey const&, bool)> func) const;
 
    void
    for_each_available(
        std::function<void(
            std::string const& manifest,
            std::uint32_t version,
            std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
            PublicKey const& pubKey,
            std::size_t maxSequence,
            uint256 const& hash)> func) const;
 
    std::optional<Json::Value>
    getAvailable(
        boost::beast::string_view const& pubKey,
        std::optional<std::uint32_t> forceVersion = {});
 
    std::size_t
    count() const;
 
    std::optional<TimeKeeper::time_point>
    expires() const;
 
    Json::Value
    getJson() const;
 
    using QuorumKeys = std::pair<std::size_t const, hash_set<PublicKey>>;
    QuorumKeys
    getQuorumKeys() const
    {
        shared_lock read_lock{mutex_};
        return {quorum_, trustedSigningKeys_};
    }
 
    hash_set<PublicKey>
    getTrustedMasterKeys() const;
 
    hash_set<PublicKey>
    getNegativeUNL() const;
 
    void
    setNegativeUNL(hash_set<PublicKey> const& negUnl);
 
    std::vector<std::shared_ptr<STValidation>>
    negativeUNLFilter(
        std::vector<std::shared_ptr<STValidation>>&& validations) const;
 
private:
    std::size_t
    count(shared_lock const&) const;
 
    bool
    trusted(shared_lock const&, PublicKey const& identity) const;
 
    std::optional<PublicKey>
    getTrustedKey(shared_lock const&, PublicKey const& identity) const;
 
    std::optional<TimeKeeper::time_point>
    expires(shared_lock const&) const;
 
    PublisherListStats
    applyList(
        std::string const& globalManifest,
        std::optional<std::string> const& localManifest,
        std::string const& blob,
        std::string const& signature,
        std::uint32_t version,
        std::string siteUri,
        std::optional<uint256> const& hash,
        lock_guard const&);
 
    // This function updates the keyListings_ counts for all the trusted
    // master keys
    void
    updatePublisherList(
        PublicKey const& pubKey,
        PublisherList const& current,
        std::vector<PublicKey> const& oldList,
        lock_guard const&);
 
    static void
    buildBlobInfos(
        std::map<std::size_t, ValidatorBlobInfo>& blobInfos,
        PublisherListCollection const& lists);
 
    static std::map<std::size_t, ValidatorBlobInfo>
    buildBlobInfos(PublisherListCollection const& lists);
 
    static void
    broadcastBlobs(
        PublicKey const& publisherKey,
        PublisherListCollection const& lists,
        std::size_t maxSequence,
        uint256 const& hash,
        Overlay& overlay,
        HashRouter& hashRouter,
        beast::Journal j);
 
    static void
    sendValidatorList(
        Peer& peer,
        std::uint64_t peerSequence,
        PublicKey const& publisherKey,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        std::vector<MessageWithHash>& messages,
        HashRouter& hashRouter,
        beast::Journal j);
 
    boost::filesystem::path
    getCacheFileName(lock_guard const&, PublicKey const& pubKey) const;
 
    static Json::Value
    buildFileData(
        std::string const& pubKey,
        PublisherListCollection const& pubCollection,
        beast::Journal j);
 
    static Json::Value
    buildFileData(
        std::string const& pubKey,
        PublisherListCollection const& pubCollection,
        std::optional<std::uint32_t> forceVersion,
        beast::Journal j);
 
    template <class Hasher>
    friend void
    hash_append(Hasher& h, PublisherListCollection pl)
    {
        using beast::hash_append;
        hash_append(h, pl.rawManifest, buildBlobInfos(pl), pl.rawVersion);
    }
 
    void
    cacheValidatorFile(lock_guard const& lock, PublicKey const& pubKey) const;
 
    std::pair<ListDisposition, std::optional<PublicKey>>
    verify(
        lock_guard const&,
        Json::Value& list,
        std::string const& manifest,
        std::string const& blob,
        std::string const& signature);
 
    bool
    removePublisherList(
        lock_guard const&,
        PublicKey const& publisherKey,
        PublisherStatus reason);
 
    std::size_t
    calculateQuorum(
        std::size_t unlSize,
        std::size_t effectiveUnlSize,
        std::size_t seenSize);
};
 
// hashing helpers
template <class Hasher>
void
hash_append(Hasher& h, ValidatorBlobInfo const& blobInfo)
{
    using beast::hash_append;
    hash_append(h, blobInfo.blob, blobInfo.signature);
    if (blobInfo.manifest)
    {
        hash_append(h, *blobInfo.manifest);
    }
}
 
template <class Hasher>
void
hash_append(Hasher& h, std::vector<ValidatorBlobInfo> const& blobs)
{
    for (auto const& item : blobs)
        hash_append(h, item);
}
 
template <class Hasher>
void
hash_append(Hasher& h, std::map<std::size_t, ValidatorBlobInfo> const& blobs)
{
    for (auto const& [_, item] : blobs)
    {
        (void)_;
        hash_append(h, item);
    }
}
 
}  // namespace ripple
 
namespace protocol {
 
template <class Hasher>
void
hash_append(Hasher& h, TMValidatorList const& msg)
{
    using beast::hash_append;
    hash_append(h, msg.manifest(), msg.blob(), msg.signature(), msg.version());
}
 
template <class Hasher>
void
hash_append(Hasher& h, TMValidatorListCollection const& msg)
{
    using beast::hash_append;
    hash_append(
        h,
        msg.manifest(),
        ripple::ValidatorList::parseBlobs(msg),
        msg.version());
}
 
}  // namespace protocol
 
#endif