Dynamize trusted validator list and quorum (RIPD-1220):

Instead of specifying a static list of trusted validators in the config
or validators file, the configuration can now include trusted validator
list publisher keys.

The trusted validator list and quorum are now reset each consensus
round using the latest validator lists and the list of recent
validations seen. The minimum validation quorum is now only
configurable via the command line.

Builds/VisualStudio2015/RippleD.vcxproj

@@ -1061,6 +1061,10 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\app\misc\impl\Manifest.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple\app\misc\impl\Transaction.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -1075,6 +1079,8 @@
     </ClCompile>
     <ClInclude Include="..\..\src\ripple\app\misc\LoadFeeTrack.h">
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\app\misc\Manifest.h">
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\app\misc\NetworkOPs.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -2297,12 +2303,6 @@
     </ClCompile>
     <ClInclude Include="..\..\src\ripple\overlay\impl\ConnectAttempt.h">
     </ClInclude>
-    <ClCompile Include="..\..\src\ripple\overlay\impl\Manifest.cpp">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
-    </ClCompile>
-    <ClInclude Include="..\..\src\ripple\overlay\impl\Manifest.h">
-    </ClInclude>
     <ClCompile Include="..\..\src\ripple\overlay\impl\Message.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -4199,6 +4199,10 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\app\Manifest_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\app\MultiSign_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -4715,10 +4719,6 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\test\overlay\manifest_test.cpp">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\src\test\overlay\short_read_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>

Builds/VisualStudio2015/RippleD.vcxproj.filters

@@ -1569,6 +1569,9 @@
     <ClCompile Include="..\..\src\ripple\app\misc\impl\LoadFeeTrack.cpp">
       <Filter>ripple\app\misc\impl</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\app\misc\impl\Manifest.cpp">
+      <Filter>ripple\app\misc\impl</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple\app\misc\impl\Transaction.cpp">
       <Filter>ripple\app\misc\impl</Filter>
     </ClCompile>
@@ -1581,6 +1584,9 @@
     <ClInclude Include="..\..\src\ripple\app\misc\LoadFeeTrack.h">
       <Filter>ripple\app\misc</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\app\misc\Manifest.h">
+      <Filter>ripple\app\misc</Filter>
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\app\misc\NetworkOPs.cpp">
       <Filter>ripple\app\misc</Filter>
     </ClCompile>
@@ -2880,12 +2886,6 @@
     <ClInclude Include="..\..\src\ripple\overlay\impl\ConnectAttempt.h">
       <Filter>ripple\overlay\impl</Filter>
     </ClInclude>
-    <ClCompile Include="..\..\src\ripple\overlay\impl\Manifest.cpp">
-      <Filter>ripple\overlay\impl</Filter>
-    </ClCompile>
-    <ClInclude Include="..\..\src\ripple\overlay\impl\Manifest.h">
-      <Filter>ripple\overlay\impl</Filter>
-    </ClInclude>
     <ClCompile Include="..\..\src\ripple\overlay\impl\Message.cpp">
       <Filter>ripple\overlay\impl</Filter>
     </ClCompile>
@@ -4983,6 +4983,9 @@
     <ClCompile Include="..\..\src\test\app\LoadFeeTrack_test.cpp">
       <Filter>test\app</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\app\Manifest_test.cpp">
+      <Filter>test\app</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\app\MultiSign_test.cpp">
       <Filter>test\app</Filter>
     </ClCompile>
@@ -5427,9 +5430,6 @@
     <ClCompile Include="..\..\src\test\overlay\cluster_test.cpp">
       <Filter>test\overlay</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\test\overlay\manifest_test.cpp">
-      <Filter>test\overlay</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\test\overlay\short_read_test.cpp">
       <Filter>test\overlay</Filter>
     </ClCompile>

bin/python/ripple/util/ValidatorManifestTest.py

@@ -104,9 +104,6 @@ n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C    RL3
 n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS    RL4
 n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA    RL5
 
-[validation_quorum]
-3
-
 [validation_seed]
 {validation_seed}
 #vaidation_public_key: {validation_public_key}

bin/python/ripple/util/test_ConfigFile.py

@@ -45,7 +45,6 @@ RESULT = {
     'websocket_public_port': '5206',
     'peer_ip': '0.0.0.0',
     'rpc_port': '5205',
-    'validation_quorum': '3',
     'websocket_ip': '127.0.0.1'}
 
 FULL = """
@@ -131,10 +130,6 @@ n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C	RL3
 n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS	RL4
 n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA	RL5
 
-# Ditto.
-[validation_quorum]
-3
-
 [validation_seed]
 sh1T8T9yGuV7Jb6DPhqSzdU2s5LcV
 

doc/rippled-example.cfg

@@ -546,8 +546,7 @@
 #
 #   These settings affect the behavior of the server instance with respect
 #   to Ripple payment protocol level activities such as validating and
-#   closing ledgers, establishing a quorum, or adjusting fees in response
+#   closing ledgers or adjusting fees in response to server overloads.
-#   to server overloads.
 #
 #
 #
@@ -607,11 +606,14 @@
 #   to always accept as validators as well as the minimum number of validators
 #   needed to accept consensus.
 #
-#   The contents of the file should include a [validators] and a
+#   The contents of the file should include a [validators] and/or
-#   [validation_quorum] entry. [validators] should be followed by
+#   [validator_list_keys] entries.
-#   a list of validation public keys of nodes, one per line, optionally
+#   [validators] should be followed by a list of validation public keys of
-#   followed by a comment separated by whitespace.
+#   nodes, one per line.
-#   [validation_quorum] should be followed by a number.
+#   [validator_list_keys] should be followed by a list of keys belonging to
+#   trusted validator list publishers. Validator lists will only be
+#   considered if the list is accompanied by a valid signature from a trusted
+#   publisher key.
 #
 #   Specify the file by its name or path.
 #   Unless an absolute path is specified, it will be considered relative to
@@ -623,14 +625,11 @@
 #
 #   Example content:
 #    [validators]
-#    n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 RL1
+#    n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7
-#    n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj RL2
+#    n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj
-#    n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C RL3
+#    n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C
-#    n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS RL4
+#    n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS
-#    n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA RL5
+#    n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA
-#
-#    [validation_quorum]
-#    3
 #
 #
 # [path_search]
@@ -1021,7 +1020,7 @@ pool.ntp.org
 [ips]
 r.ripple.com 51235
 
-# File containing validation quorum and trusted validator keys.
+# File containing trusted validator keys or validator list publishers.
 # Unless an absolute path is specified, it will be considered relative to the
 # folder in which the rippled.cfg file is located.
 [validators_file]

doc/validators-example.txt

@@ -13,8 +13,6 @@
 # [validators]
 #
 #   List of the validation public keys of nodes to always accept as validators.
-#   A comment may, optionally, be associated with each entry, separated by
-#   whitespace from the validation public key.
 #
 #   The latest list of recommended validators can be obtained from
 #   https://ripple.com/ripple.txt
@@ -23,26 +21,27 @@
 #
 #   Examples:
 #    n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5
-#    n9MqiExBcoG19UXwoLjBJnhsxEhAZMuWwJDRdkyDz1EkEkwzQTNt John Doe
+#    n9MqiExBcoG19UXwoLjBJnhsxEhAZMuWwJDRdkyDz1EkEkwzQTNt
 #
 #
 #
-# [validation_quorum]
+# [validator_list_keys]
 #
-#   Sets the minimum number of trusted validations a ledger must have before
+#   List of keys belonging to trusted validator list publishers.
-#   the server considers it fully validated. Note that if you are validating,
+#   Validator lists will only be considered if the list is accompanied by a
-#   your validation counts.
+#   valid signature from a trusted publisher key.
+#   Validator list keys should be hex-encoded.
+#
+#   Examples:
+#    ed499d732bded01504a7407c224412ef550cc1ade638a4de4eb88af7c36cb8b282
+#    0202d3f36a801349f3be534e3f64cfa77dede6e1b6310a0b48f40f20f955cec945
+#    02dd8b7075f64d77d9d2bdb88da364f29fcd975f9ea6f21894abcc7564efda8054
 #
 
 # Public keys of the validators that this rippled instance trusts.
 [validators]
-n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 RL1
+n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7
-n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj RL2
+n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj
-n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C RL3
+n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C
-n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS RL4
+n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS
-n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA RL5
+n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA
-
-# The number of validators rippled needs to accept a consensus.
-# Don't change this unless you know what you're doing.
-[validation_quorum]
-3

src/ripple/app/ledger/LedgerMaster.h

@@ -113,10 +113,6 @@ public:
     std::chrono::seconds getValidatedLedgerAge ();
     bool isCaughtUp(std::string& reason);
 
-    int getMinValidations ();
-
-    void setMinValidations (int v, bool strict);
-
     std::uint32_t getEarliestFetch ();
 
     bool storeLedger (std::shared_ptr<Ledger const> ledger);
@@ -257,7 +253,7 @@ private:
 
     void getFetchPack(LedgerHash missingHash, LedgerIndex missingIndex);
     boost::optional<LedgerHash> getLedgerHashForHistory(LedgerIndex index);
-    int getNeededValidations();
+    std::size_t getNeededValidations();
     void advanceThread();
     // Try to publish ledgers, acquire missing ledgers
     void doAdvance();
@@ -313,8 +309,6 @@ private:
 
     std::unique_ptr <detail::LedgerCleaner> mLedgerCleaner;
 
-    int mMinValidations;    // The minimum validations to publish a ledger.
-    bool mStrictValCount;   // Don't raise the minimum
     uint256 mLastValidateHash;
     std::uint32_t mLastValidateSeq;
 

src/ripple/app/ledger/impl/LedgerConsensusImp.cpp

@@ -35,6 +35,7 @@
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/app/misc/Validations.h>
+#include <ripple/app/misc/ValidatorList.h>
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/contract.h>
 #include <ripple/basics/CountedObject.h>
@@ -1267,14 +1268,14 @@ LedgerConsensusImp<Traits>::makeInitialPosition () ->
             app_.getValidations().getValidations (
                 previousLedger_->info().parentHash);
 
-        auto const count = std::count_if (
+        std::size_t const count = std::count_if (
             validations.begin(), validations.end(),
             [](auto const& v)
             {
                 return v.second->isTrusted();
             });
 
-        if (count >= ledgerMaster_.getMinValidations())
+        if (count >= app_.validators ().quorum ())
         {
             feeVote_.doVoting (
                 previousLedger_,

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -33,6 +33,7 @@
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/app/misc/Validations.h>
+#include <ripple/app/misc/ValidatorList.h>
 #include <ripple/app/paths/PathRequests.h>
 #include <ripple/basics/contract.h>
 #include <ripple/basics/Log.h>
@@ -53,9 +54,6 @@ namespace ripple {
 
 using namespace std::chrono_literals;
 
-// 150/256ths of validations of previous ledger
-#define MIN_VALIDATION_RATIO    150
-
 // Don't catch up more than 100 ledgers (cannot exceed 256)
 #define MAX_LEDGER_GAP          100
 
@@ -73,8 +71,6 @@ LedgerMaster::LedgerMaster (Application& app, Stopwatch& stopwatch,
     , mHeldTransactions (uint256 ())
     , mLedgerCleaner (detail::make_LedgerCleaner (
         app, *this, app_.journal("LedgerCleaner")))
-    , mMinValidations (0)
-    , mStrictValCount (false)
     , mLastValidateSeq (0)
     , mAdvanceThread (false)
     , mAdvanceWork (false)
@@ -115,13 +111,6 @@ LedgerMaster::isCompatible (
     beast::Journal::Stream s,
     char const* reason)
 {
-    if (mStrictValCount)
-    {
-        // If we're only using validation count, then we can't
-        // reject a ledger even if it's incompatible
-        return true;
-    }
-
     auto validLedger = getValidatedLedger();
 
     if (validLedger &&
@@ -217,7 +206,7 @@ LedgerMaster::setValidLedger(
 
     NetClock::time_point signTime;
 
-    if (! times.empty () && times.size() >= mMinValidations)
+    if (! times.empty () && times.size() >= app_.validators ().quorum ())
     {
         // Calculate the sample median
         std::sort (times.begin (), times.end ());
@@ -702,7 +691,7 @@ LedgerMaster::failedSave(std::uint32_t seq, uint256 const& hash)
 void
 LedgerMaster::checkAccept (uint256 const& hash, std::uint32_t seq)
 {
-    int valCount = 0;
+    std::size_t valCount = 0;
 
     if (seq != 0)
     {
@@ -713,18 +702,11 @@ LedgerMaster::checkAccept (uint256 const& hash, std::uint32_t seq)
         valCount =
             app_.getValidations().getTrustedValidationCount (hash);
 
-        if (valCount >= mMinValidations)
+        if (valCount >= app_.validators ().quorum ())
         {
             ScopedLockType ml (m_mutex);
             if (seq > mLastValidLedger.second)
                 mLastValidLedger = std::make_pair (hash, seq);
-
-            if (!mStrictValCount && (mMinValidations < (valCount/2 + 1)))
-            {
-                mMinValidations = (valCount/2 + 1);
-                JLOG (m_journal.info())
-                    << "Raising minimum validations to " << mMinValidations;
-            }
         }
 
         if (seq == mValidLedgerSeq)
@@ -742,7 +724,7 @@ LedgerMaster::checkAccept (uint256 const& hash, std::uint32_t seq)
         if ((seq != 0) && (getValidLedgerIndex() == 0))
         {
             // Set peers sane early if we can
-            if (valCount >= mMinValidations)
+            if (valCount >= app_.validators ().quorum ())
                 app_.overlay().checkSanity (seq);
         }
 
@@ -757,30 +739,14 @@ LedgerMaster::checkAccept (uint256 const& hash, std::uint32_t seq)
 }
 
 /**
-    * Determines how many validations are needed to fully-validated a ledger
+    * Determines how many validations are needed to fully validate a ledger
     *
     * @return Number of validations needed
     */
-int
+std::size_t
 LedgerMaster::getNeededValidations ()
 {
-    if (standalone_)
+    return standalone_ ? 0 : app_.validators().quorum ();
-        return 0;
-
-    int minVal = mMinValidations;
-
-    if (mLastValidateHash.isNonZero ())
-    {
-        int val = app_.getValidations ().getTrustedValidationCount (
-            mLastValidateHash);
-        val *= MIN_VALIDATION_RATIO;
-        val /= 256;
-
-        if (val > minVal)
-            minVal = val;
-    }
-
-    return minVal;
 }
 
 void
@@ -904,7 +870,7 @@ LedgerMaster::consensusBuilt(
                 ledgerSeq_ = seq;
         }
 
-        int valCount_;
+        std::size_t valCount_;
         LedgerIndex ledgerSeq_;
     };
 
@@ -916,7 +882,7 @@ LedgerMaster::consensusBuilt(
         vs.mergeValidation (v->getFieldU32 (sfLedgerSequence));
     }
 
-    auto neededValidations = getNeededValidations ();
+    auto const neededValidations = getNeededValidations ();
     auto maxSeq = mValidLedgerSeq.load();
     auto maxLedger = ledger->info().hash;
 
@@ -1313,21 +1279,6 @@ LedgerMaster::getPublishedLedger ()
     return mPubLedger;
 }
 
-int
-LedgerMaster::getMinValidations ()
-{
-    return mMinValidations;
-}
-
-void
-LedgerMaster::setMinValidations (int v, bool strict)
-{
-    JLOG (m_journal.info()) << "Validation quorum: " << v
-        << (strict ? " strict" : "");
-    mMinValidations = v;
-    mStrictValCount = strict;
-}
-
 std::string
 LedgerMaster::getCompleteLedgers ()
 {

src/ripple/app/main/Application.cpp

@@ -39,6 +39,7 @@
 #include <ripple/app/misc/AmendmentTable.h>
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
+#include <ripple/app/misc/Manifest.h>
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/misc/SHAMapStore.h>
 #include <ripple/app/misc/TxQ.h>
@@ -347,6 +348,8 @@ public:
     TaggedCache <uint256, AcceptedLedger> m_acceptedLedgerCache;
     std::unique_ptr <NetworkOPs> m_networkOPs;
     std::unique_ptr <Cluster> cluster_;
+    std::unique_ptr <ManifestCache> validatorManifests_;
+    std::unique_ptr <ManifestCache> publisherManifests_;
     std::unique_ptr <ValidatorList> validators_;
     std::unique_ptr <ServerHandler> serverHandler_;
     std::unique_ptr <AmendmentTable> m_amendmentTable;
@@ -473,8 +476,15 @@ public:
         , cluster_ (std::make_unique<Cluster> (
             logs_->journal("Overlay")))
 
+        , validatorManifests_ (std::make_unique<ManifestCache> (
+            logs_->journal("ManifestCache")))
+
+        , publisherManifests_ (std::make_unique<ManifestCache> (
+            logs_->journal("ManifestCache")))
+
         , validators_ (std::make_unique<ValidatorList> (
-            logs_->journal("UniqueNodeList")))
+            *validatorManifests_, *publisherManifests_, *timeKeeper_,
+            logs_->journal("ValidatorList"), config_->VALIDATION_QUORUM))
 
         , serverHandler_ (make_ServerHandler (*this, *m_networkOPs, get_io_service (),
             *m_jobQueue, *m_networkOPs, *m_resourceManager, *m_collectorManager))
@@ -691,6 +701,16 @@ public:
         return *validators_;
     }
 
+    ManifestCache& validatorManifests() override
+    {
+        return *validatorManifests_;
+    }
+
+    ManifestCache& publisherManifests() override
+    {
+        return *publisherManifests_;
+    }
+
     Cluster& cluster () override
     {
         return *cluster_;
@@ -846,7 +866,18 @@ public:
 
         mValidations->flush ();
 
-        m_overlay->saveValidatorKeyManifests (getWalletDB ());
+        // TODO Store manifests in manifests.sqlite instead of wallet.db
+        validatorManifests_->save (getWalletDB (), "ValidatorManifests",
+            [this](PublicKey const& pubKey)
+            {
+                return validators().listed (pubKey);
+            });
+
+        publisherManifests_->save (getWalletDB (), "PublisherManifests",
+            [this](PublicKey const& pubKey)
+            {
+                return validators().trustedPublisher (pubKey);
+            });
 
         stopped ();
     }
@@ -1013,9 +1044,6 @@ bool ApplicationImp::setup()
 
     Pathfinder::initPathTable();
 
-    m_ledgerMaster->setMinValidations (
-        config_->VALIDATION_QUORUM, config_->LOCK_QUORUM);
-
     auto const startUp = config_->START_UP;
     if (startUp == Config::FRESH)
     {
@@ -1062,15 +1090,26 @@ bool ApplicationImp::setup()
         return false;
     }
 
-    if (!validators_->load (config().section (SECTION_VALIDATORS)))
+    if (!validatorManifests_->load (
+        getWalletDB (), "ValidatorManifests",
+        config().section (SECTION_VALIDATION_MANIFEST).lines()))
     {
-        JLOG(m_journal.fatal()) << "Invalid entry in validator configuration.";
+        JLOG(m_journal.fatal()) << "Invalid configured validator manifest.";
         return false;
     }
 
-    if (validators_->size () == 0 && !config_->standalone())
+    publisherManifests_->load (
+        getWalletDB (), "PublisherManifests");
+
+    // Setup trusted validators
+    if (!validators_->load (
+            config_->VALIDATION_PUB,
+            config().section (SECTION_VALIDATORS).values (),
+            config().section (SECTION_VALIDATOR_LIST_KEYS).values ()))
     {
-        JLOG(m_journal.warn()) << "No validators are configured.";
+        JLOG(m_journal.fatal()) <<
+            "Invalid entry in validator configuration.";
+        return false;
     }
 
     m_nodeStore->tune (config_->getSize (siNodeCacheSize), config_->getSize (siNodeCacheAge));
@@ -1101,8 +1140,6 @@ bool ApplicationImp::setup()
         return false;
     }
 
-    m_overlay->setupValidatorKeyManifests (*config_, getWalletDB ());
-
     {
         auto setup = setup_ServerHandler(
             *config_,

src/ripple/app/main/Application.h

@@ -50,6 +50,7 @@ class InboundTransactions;
 class AcceptedLedger;
 class LedgerMaster;
 class LoadManager;
+class ManifestCache;
 class NetworkOPs;
 class OpenLedger;
 class OrderBookDB;
@@ -120,6 +121,8 @@ public:
     virtual Overlay&                overlay () = 0;
     virtual TxQ&                    getTxQ() = 0;
     virtual ValidatorList&          validators () = 0;
+    virtual ManifestCache&          validatorManifests () = 0;
+    virtual ManifestCache&          publisherManifests () = 0;
     virtual Cluster&                cluster () = 0;
     virtual Validations&            getValidations () = 0;
     virtual NodeStore::Database&    getNodeStore () = 0;

src/ripple/app/main/DBInit.cpp

@@ -135,6 +135,10 @@ const char* WalletDBInit[] =
         RawData          BLOB NOT NULL               \
     );",
 
+    "CREATE TABLE IF NOT EXISTS PublisherManifests ( \
+        RawData          BLOB NOT NULL               \
+    );",
+
     // Old tables that were present in wallet.db and we
     // no longer need or use.
     "DROP INDEX IF EXISTS SeedNodeNext;",

src/ripple/app/main/Main.cpp

@@ -218,7 +218,7 @@ int run (int argc, char** argv)
     ("unittest-arg", po::value <std::string> ()->implicit_value (""), "Supplies argument to unit tests.")
     ("parameters", po::value< vector<string> > (), "Specify comma separated parameters.")
     ("quiet,q", "Reduce diagnotics.")
-    ("quorum", po::value <int> (), "Set the validation quorum.")
+    ("quorum", po::value <std::size_t> (), "Override the minimum validation quorum.")
     ("silent", "No output to the console after startup.")
     ("verbose,v", "Verbose logging.")
     ("load", "Load the current ledger from the local DB.")
@@ -338,9 +338,6 @@ int run (int argc, char** argv)
         }
 
         config->START_UP = Config::NETWORK;
-
-        if (config->VALIDATION_QUORUM < 2)
-            config->VALIDATION_QUORUM = 2;
     }
 
     // Override the RPC destination IP address. This must
@@ -385,11 +382,7 @@ int run (int argc, char** argv)
     {
         try
         {
-            config->VALIDATION_QUORUM = vm["quorum"].as <int> ();
+            config->VALIDATION_QUORUM = vm["quorum"].as <std::size_t> ();
-            config->LOCK_QUORUM = true;
-
-            if (config->VALIDATION_QUORUM < 0)
-                Throw<std::domain_error> ("");
         }
         catch(std::exception const&)
         {

src/ripple/app/misc/Manifest.h

@@ -17,14 +17,11 @@
 */
 //==============================================================================
 
-#ifndef RIPPLE_OVERLAY_MANIFEST_H_INCLUDED
+#ifndef RIPPLE_APP_MISC_MANIFEST_H_INCLUDED
-#define RIPPLE_OVERLAY_MANIFEST_H_INCLUDED
+#define RIPPLE_APP_MISC_MANIFEST_H_INCLUDED
 
-#include <ripple/app/misc/ValidatorList.h>
-#include <ripple/basics/BasicConfig.h>
 #include <ripple/basics/UnorderedContainers.h>
 #include <ripple/protocol/PublicKey.h>
-#include <ripple/protocol/STExchange.h>
 #include <ripple/beast/utility/Journal.h>
 #include <boost/optional.hpp>
 #include <string>
@@ -57,13 +54,10 @@ namespace ripple {
     There are two stores of information within rippled related to manifests.
     An instance of ManifestCache stores, for each trusted validator, (a) its
     master public key, and (b) the most senior of all valid manifests it has
-    seen for that validator, if any.  On startup, the [validator_keys] config
+    seen for that validator, if any.  On startup, the [validation_manifest]
-    entries are used to prime the manifest cache with the trusted master keys.
+    config entry (which is the manifest for this validator) is decoded and
-    At this point, the manifest cache has all the entries it will ever have,
+    added to the manifest cache.  Other manifests are added as "gossip" is
-    but none of them have manifests.  The [validation_manifest] config entry
+    received from rippled peers.
-    (which is the manifest for this validator) is then decoded and added to
-    the manifest cache.  Other manifests are added as "gossip" is received
-    from rippled peers.
 
     The other data store (which does not involve manifests per se) contains
     the set of active ephemeral validator keys.  Keys are added to the set
@@ -97,127 +91,219 @@ struct Manifest
     Manifest(Manifest&& other) = default;
     Manifest& operator=(Manifest&& other) = default;
 
+    inline bool
+    operator==(Manifest const& rhs) const
+    {
+        return sequence == rhs.sequence && masterKey == rhs.masterKey &&
+               signingKey == rhs.signingKey && serialized == rhs.serialized;
+    }
+
+    inline bool
+    operator!=(Manifest const& rhs) const
+    {
+        return !(*this == rhs);
+    }
+
+    /** Constructs Manifest from serialized string
+
+        @param s Serialized manifest string
+
+        @return `boost::none` if string is invalid
+    */
+    static boost::optional<Manifest> make_Manifest(std::string s);
+
+    /// Returns `true` if manifest signature is valid
     bool verify () const;
+
+    /// Returns hash of serialized manifest data
     uint256 hash () const;
+
+    /// Returns `true` if manifest revokes master key
     bool revoked () const;
+
+    /// Returns manifest signature
     Blob getSignature () const;
 
     /// Returns manifest master key signature
     Blob getMasterSignature () const;
 };
 
-boost::optional<Manifest> make_Manifest(std::string s);
-
-inline bool operator==(Manifest const& lhs, Manifest const& rhs)
-{
-    return lhs.serialized == rhs.serialized && lhs.masterKey == rhs.masterKey &&
-           lhs.signingKey == rhs.signingKey && lhs.sequence == rhs.sequence;
-}
-
-inline bool operator!=(Manifest const& lhs, Manifest const& rhs)
-{
-    return !(lhs == rhs);
-}
-
 enum class ManifestDisposition
 {
-    accepted = 0, // everything checked out
+    /// Manifest is valid
+    accepted = 0,
 
-    untrusted,    // manifest declares a master key we don't trust
+    /// Sequence is too old
-    stale,        // trusted master key, but seq is too old
+    stale,
-    invalid,      // trusted and timely, but invalid signature
+
+    /// Timely, but invalid signature
+    invalid
 };
 
 class DatabaseCon;
+
 /** Remembers manifests with the highest sequence number. */
 class ManifestCache
 {
 private:
-    struct MappedType
+    beast::Journal mutable j_;
-    {
+    std::mutex apply_mutex_;
-        MappedType() = default;
+    std::mutex mutable read_mutex_;
-        MappedType(MappedType&&) = default;
-        MappedType& operator=(MappedType&&) = default;
 
-        MappedType(std::string comment,
+    /** Active manifests stored by master public key. */
-                   std::string serialized,
+    hash_map <PublicKey, Manifest> map_;
-                   PublicKey pk, PublicKey spk, std::uint32_t seq)
-            :comment (std::move(comment))
-        {
-            m.emplace (std::move(serialized), std::move(pk), std::move(spk),
-                       seq);
-        }
 
-        std::string comment;
+    /** Master public keys stored by current ephemeral public key. */
-        boost::optional<Manifest> m;
+    hash_map <PublicKey, PublicKey> signingToMasterKeys_;
-    };
-
-    using MapType = hash_map<PublicKey, MappedType>;
-
-    mutable std::mutex mutex_;
-    MapType map_;
-
-    ManifestDisposition
-    canApply (PublicKey const& pk, std::uint32_t seq,
-        beast::Journal journal) const;
 
 public:
-    ManifestCache() = default;
+    explicit
-    ManifestCache (ManifestCache const&) = delete;
+    ManifestCache (beast::Journal j = beast::Journal())
-    ManifestCache& operator= (ManifestCache const&) = delete;
+        : j_ (j)
-    ~ManifestCache() = default;
+    {
+    };
 
-    bool loadValidatorKeys(
+    /** Returns master key's current signing key.
-        Section const& keys,
-        beast::Journal journal);
 
-    void configManifest (Manifest m, ValidatorList& unl, beast::Journal journal);
+        @param pk Master public key
 
-    /** Determines whether a node is in the trusted master key list */
+        @return pk if no known signing key from a manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    PublicKey
+    getSigningKey (PublicKey const& pk) const;
+
+    /** Returns ephemeral signing key's master public key.
+
+        @param pk Ephemeral signing public key
+
+        @return pk if signing key is not in a valid manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    PublicKey
+    getMasterKey (PublicKey const& pk) const;
+
+    /** Returns `true` if master key has been revoked in a manifest.
+
+        @param pk Master public key
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     bool
-    trusted (
+    revoked (PublicKey const& pk) const;
-        PublicKey const& identity) const;
 
-    void addTrustedKey (PublicKey const& pk, std::string comment);
+    /** Add manifest to cache.
 
-    /** The number of installed trusted master keys */
+        @param m Manifest to add
-    std::size_t
-    size () const;
 
+        @return `ManifestDisposition::accepted` if successful, or
+                `stale` or `invalid` otherwise
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     ManifestDisposition
     applyManifest (
-        Manifest m, ValidatorList& unl, beast::Journal journal);
+        Manifest m);
 
+    /** Populate manifest cache with manifests in database and config.
+
+        @param dbCon Database connection with dbTable
+
+        @param dbTable Database table
+
+        @param configManifest Base64 encoded manifest for local node's
+            validator keys
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    bool load (
+        DatabaseCon& dbCon, std::string const& dbTable,
+        std::vector<std::string> const& configManifest);
+
+    /** Populate manifest cache with manifests in database.
+
+        @param dbCon Database connection with dbTable
+
+        @param dbTable Database table
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     void load (
-        DatabaseCon& dbCon, ValidatorList& unl, beast::Journal journal);
+        DatabaseCon& dbCon, std::string const& dbTable);
-    void save (DatabaseCon& dbCon) const;
 
-    // A "for_each" for populated manifests only
+    /** Save cached manifests to database.
+
+        @param dbCon Database connection with `ValidatorManifests` table
+
+        @param isTrusted Function that returns true if manifest is trusted
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    void save (
+        DatabaseCon& dbCon, std::string const& dbTable,
+        std::function <bool (PublicKey const&)> isTrusted);
+
+    /** Invokes the callback once for every populated manifest.
+
+        @note Undefined behavior results when calling ManifestCache members from
+        within the callback
+
+        @param f Function called for each manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     template <class Function>
     void
     for_each_manifest(Function&& f) const
     {
-        std::lock_guard<std::mutex> lock (mutex_);
+        std::lock_guard<std::mutex> lock{read_mutex_};
-        for (auto const& e : map_)
+        for (auto const& m : map_)
         {
-            if (auto const& m = e.second.m)
+            f(m.second);
-                f(*m);
         }
     }
 
-    // A "for_each" for populated manifests only
+    /** Invokes the callback once for every populated manifest.
-    // The PreFun is called with the maximum number of
+
-    // times EachFun will be called (useful for memory allocations)
+        @note Undefined behavior results when calling ManifestCache members from
+        within the callback
+
+        @param pf Pre-function called with the maximum number of times f will be
+            called (useful for memory allocations)
+
+        @param f Function called for each manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     template <class PreFun, class EachFun>
     void
     for_each_manifest(PreFun&& pf, EachFun&& f) const
     {
-        std::lock_guard<std::mutex> lock (mutex_);
+        std::lock_guard<std::mutex> lock{read_mutex_};
         pf(map_.size ());
-        for (auto const& e : map_)
+        for (auto const& m : map_)
         {
-            if (auto const& m = e.second.m)
+            f(m.second);
-                f(*m);
         }
     }
 };

src/ripple/app/misc/NetworkOPs.cpp

@@ -1202,8 +1202,8 @@ public:
             if (nodesUsing > v.nodesUsing)
                 return true;
 
-            if (nodesUsing < v.nodesUsing) return
+            if (nodesUsing < v.nodesUsing)
-                false;
+                return false;
 
             return highNodeUsing > v.highNodeUsing;
         }
@@ -1485,6 +1485,9 @@ bool NetworkOPsImp::beginConsensus (uint256 const& networkClosed)
     assert (closingInfo.parentHash ==
             m_ledgerMaster.getClosedLedger()->info().hash);
 
+    app_.validators().onConsensusStart (
+        app_.getValidations().getCurrentPublicKeys ());
+
     mConsensus->startRound (
         *mLedgerConsensus,
         networkClosed,
@@ -2032,7 +2035,8 @@ Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin)
     if (mNeedNetworkLedger)
         info[jss::network_ledger] = "waiting";
 
-    info[jss::validation_quorum] = m_ledgerMaster.getMinValidations ();
+    info[jss::validation_quorum] = static_cast<Json::UInt>(
+        app_.validators ().quorum ());
 
     info[jss::io_latency_ms] = static_cast<Json::UInt> (
         app_.getIOLatency().count());
@@ -2050,7 +2054,7 @@ Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin)
                 s.reserve (Manifest::textLength);
                 for (auto const& line : validation_manifest.lines())
                     s += beast::rfc2616::trim(line);
-                if (auto mo = make_Manifest (beast::detail::base64_decode(s)))
+                if (auto mo = Manifest::make_Manifest (beast::detail::base64_decode(s)))
                 {
                     Json::Value valManifest = Json::objectValue;
                     valManifest [jss::master_key] = toBase58 (

src/ripple/app/misc/Validations.cpp

@@ -88,9 +88,12 @@ private:
     bool addValidation (STValidation::ref val, std::string const& source) override
     {
         auto signer = val->getSignerPublic ();
+        auto hash = val->getLedgerHash ();
         bool isCurrent = current (val);
 
-        if (!val->isTrusted() && app_.validators().trusted (signer))
+        auto pubKey = app_.validators ().getTrustedKey (signer);
+
+        if (!val->isTrusted() && pubKey)
             val->setTrusted();
 
         if (!val->isTrusted ())
@@ -98,36 +101,61 @@ private:
             JLOG (j_.trace()) <<
                 "Node " << toBase58 (TokenType::TOKEN_NODE_PUBLIC, signer) <<
                 " not in UNL st=" << val->getSignTime().time_since_epoch().count() <<
-                ", hash=" << val->getLedgerHash () <<
+                ", hash=" << hash <<
                 ", shash=" << val->getSigningHash () <<
                 " src=" << source;
         }
 
-        auto hash = val->getLedgerHash ();
+        if (! pubKey)
-        auto node = val->getNodeID ();
+            pubKey = app_.validators ().getListedKey (signer);
 
-        if (val->isTrusted () && isCurrent)
+        if (isCurrent &&
+            (val->isTrusted () || pubKey))
         {
             ScopedLockType sl (mLock);
 
-            if (!findCreateSet (hash)->insert (std::make_pair (node, val)).second)
+            if (!findCreateSet (hash)->insert (
+                    std::make_pair (*pubKey, val)).second)
                 return false;
 
-            auto it = mCurrentValidations.find (node);
+            auto it = mCurrentValidations.find (*pubKey);
 
             if (it == mCurrentValidations.end ())
             {
                 // No previous validation from this validator
-                mCurrentValidations.emplace (node, val);
+                mCurrentValidations.emplace (*pubKey, val);
             }
             else if (!it->second)
             {
                 // Previous validation has expired
                 it->second = val;
             }
-            else if (val->getSignTime () > it->second->getSignTime ())
+            else
             {
-                // This is a newer validation
+                auto const oldSeq = (*it->second)[~sfLedgerSequence];
+                auto const newSeq = (*val)[~sfLedgerSequence];
+
+                if (oldSeq && newSeq && *oldSeq == *newSeq)
+                {
+                    JLOG (j_.warn()) <<
+                        "Trusted node " <<
+                        toBase58 (TokenType::TOKEN_NODE_PUBLIC, *pubKey) <<
+                        " published multiple validations for ledger " <<
+                        *oldSeq;
+
+                    // Remove current validation for the revoked signing key
+                    if (signer != it->second->getSignerPublic())
+                    {
+                        auto set = findSet (it->second->getLedgerHash ());
+                        if (set)
+                            set->erase (*pubKey);
+                    }
+                }
+
+                if (val->getSignTime () > it->second->getSignTime () ||
+                    signer != it->second->getSignerPublic())
+                {
+                    // This is either a newer validation or a new signing key
                     val->setPreviousHash (it->second->getLedgerHash ());
                     mStaleValidations.push_back (it->second);
                     it->second = val;
@@ -139,6 +167,7 @@ private:
                     isCurrent = false;
                 }
             }
+        }
 
         JLOG (j_.debug()) <<
             "Val for " << hash <<
@@ -185,9 +214,10 @@ private:
                 (val->getSeenTime() < (now + VALIDATION_VALID_LOCAL)));
     }
 
-    int getTrustedValidationCount (uint256 const& ledger) override
+    std::size_t
+    getTrustedValidationCount (uint256 const& ledger) override
     {
-        int trusted = 0;
+        std::size_t trusted = 0;
         ScopedLockType sl (mLock);
         auto set = findSet (ledger);
 
@@ -292,6 +322,37 @@ private:
         return ret;
     }
 
+    hash_set<PublicKey> getCurrentPublicKeys () override
+    {
+        hash_set<PublicKey> ret;
+
+        ScopedLockType sl (mLock);
+        auto it = mCurrentValidations.begin ();
+
+        while (it != mCurrentValidations.end ())
+        {
+            if (!it->second) // contains no record
+                it = mCurrentValidations.erase (it);
+            else if (! current (it->second))
+            {
+                // contains a stale record
+                mStaleValidations.push_back (it->second);
+                it->second.reset ();
+                condWrite ();
+                it = mCurrentValidations.erase (it);
+            }
+            else
+            {
+                // contains a live record
+                ret.insert (it->first);
+
+                ++it;
+            }
+        }
+
+        return ret;
+    }
+
     LedgerToValidationCounter getCurrentValidations (
         uint256 currentLedger,
         uint256 priorLedger,
@@ -317,6 +378,8 @@ private:
                 condWrite ();
                 it = mCurrentValidations.erase (it);
             }
+            else if (! it->second->isTrusted())
+                ++it;
             else if (! it->second->isFieldPresent (sfLedgerSequence) ||
                 (it->second->getFieldU32 (sfLedgerSequence) >= cutoffBefore))
             {

src/ripple/app/misc/Validations.h

@@ -31,7 +31,7 @@ namespace ripple {
 // VFALCO TODO rename and move these type aliases into the Validations interface
 
 // nodes validating and highest node ID validating
-using ValidationSet = hash_map<NodeID, STValidation::pointer>;
+using ValidationSet = hash_map<PublicKey, STValidation::pointer>;
 
 using ValidationCounter = std::pair<int, NodeID>;
 using LedgerToValidationCounter = hash_map<uint256, ValidationCounter>;
@@ -47,7 +47,7 @@ public:
 
     virtual ValidationSet getValidations (uint256 const& ledger) = 0;
 
-    virtual int getTrustedValidationCount (uint256 const& ledger) = 0;
+    virtual std::size_t getTrustedValidationCount (uint256 const& ledger) = 0;
 
     /** Returns fees reported by trusted validators in the given ledger. */
     virtual
@@ -57,6 +57,8 @@ public:
     virtual int getNodesAfter (uint256 const& ledger) = 0;
     virtual int getLoadRatio (bool overLoaded) = 0;
 
+    virtual hash_set<PublicKey> getCurrentPublicKeys () = 0;
+
     // VFALCO TODO make a type alias for this ugly return value!
     virtual LedgerToValidationCounter getCurrentValidations (
         uint256 currentLedger, uint256 previousLedger,

src/ripple/app/misc/ValidatorList.h

@@ -20,102 +20,437 @@
 #ifndef RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED
 #define RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED
 
-#include <ripple/basics/BasicConfig.h>
+#include <ripple/app/misc/Manifest.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/UnorderedContainers.h>
+#include <ripple/core/TimeKeeper.h>
+#include <ripple/crypto/csprng.h>
 #include <ripple/protocol/PublicKey.h>
-#include <boost/optional.hpp>
+#include <boost/iterator/counting_iterator.hpp>
-#include <functional>
+#include <boost/range/adaptors.hpp>
-#include <memory>
+#include <boost/thread/locks.hpp>
+#include <boost/thread/shared_mutex.hpp>
 #include <mutex>
+#include <numeric>
 
 namespace ripple {
 
+enum class ListDisposition
+{
+    /// List is valid
+    accepted = 0,
+
+    /// List version is not supported
+    unsupported_version,
+
+    /// List signed by untrusted publisher key
+    untrusted,
+
+    /// Trusted publisher key, but seq is too old
+    stale,
+
+    /// Invalid format or signature
+    invalid,
+};
+
+/**
+    Trusted Validators List
+    -----------------------
+
+    Rippled accepts ledger proposals and validations from trusted validator
+    nodes. A ledger is considered fully-validated once the number of received
+    trusted validations for a ledger meets or exceeds a quorum value.
+
+    This class manages the set of validation public keys the local rippled node
+    trusts. The list of trusted keys is populated using the keys listed in the
+    configuration file as well as lists signed by trusted publishers. The
+    trusted publisher public keys are specified in the config.
+
+    New lists are expected to include the following data:
+
+    @li @c "blob": Base64-encoded JSON string containing a @c "sequence", @c
+        "expiration", and @c "validators" field. @c "expiration" contains the
+        Ripple timestamp (seconds since January 1st, 2000 (00:00 UTC)) for when
+        the list expires. @c "validators" contains an array of objects with a
+        @c "validation_public_key" field.
+        @c "validation_public_key" should be the hex-encoded master public key.
+
+    @li @c "manifest": Base64-encoded serialization of a manifest containing the
+        publisher's master and signing public keys.
+
+    @li @c "signature": Hex-encoded signature of the blob using the publisher's
+        signing key.
+
+    @li @c "version": 1
+
+    Individual validator lists are stored separately by publisher. The number of
+    lists on which a validator's public key appears is also tracked.
+
+    The list of trusted validation public keys is reset at the start of each
+    consensus round to take into account the latest known lists as well as the
+    set of validators from whom validations are being received. Listed
+    validation public keys are shuffled and then sorted by the number of lists
+    they appear on. (The shuffling makes the order/rank of validators with the
+    same number of listings non-deterministic.) A quorum value is calculated for
+    the new trusted validator list. If there is only one list, all listed keys
+    are trusted. Otherwise, the trusted list size is set to 125% of the quorum.
+*/
 class ValidatorList
 {
-private:
+    struct PublisherList
-    /** The non-ephemeral public keys from the configuration file. */
+    {
-    hash_map<PublicKey, std::string> permanent_;
+        bool available;
+        std::vector<PublicKey> list;
+        std::size_t sequence;
+        std::size_t expiration;
+    };
 
-    /** The ephemeral public keys from manifests. */
+    ManifestCache& validatorManifests_;
-    hash_map<PublicKey, std::string> ephemeral_;
+    ManifestCache& publisherManifests_;
+    TimeKeeper& timeKeeper_;
+    beast::Journal j_;
+    boost::shared_mutex mutable mutex_;
 
-    std::mutex mutable mutex_;
+    std::atomic<std::size_t> quorum_;
-    beast::Journal mutable j_;
+    boost::optional<std::size_t> minimumQuorum_;
+
+    // Published lists stored by publisher master public key
+    hash_map<PublicKey, PublisherList> publisherLists_;
+
+    // Listed master public keys with the number of lists they appear on
+    hash_map<PublicKey, std::size_t> keyListings_;
+
+    // The current list of trusted master keys
+    hash_set<PublicKey> trustedKeys_;
+
+    PublicKey localPubKey_;
 
 public:
-    explicit
+    ValidatorList (
-    ValidatorList (beast::Journal j);
+        ManifestCache& validatorManifests,
+        ManifestCache& publisherManifests,
+        TimeKeeper& timeKeeper,
+        beast::Journal j,
+        boost::optional<std::size_t> minimumQuorum = boost::none);
+    ~ValidatorList ();
 
-    /** Determines whether a node is in the UNL
+    /** Load configured trusted keys.
-        @return boost::none if the node isn't a member,
+
-                otherwise, the comment associated with the
+        @param localSigningKey This node's validation public key
-                node (which may be an empty string).
+
+        @param configKeys List of trusted keys from config. Each entry consists
+        of a base58 encoded validation public key, optionally followed by a
+        comment.
+
+        @param publisherKeys List of trusted publisher public keys. Each entry
+        contains a base58 encoded account public key.
+
+        @par Thread Safety
+
+        May be called concurrently
+
+        @return `false` if an entry is invalid or unparsable
     */
-    boost::optional<std::string>
+    bool
-    member (
+    load (
-        PublicKey const& identity) const;
+        PublicKey const& localSigningKey,
+        std::vector<std::string> const& configKeys,
+        std::vector<std::string> const& publisherKeys);
 
-    /** Determines whether a node is in the UNL */
+    /** Apply published list of public keys
+
+        @param manifest base64-encoded publisher key manifest
+
+        @param blob base64-encoded json containing published validator list
+
+        @param signature Signature of the decoded blob
+
+        @param version Version of published list format
+
+        @return `ListDisposition::accepted` if list was successfully applied
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    ListDisposition
+    applyList (
+        std::string const& manifest,
+        std::string const& blob,
+        std::string const& signature,
+        std::uint32_t version);
+
+    /** Update trusted keys
+
+        Reset the trusted keys based on latest manifests, received validations,
+        and lists.
+
+        @param seenValidators Set of public keys used to sign recently
+        received validations
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    template<class KeySet>
+    void
+    onConsensusStart (
+        KeySet const& seenValidators);
+
+    /** Get quorum value for current trusted key set
+
+        The quorum is the minimum number of validations needed for a ledger to
+        be fully validated. It can change when the set of trusted validation
+        keys is updated (at the start of each consensus round) and primarily
+        depends on the number of trusted keys.
+
+        @par Thread Safety
+
+        May be called concurrently
+
+        @return quorum value
+    */
+    std::size_t
+    quorum () const
+    {
+        return quorum_;
+    };
+
+    /** Returns `true` if public key is trusted
+
+        @param identity Validation public key
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     bool
     trusted (
         PublicKey const& identity) const;
 
-    /** Insert a short-term validator key published in a manifest. */
+    /** Returns `true` if public key is included on any lists
+
+        @param identity Validation public key
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     bool
-    insertEphemeralKey (
+    listed (
-        PublicKey const& identity,
+        PublicKey const& identity) const;
-        std::string const& comment);
 
-    /** Remove a short-term validator revoked in a manifest. */
+    /** Returns master public key if public key is trusted
+
+        @param identity Validation public key
+
+        @return `boost::none` if key is not trusted
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    boost::optional<PublicKey>
+    getTrustedKey (
+        PublicKey const& identity) const;
+
+    /** Returns listed master public if public key is included on any lists
+
+        @param identity Validation public key
+
+        @return `boost::none` if key is not listed
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    boost::optional<PublicKey>
+    getListedKey (
+        PublicKey const& identity) const;
+
+    /** Returns `true` if public key is a trusted publisher
+
+        @param identity Publisher public key
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
     bool
-    removeEphemeralKey (
+    trustedPublisher (
-        PublicKey const& identity);
+        PublicKey const& identity) const;
 
-    /** Insert a long-term validator key. */
+    /** Invokes the callback once for every listed validation public key.
-    bool
-    insertPermanentKey (
-        PublicKey const& identity,
-        std::string const& comment);
 
-    /** Remove a long-term validator key. */
+        @note Undefined behavior results when calling ValidatorList members from
-    bool
+        within the callback
-    removePermanentKey (
-        PublicKey const& identity);
-
-    /** The number of installed permanent and ephemeral keys */
-    std::size_t
-    size () const;
-
-    /** Invokes the callback once for every node in the UNL.
-        @note You are not allowed to insert or remove any
-              nodes in the UNL from within the callback.
 
         The arguments passed into the lambda are:
-            - The public key of the validator;
+
-            - A (possibly empty) comment.
+        @li The validation public key
-            - A boolean indicating whether this is a
+
-              permanent or ephemeral key;
+        @li A boolean indicating whether this is a trusted key
+
+        @par Thread Safety
+
+        May be called concurrently
     */
     void
-    for_each (
+    for_each_listed (
-        std::function<void(PublicKey const&, std::string const&, bool)> func) const;
+        std::function<void(PublicKey const&, bool)> func) const;
 
-    /** Load the list of trusted validators.
+private:
+    /** Check response for trusted valid published list
 
-        The section contains entries consisting of a base58
+        @return `ListDisposition::accepted` if list can be applied
-        encoded validator public key, optionally followed by
-        a comment.
 
-        @return false if an entry could not be parsed or
+        @par Thread Safety
-                contained an invalid validator public key,
+
-                true otherwise.
+        Calling public member function is expected to lock mutex
+    */
+    ListDisposition
+    verify (
+        Json::Value& list,
+        PublicKey& pubKey,
+        std::string const& manifest,
+        std::string const& blob,
+        std::string const& signature);
+
+    /** Stop trusting publisher's list of keys.
+
+        @param publisherKey Publisher public key
+
+        @return `false` if key was not trusted
+
+        @par Thread Safety
+
+        Calling public member function is expected to lock mutex
     */
     bool
-    load (
+    removePublisherList (PublicKey const& publisherKey);
-        Section const& validators);
+
 };
 
+//------------------------------------------------------------------------------
+
+template<class KeySet>
+void
+ValidatorList::onConsensusStart (
+    KeySet const& seenValidators)
+{
+    boost::unique_lock<boost::shared_mutex> lock{mutex_};
+
+    // Check that lists from all configured publishers are available
+    bool allListsAvailable = true;
+
+    for (auto const& list : publisherLists_)
+    {
+        // Remove any expired published lists
+        if (list.second.expiration &&
+                list.second.expiration <=
+                timeKeeper_.now().time_since_epoch().count())
+            removePublisherList (list.first);
+        else if (! list.second.available)
+            allListsAvailable = false;
+    }
+
+    std::multimap<std::size_t, PublicKey> rankedKeys;
+    bool localKeyListed = false;
+
+    // "Iterate" the listed keys in random order so that the rank of multiple
+    // keys with the same number of listings is not deterministic
+    std::vector<std::size_t> indexes (keyListings_.size());
+    std::iota (indexes.begin(), indexes.end(), 0);
+    std::shuffle (indexes.begin(), indexes.end(), crypto_prng());
+
+    for (auto const& index : indexes)
+    {
+        auto const& val = std::next (keyListings_.begin(), index);
+
+        if (validatorManifests_.revoked (val->first))
+            continue;
+
+        if (val->first == localPubKey_)
+        {
+            localKeyListed = val->second > 1;
+            rankedKeys.insert (
+                std::pair<std::size_t,PublicKey>(
+                    std::numeric_limits<std::size_t>::max(), localPubKey_));
+        }
+        // If no validations are being received, use all validators.
+        // Otherwise, do not use validators whose validations aren't being received
+        else if (seenValidators.empty() ||
+            seenValidators.find (val->first) != seenValidators.end ())
+        {
+            rankedKeys.insert (
+                std::pair<std::size_t,PublicKey>(val->second, val->first));
+        }
+    }
+
+    // This quorum guarantees sufficient overlap with the trusted sets of other
+    // nodes using the same set of published lists.
+    std::size_t quorum = keyListings_.size()/2 + 1;
+
+    // Increment the quorum to prevent two unlisted validators using the same
+    // even number of listed validators from forking.
+    if (localPubKey_.size() && ! localKeyListed &&
+            rankedKeys.size () > 1 && keyListings_.size () % 2 != 0)
+        ++quorum;
+
+    JLOG (j_.debug()) <<
+        rankedKeys.size() << "  of " << keyListings_.size() <<
+        " listed validators eligible for inclusion in the trusted set";
+
+    auto size = rankedKeys.size();
+
+    // Use all eligible keys if there is only one trusted list.
+    if (publisherLists_.size() == 1)
+    {
+        // Raise the quorum to 80% of the trusted set
+        std::size_t const targetQuorum = std::ceil (size * 0.8);
+        if (targetQuorum > quorum)
+            quorum = targetQuorum;
+    }
+    else
+    {
+        // reduce the trusted set size so that the quorum represents
+        // at least 80%
+        size = quorum * 1.25;
+    }
+
+    if (minimumQuorum_ && (seenValidators.empty() ||
+            rankedKeys.size() < quorum))
+        quorum = *minimumQuorum_;
+
+    // Do not use achievable quorum until lists from all configured
+    // publishers are available
+    else if (! allListsAvailable)
+        quorum = std::numeric_limits<std::size_t>::max();
+
+    trustedKeys_.clear();
+    quorum_ = quorum;
+
+    for (auto const& val : boost::adaptors::reverse (rankedKeys))
+    {
+        if (size <= trustedKeys_.size())
+            break;
+
+        trustedKeys_.insert (val.second);
+    }
+
+    JLOG (j_.debug()) <<
+        "Using quorum of " << quorum_ << " for new set of " <<
+        trustedKeys_.size() << " trusted validators";
+
+    if (trustedKeys_.size() < quorum_)
+    {
+        JLOG (j_.warn()) <<
+            "New quorum of " << quorum_ <<
+            " exceeds the number of trusted validators (" <<
+            trustedKeys_.size() << ")";
+    }
 }
 
+} // ripple
+
 #endif

src/ripple/app/misc/impl/Manifest.cpp

@@ -0,0 +1,373 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/misc/Manifest.h>
+#include <ripple/basics/contract.h>
+#include <ripple/basics/Log.h>
+#include <ripple/beast/rfc2616.h>
+#include <ripple/core/DatabaseCon.h>
+#include <ripple/protocol/PublicKey.h>
+#include <ripple/protocol/Sign.h>
+#include <beast/core/detail/base64.hpp>
+#include <stdexcept>
+
+namespace ripple {
+
+boost::optional<Manifest>
+Manifest::make_Manifest (std::string s)
+{
+    try
+    {
+        STObject st (sfGeneric);
+        SerialIter sit (s.data (), s.size ());
+        st.set (sit);
+        auto const opt_pk = get<PublicKey>(st, sfPublicKey);
+        auto const opt_spk = get<PublicKey>(st, sfSigningPubKey);
+        auto const opt_seq = get (st, sfSequence);
+        auto const opt_sig = get (st, sfSignature);
+        auto const opt_msig = get (st, sfMasterSignature);
+        if (!opt_pk || !opt_spk || !opt_seq || !opt_sig || !opt_msig)
+        {
+            return boost::none;
+        }
+        return Manifest (std::move (s), *opt_pk, *opt_spk, *opt_seq);
+    }
+    catch (std::exception const&)
+    {
+        return boost::none;
+    }
+}
+
+template<class Stream>
+Stream&
+logMftAct (
+    Stream& s,
+    std::string const& action,
+    PublicKey const& pk,
+    std::uint32_t seq)
+{
+    s << "Manifest: " << action <<
+         ";Pk: " << toBase58 (TokenType::TOKEN_NODE_PUBLIC, pk) <<
+         ";Seq: " << seq << ";";
+    return s;
+}
+
+template<class Stream>
+Stream& logMftAct (
+    Stream& s,
+    std::string const& action,
+    PublicKey const& pk,
+    std::uint32_t seq,
+    std::uint32_t oldSeq)
+{
+    s << "Manifest: " << action <<
+         ";Pk: " << toBase58 (TokenType::TOKEN_NODE_PUBLIC, pk) <<
+         ";Seq: " << seq <<
+         ";OldSeq: " << oldSeq << ";";
+    return s;
+}
+
+Manifest::Manifest (std::string s,
+                    PublicKey pk,
+                    PublicKey spk,
+                    std::uint32_t seq)
+    : serialized (std::move (s))
+    , masterKey (std::move (pk))
+    , signingKey (std::move (spk))
+    , sequence (seq)
+{
+}
+
+bool Manifest::verify () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    if (! ripple::verify (st, HashPrefix::manifest, signingKey))
+        return false;
+
+    return ripple::verify (
+        st, HashPrefix::manifest, masterKey, sfMasterSignature);
+}
+
+uint256 Manifest::hash () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    return st.getHash (HashPrefix::manifest);
+}
+
+bool Manifest::revoked () const
+{
+    /*
+        The maximum possible sequence number means that the master key
+        has been revoked.
+    */
+    return sequence == std::numeric_limits<std::uint32_t>::max ();
+}
+
+Blob Manifest::getSignature () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    return st.getFieldVL (sfSignature);
+}
+
+Blob Manifest::getMasterSignature () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    return st.getFieldVL (sfMasterSignature);
+}
+
+PublicKey
+ManifestCache::getSigningKey (PublicKey const& pk) const
+{
+    std::lock_guard<std::mutex> lock{read_mutex_};
+    auto const iter = map_.find (pk);
+
+    if (iter != map_.end () && !iter->second.revoked ())
+        return iter->second.signingKey;
+
+    return pk;
+}
+
+PublicKey
+ManifestCache::getMasterKey (PublicKey const& pk) const
+{
+    std::lock_guard<std::mutex> lock{read_mutex_};
+    auto const iter = signingToMasterKeys_.find (pk);
+
+    if (iter != signingToMasterKeys_.end ())
+        return iter->second;
+
+    return pk;
+}
+
+bool
+ManifestCache::revoked (PublicKey const& pk) const
+{
+    std::lock_guard<std::mutex> lock{read_mutex_};
+    auto const iter = map_.find (pk);
+
+    if (iter != map_.end ())
+        return iter->second.revoked ();
+
+    return false;
+}
+
+ManifestDisposition
+ManifestCache::applyManifest (Manifest m)
+{
+    std::lock_guard<std::mutex> applyLock{apply_mutex_};
+
+    /*
+        before we spend time checking the signature, make sure the
+        sequence number is newer than any we have.
+    */
+    auto const iter = map_.find(m.masterKey);
+
+    if (iter != map_.end() &&
+        m.sequence <= iter->second.sequence)
+    {
+        /*
+            A manifest was received for a validator we're tracking, but
+            its sequence number is not higher than the one already stored.
+            This will happen normally when a peer without the latest gossip
+            connects.
+        */
+        if (auto stream = j_.debug())
+            logMftAct(stream, "Stale", m.masterKey, m.sequence, iter->second.sequence);
+        return ManifestDisposition::stale;  // not a newer manifest, ignore
+    }
+
+    if (! m.verify())
+    {
+        /*
+          A manifest's signature is invalid.
+          This shouldn't happen normally.
+        */
+        if (auto stream = j_.warn())
+            logMftAct(stream, "Invalid", m.masterKey, m.sequence);
+        return ManifestDisposition::invalid;
+    }
+
+    std::lock_guard<std::mutex> readLock{read_mutex_};
+
+    bool const revoked = m.revoked();
+
+    if (iter == map_.end ())
+    {
+        /*
+            This is the first received manifest for a trusted master key
+            (possibly our own).  This only happens once per validator per
+            run.
+        */
+        if (auto stream = j_.info())
+            logMftAct(stream, "AcceptedNew", m.masterKey, m.sequence);
+
+        if (! revoked)
+            signingToMasterKeys_[m.signingKey] = m.masterKey;
+
+        map_.emplace (std::make_pair(m.masterKey, std::move (m)));
+    }
+    else
+    {
+        /*
+            An ephemeral key was revoked and superseded by a new key.
+            This is expected, but should happen infrequently.
+        */
+        if (auto stream = j_.info())
+            logMftAct(stream, "AcceptedUpdate",
+                      m.masterKey, m.sequence, iter->second.sequence);
+
+        signingToMasterKeys_.erase (iter->second.signingKey);
+
+        if (! revoked)
+            signingToMasterKeys_[m.signingKey] = m.masterKey;
+
+        iter->second = std::move (m);
+    }
+
+    if (revoked)
+    {
+        /*
+            A validator master key has been compromised, so its manifests
+            are now untrustworthy.  In order to prevent us from accepting
+            a forged manifest signed by the compromised master key, store
+            this manifest, which has the highest possible sequence number
+            and therefore can't be superseded by a forged one.
+        */
+        if (auto stream = j_.warn())
+            logMftAct(stream, "Revoked", m.masterKey, m.sequence);
+    }
+
+    return ManifestDisposition::accepted;
+}
+
+void
+ManifestCache::load (
+    DatabaseCon& dbCon, std::string const& dbTable)
+{
+    // Load manifests stored in database
+    std::string const sql =
+        "SELECT RawData FROM " + dbTable + ";";
+    auto db = dbCon.checkoutDb ();
+    soci::blob sociRawData (*db);
+    soci::statement st =
+        (db->prepare << sql,
+             soci::into (sociRawData));
+    st.execute ();
+    while (st.fetch ())
+    {
+        std::string serialized;
+        convert (sociRawData, serialized);
+        if (auto mo = Manifest::make_Manifest (std::move (serialized)))
+        {
+            if (!mo->verify())
+            {
+                JLOG(j_.warn())
+                    << "Unverifiable manifest in db";
+                continue;
+            }
+
+            applyManifest (std::move(*mo));
+        }
+        else
+        {
+            JLOG(j_.warn())
+                << "Malformed manifest in database";
+        }
+    }
+}
+
+bool
+ManifestCache::load (
+    DatabaseCon& dbCon, std::string const& dbTable,
+    std::vector<std::string> const& configManifest)
+{
+    load (dbCon, dbTable);
+
+    if (! configManifest.empty())
+    {
+        std::string s;
+        s.reserve (Manifest::textLength);
+        for (auto const& line : configManifest)
+            s += beast::rfc2616::trim(line);
+
+        auto mo = Manifest::make_Manifest (beast::detail::base64_decode(s));
+        if (! mo)
+        {
+            JLOG (j_.error()) << "Malformed manifest in config";
+            return false;
+        }
+
+        if (mo->revoked())
+        {
+            JLOG (j_.warn()) <<
+                "Configured manifest revokes public key";
+        }
+
+        if (applyManifest (std::move(*mo)) ==
+            ManifestDisposition::invalid)
+        {
+            JLOG (j_.error()) << "Manifest in config was rejected";
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void ManifestCache::save (
+    DatabaseCon& dbCon, std::string const& dbTable,
+    std::function <bool (PublicKey const&)> isTrusted)
+{
+    std::lock_guard<std::mutex> lock{apply_mutex_};
+
+    auto db = dbCon.checkoutDb ();
+
+    soci::transaction tr(*db);
+    *db << "DELETE FROM " << dbTable;
+    std::string const sql =
+        "INSERT INTO " + dbTable + " (RawData) VALUES (:rawData);";
+    for (auto const& v : map_)
+    {
+        if (! isTrusted (v.second.masterKey))
+        {
+
+            JLOG(j_.info())
+               << "Untrusted manifest in cache not saved to db";
+            continue;
+        }
+
+        // soci does not support bulk insertion of blob data
+        // Do not reuse blob because manifest ecdsa signatures vary in length
+        // but blob write length is expected to be >= the last write
+        soci::blob rawData(*db);
+        convert (v.second.serialized, rawData);
+        *db << sql,
+            soci::use (rawData);
+    }
+    tr.commit ();
+}
+}

src/ripple/app/misc/impl/ValidatorList.cpp

@@ -17,117 +17,39 @@
 */
 //==============================================================================
 
-#include <BeastConfig.h>
 #include <ripple/app/misc/ValidatorList.h>
 #include <ripple/basics/Slice.h>
 #include <ripple/basics/StringUtilities.h>
+#include <ripple/json/json_reader.h>
+#include <beast/core/detail/base64.hpp>
 #include <boost/regex.hpp>
 
 namespace ripple {
 
-ValidatorList::ValidatorList (beast::Journal j)
+ValidatorList::ValidatorList (
-    : j_ (j)
+    ManifestCache& validatorManifests,
+    ManifestCache& publisherManifests,
+    TimeKeeper& timeKeeper,
+    beast::Journal j,
+    boost::optional<std::size_t> minimumQuorum)
+    : validatorManifests_ (validatorManifests)
+    , publisherManifests_ (publisherManifests)
+    , timeKeeper_ (timeKeeper)
+    , j_ (j)
+    , quorum_ (minimumQuorum ? *minimumQuorum : 1) // Genesis ledger quorum
+    , minimumQuorum_ (minimumQuorum)
 {
 }
 
-boost::optional<std::string>
+ValidatorList::~ValidatorList()
-ValidatorList::member (PublicKey const& identity) const
 {
-    std::lock_guard <std::mutex> sl (mutex_);
-
-    auto ret = ephemeral_.find (identity);
-
-    if (ret != ephemeral_.end())
-        return ret->second;
-
-    ret = permanent_.find (identity);
-
-    if (ret != permanent_.end())
-        return ret->second;
-
-    return boost::none;
-}
-
-bool
-ValidatorList::trusted (PublicKey const& identity) const
-{
-    return static_cast<bool> (member(identity));
-}
-
-bool
-ValidatorList::insertEphemeralKey (
-    PublicKey const& identity,
-    std::string const& comment)
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-
-    if (permanent_.find (identity) != permanent_.end())
-    {
-        JLOG (j_.error()) <<
-            toBase58 (TokenType::TOKEN_NODE_PUBLIC, identity) <<
-            ": ephemeral key exists in permanent table!";
-        return false;
-    }
-
-    return ephemeral_.emplace (identity, comment).second;
-}
-
-bool
-ValidatorList::removeEphemeralKey (
-    PublicKey const& identity)
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-    return ephemeral_.erase (identity);
-}
-
-bool
-ValidatorList::insertPermanentKey (
-    PublicKey const& identity,
-    std::string const& comment)
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-
-    if (ephemeral_.find (identity) != ephemeral_.end())
-    {
-        JLOG (j_.error()) <<
-            toBase58 (TokenType::TOKEN_NODE_PUBLIC, identity) <<
-            ": permanent key exists in ephemeral table!";
-        return false;
-    }
-
-    return permanent_.emplace (identity, comment).second;
-}
-
-bool
-ValidatorList::removePermanentKey (
-    PublicKey const& identity)
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-    return permanent_.erase (identity);
-}
-
-std::size_t
-ValidatorList::size () const
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-    return permanent_.size () + ephemeral_.size ();
-}
-
-void
-ValidatorList::for_each (
-    std::function<void(PublicKey const&, std::string const&, bool)> func) const
-{
-    std::lock_guard <std::mutex> sl (mutex_);
-
-    for (auto const& v : permanent_)
-        func (v.first, v.second, false);
-    for (auto const& v : ephemeral_)
-        func (v.first, v.second, true);
 }
 
 bool
 ValidatorList::load (
-    Section const& validators)
+    PublicKey const& localSigningKey,
+    std::vector<std::string> const& configKeys,
+    std::vector<std::string> const& publisherKeys)
 {
     static boost::regex const re (
         "[[:space:]]*"            // skip leading whitespace
@@ -141,12 +63,61 @@ ValidatorList::load (
         ")?"                      // end optional comment block
     );
 
+    boost::unique_lock<boost::shared_mutex> read_lock{mutex_};
+
     JLOG (j_.debug()) <<
-        "Loading configured validators";
+        "Loading configured trusted validator list publisher keys";
 
     std::size_t count = 0;
+    for (auto key : publisherKeys)
+    {
+        JLOG (j_.trace()) <<
+            "Processing '" << key << "'";
 
-    for (auto const& n : validators.values ())
+        auto const ret = strUnHex (key);
+
+        if (! ret.second || ! ret.first.size ())
+        {
+            JLOG (j_.error()) <<
+                "Invalid validator list publisher key: " << key;
+            return false;
+        }
+
+        auto id = PublicKey(Slice{ ret.first.data (), ret.first.size() });
+
+        if (validatorManifests_.revoked (id))
+        {
+            JLOG (j_.warn()) <<
+                "Configured validator list publisher key is revoked: " << key;
+            continue;
+        }
+
+        if (publisherLists_.count(id))
+        {
+            JLOG (j_.warn()) <<
+                "Duplicate validator list publisher key: " << key;
+            continue;
+        }
+
+        publisherLists_[id].available = false;
+        ++count;
+    }
+
+    JLOG (j_.debug()) <<
+        "Loaded " << count << " keys";
+
+    localPubKey_ = validatorManifests_.getMasterKey (localSigningKey);
+
+    // Treat local validator key as though it was listed in the config
+    if (localPubKey_.size())
+        keyListings_.insert ({ localPubKey_, 1 });
+
+    JLOG (j_.debug()) <<
+        "Loading configured validator keys";
+
+    count = 0;
+    PublicKey local;
+    for (auto const& n : configKeys)
     {
         JLOG (j_.trace()) <<
             "Processing '" << n << "'";
@@ -165,19 +136,22 @@ ValidatorList::load (
 
         if (!id)
         {
-            JLOG (j_.error()) <<
+            JLOG (j_.error()) << "Invalid node identity: " << match[1];
-                "Invalid node identity: " << match[1];
             return false;
         }
 
-        if (trusted (*id))
+        // Skip local key which was already added
+        if (*id == localPubKey_ || *id == localSigningKey)
+            continue;
+
+        auto ret = keyListings_.insert ({*id, 1});
+        if (! ret.second)
         {
-            JLOG (j_.warn()) <<
+            JLOG (j_.warn()) << "Duplicate node identity: " << match[1];
-                "Duplicate node identity: " << match[1];
             continue;
         }
-
+        publisherLists_[local].list.emplace_back (std::move(*id));
-        if (insertPermanentKey(*id, trim_whitespace (match[2])))
+        publisherLists_[local].available = true;
         ++count;
     }
 
@@ -187,4 +161,244 @@ ValidatorList::load (
     return true;
 }
 
+
+ListDisposition
+ValidatorList::applyList (
+    std::string const& manifest,
+    std::string const& blob,
+    std::string const& signature,
+    std::uint32_t version)
+{
+    if (version != 1)
+        return ListDisposition::unsupported_version;
+
+    boost::unique_lock<boost::shared_mutex> lock{mutex_};
+
+    Json::Value list;
+    PublicKey pubKey;
+    auto const result = verify (list, pubKey, manifest, blob, signature);
+    if (result != ListDisposition::accepted)
+        return result;
+
+    // Update publisher's list
+    Json::Value const& newList = list["validators"];
+    publisherLists_[pubKey].available = true;
+    publisherLists_[pubKey].sequence = list["sequence"].asUInt ();
+    publisherLists_[pubKey].expiration = list["expiration"].asUInt ();
+    std::vector<PublicKey>& publisherList = publisherLists_[pubKey].list;
+
+    std::vector<PublicKey> oldList = publisherList;
+    publisherList.clear ();
+    publisherList.reserve (newList.size ());
+    for (auto const& val : newList)
+    {
+        if (val.isObject () &&
+            val.isMember ("validation_public_key") &&
+            val["validation_public_key"].isString ())
+        {
+            std::pair<Blob, bool> ret (strUnHex (
+                val["validation_public_key"].asString ()));
+
+            if (! ret.second || ! ret.first.size ())
+            {
+                JLOG (j_.error()) <<
+                    "Invalid node identity: " <<
+                    val["validation_public_key"].asString ();
+            }
+            else
+            {
+                publisherList.push_back (
+                    PublicKey(Slice{ ret.first.data (), ret.first.size() }));
+            }
+        }
+    }
+
+    // Update keyListings_ for added and removed keys
+    std::sort (
+        publisherList.begin (),
+        publisherList.end ());
+
+    auto iNew = publisherList.begin ();
+    auto iOld = oldList.begin ();
+    while (iNew != publisherList.end () ||
+        iOld != oldList.end ())
+    {
+        if (iOld == oldList.end () ||
+            (iNew != publisherList.end () &&
+            *iNew < *iOld))
+        {
+            // Increment list count for added keys
+            ++keyListings_[*iNew];
+            ++iNew;
+        }
+        else if (iNew == publisherList.end () ||
+            (iOld != oldList.end () && *iOld < *iNew))
+        {
+            // Decrement list count for removed keys
+            if (keyListings_[*iOld] == 1)
+                keyListings_.erase (*iOld);
+            else
+                --keyListings_[*iOld];
+            ++iOld;
+        }
+        else
+        {
+            ++iNew;
+            ++iOld;
+        }
+    }
+
+    if (publisherList.empty())
+    {
+        JLOG (j_.warn()) <<
+            "No validator keys included in valid list";
+    }
+
+    return ListDisposition::accepted;
 }
+
+ListDisposition
+ValidatorList::verify (
+    Json::Value& list,
+    PublicKey& pubKey,
+    std::string const& manifest,
+    std::string const& blob,
+    std::string const& signature)
+{
+    auto m = Manifest::make_Manifest (beast::detail::base64_decode(manifest));
+
+    if (! m || ! publisherLists_.count (m->masterKey))
+        return ListDisposition::untrusted;
+
+    pubKey = m->masterKey;
+    auto const revoked = m->revoked();
+
+    auto const result = publisherManifests_.applyManifest (
+        std::move(*m));
+
+    if (revoked && result == ManifestDisposition::accepted)
+    {
+        removePublisherList (pubKey);
+        publisherLists_.erase (pubKey);
+    }
+
+    if (revoked || result == ManifestDisposition::invalid)
+        return ListDisposition::untrusted;
+
+    auto const sig = strUnHex(signature);
+    auto const data = beast::detail::base64_decode (blob);
+    if (! sig.second ||
+        ! ripple::verify (
+            publisherManifests_.getSigningKey(pubKey),
+            makeSlice(data),
+            makeSlice(sig.first)))
+        return ListDisposition::invalid;
+
+    Json::Reader r;
+    if (! r.parse (data, list))
+        return ListDisposition::invalid;
+
+    if (list.isMember("sequence") && list["sequence"].isInt() &&
+        list.isMember("expiration") && list["expiration"].isInt() &&
+        list.isMember("validators") && list["validators"].isArray())
+    {
+        auto const sequence = list["sequence"].asUInt ();
+        auto const expiration = list["expiration"].asUInt ();
+        if (sequence <= publisherLists_[pubKey].sequence ||
+                expiration <= timeKeeper_.now().time_since_epoch().count())
+            return ListDisposition::stale;
+    }
+    else
+    {
+        return ListDisposition::invalid;
+    }
+
+    return ListDisposition::accepted;
+}
+
+bool
+ValidatorList::listed (
+    PublicKey const& identity) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+
+    auto const pubKey = validatorManifests_.getMasterKey (identity);
+    return keyListings_.find (pubKey) != keyListings_.end ();
+}
+
+bool
+ValidatorList::trusted (PublicKey const& identity) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+
+    auto const pubKey = validatorManifests_.getMasterKey (identity);
+    return trustedKeys_.find (pubKey) != trustedKeys_.end();
+}
+
+boost::optional<PublicKey>
+ValidatorList::getListedKey (
+    PublicKey const& identity) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+
+    auto const pubKey = validatorManifests_.getMasterKey (identity);
+    if (keyListings_.find (pubKey) != keyListings_.end ())
+        return pubKey;
+    return boost::none;
+}
+
+boost::optional<PublicKey>
+ValidatorList::getTrustedKey (PublicKey const& identity) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+
+    auto const pubKey = validatorManifests_.getMasterKey (identity);
+    if (trustedKeys_.find (pubKey) != trustedKeys_.end())
+        return pubKey;
+    return boost::none;
+}
+
+bool
+ValidatorList::trustedPublisher (PublicKey const& identity) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+    return identity.size() && publisherLists_.count (identity);
+}
+
+bool
+ValidatorList::removePublisherList (PublicKey const& publisherKey)
+{
+    auto const iList = publisherLists_.find (publisherKey);
+    if (iList == publisherLists_.end ())
+        return false;
+
+    JLOG (j_.debug()) <<
+        "Removing validator list for revoked publisher " <<
+        toBase58(TokenType::TOKEN_NODE_PUBLIC, publisherKey);
+
+    for (auto const& val : iList->second.list)
+    {
+        auto const& iVal = keyListings_.find (val);
+        if (iVal == keyListings_.end())
+            continue;
+
+        if (iVal->second <= 1)
+            keyListings_.erase (iVal);
+        else
+            --iVal->second;
+    }
+
+    return true;
+}
+
+void
+ValidatorList::for_each_listed (
+    std::function<void(PublicKey const&, bool)> func) const
+{
+    boost::shared_lock<boost::shared_mutex> read_lock{mutex_};
+
+    for (auto const& v : keyListings_)
+        func (v.first, trusted(v.first));
+}
+
+} // ripple

src/ripple/core/Config.h

@@ -138,8 +138,6 @@ public:
 
     // Note: The following parameters do not relate to the UNL or trust at all
     std::size_t                 NETWORK_QUORUM = 0;         // Minimum number of nodes to consider the network present
-    int                         VALIDATION_QUORUM = 1;      // Minimum validations to consider ledger authoritative
-    bool                        LOCK_QUORUM = false;        // Do not raise the quorum
 
     // Peer networking parameters
     bool                        PEER_PRIVATE = false;           // True to ask peers not to relay current IP.
@@ -156,6 +154,7 @@ public:
     // Validation
     PublicKey                   VALIDATION_PUB;
     SecretKey                   VALIDATION_PRIV;
+    boost::optional<std::size_t> VALIDATION_QUORUM;     // Minimum validations to consider ledger authoritative
 
     // Node Identity
     std::string                 NODE_SEED;

src/ripple/core/ConfigSections.h

@@ -60,11 +60,11 @@ struct ConfigSection
 #define SECTION_SSL_VERIFY_FILE         "ssl_verify_file"
 #define SECTION_SSL_VERIFY_DIR          "ssl_verify_dir"
 #define SECTION_VALIDATORS_FILE         "validators_file"
-#define SECTION_VALIDATION_QUORUM       "validation_quorum"
 #define SECTION_VALIDATION_SEED         "validation_seed"
 #define SECTION_WEBSOCKET_PING_FREQ     "websocket_ping_frequency"
-#define SECTION_VALIDATORS              "validators"
 #define SECTION_VALIDATOR_KEYS          "validator_keys"
+#define SECTION_VALIDATOR_LIST_KEYS     "validator_list_keys"
+#define SECTION_VALIDATORS              "validators"
 #define SECTION_VALIDATION_MANIFEST     "validation_manifest"
 #define SECTION_VETO_AMENDMENTS         "veto_amendments"
 

src/ripple/core/Job.h

@@ -48,7 +48,6 @@ enum JobType
     jtUPDATE_PF,     // Update pathfinding requests
     jtTRANSACTION,   // A transaction received from the network
     jtBATCH,         // Apply batched transactions
-    jtUNL,           // A Score or Fetch of the UNL (DEPRECATED)
     jtADVANCE,       // Advance validated/acquired ledgers
     jtPUBLEDGER,     // Publish a fully-accepted ledger
     jtTXN_DATA,      // Fetch a proposed set

src/ripple/core/JobTypes.h

@@ -50,7 +50,6 @@ add(    jtRPC,           "RPC",                     maxLimit, false, 0,     0);
 add(    jtUPDATE_PF,     "updatePaths",             maxLimit, false, 0,     0);
 add(    jtTRANSACTION,   "transaction",             maxLimit, false, 250,   1000);
 add(    jtBATCH,         "batch",                   maxLimit, false, 250,   1000);
-add(    jtUNL,           "unl",                     1,        false, 0,     0);
 add(    jtADVANCE,       "advanceLedger",           maxLimit, false, 0,     0);
 add(    jtPUBLEDGER,     "publishNewLedger",        maxLimit, false, 3000,  4500);
 add(    jtTXN_DATA,      "fetchTxnData",            1,        false, 0,     0);

src/ripple/core/impl/Config.cpp

@@ -378,9 +378,6 @@ void Config::loadFromString (std::string const& fileContents)
     if (getSingleSection (secConfig, SECTION_NETWORK_QUORUM, strTemp, j_))
         NETWORK_QUORUM      = beast::lexicalCastThrow <std::size_t> (strTemp);
 
-    if (getSingleSection (secConfig, SECTION_VALIDATION_QUORUM, strTemp, j_))
-        VALIDATION_QUORUM   = std::max (0, beast::lexicalCastThrow <int> (strTemp));
-
     if (getSingleSection (secConfig, SECTION_FEE_ACCOUNT_RESERVE, strTemp, j_))
         FEE_ACCOUNT_RESERVE = beast::lexicalCastThrow <std::uint64_t> (strTemp);
 
@@ -425,6 +422,12 @@ void Config::loadFromString (std::string const& fileContents)
     if (getSingleSection (secConfig, SECTION_PATH_SEARCH_MAX, strTemp, j_))
         PATH_SEARCH_MAX     = beast::lexicalCastThrow <int> (strTemp);
 
+    if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_))
+        DEBUG_LOGFILE       = strTemp;
+
+    // Do not load trusted validator configuration for standalone mode
+    if (! RUN_STANDALONE)
+    {
         // If a file was explicitly specified, then throw if the
         // path is malformed or if the file does not exist or is
         // not a file.
@@ -500,31 +503,27 @@ void Config::loadFromString (std::string const& fileContents)
             if (valKeyEntries)
                 section (SECTION_VALIDATOR_KEYS).append (*valKeyEntries);
 
-        if (!entries && !valKeyEntries)
+            auto valListKeys = getIniFileSection(
+                iniFile,
+                SECTION_VALIDATOR_LIST_KEYS);
+
+            if (valListKeys)
+                section (SECTION_VALIDATOR_LIST_KEYS).append (*valListKeys);
+
+            if (!entries && !valKeyEntries && !valListKeys)
                 Throw<std::runtime_error> (
                     "The file specified in [" SECTION_VALIDATORS_FILE "] "
-                "does not contain a [" SECTION_VALIDATORS "] or "
+                    "does not contain a [" SECTION_VALIDATORS "], "
-                "[" SECTION_VALIDATOR_KEYS "] section: " +
+                    "[" SECTION_VALIDATOR_KEYS "] or "
+                    "[" SECTION_VALIDATOR_LIST_KEYS "]"
+                    " section: " +
                     validatorsFile.string());
-
-        // Look for [validation_quorum] in the validators file
-        // if it was not in the config
-        if (!getIniFileSection (secConfig, SECTION_VALIDATION_QUORUM))
-        {
-            if (!getSingleSection (
-                iniFile, SECTION_VALIDATION_QUORUM, strTemp, j_))
-                Throw<std::runtime_error> (
-                    "The file specified in [" SECTION_VALIDATORS_FILE "] "
-                    "does not contain a [" SECTION_VALIDATION_QUORUM "] "
-                    "section: " + validatorsFile.string());
-            else
-                VALIDATION_QUORUM = std::max (
-                    0, beast::lexicalCastThrow <int> (strTemp));
-        }
         }
 
-    if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_))
+        // Consolidate [validator_keys] and [validators]
-        DEBUG_LOGFILE       = strTemp;
+        section (SECTION_VALIDATORS).append (
+            section (SECTION_VALIDATOR_KEYS).lines ());
+    }
 
     {
         auto const part = section("features");

src/ripple/net/InfoSub.h

@@ -22,7 +22,7 @@
 
 #include <ripple/basics/CountedObject.h>
 #include <ripple/json/json_value.h>
-#include <ripple/overlay/impl/Manifest.h>
+#include <ripple/app/misc/Manifest.h>
 #include <ripple/resource/Consumer.h>
 #include <ripple/protocol/Book.h>
 #include <ripple/core/Stoppable.h>

src/ripple/overlay/Overlay.h

@@ -38,9 +38,6 @@ namespace boost { namespace asio { namespace ssl { class context; } } }
 
 namespace ripple {
 
-class DatabaseCon;
-class BasicConfig;
-
 /** Manages the set of connected peers. */
 class Overlay
     : public Stoppable
@@ -164,15 +161,6 @@ public:
     relay (protocol::TMValidation& m,
         uint256 const& uid) = 0;
 
-    virtual
-    void
-    setupValidatorKeyManifests (BasicConfig const& config,
-                                DatabaseCon& db) = 0;
-
-    virtual
-    void
-    saveValidatorKeyManifests (DatabaseCon& db) const = 0;
-
     /** Visit every active peer and return a value
         The functor must:
         - Be callable as:

src/ripple/overlay/impl/Manifest.cpp

@@ -1,477 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    This file is part of rippled: https://github.com/ripple/rippled
-    Copyright (c) 2012, 2013 Ripple Labs Inc.
-
-    Permission to use, copy, modify, and/or distribute this software for any
-    purpose  with  or without fee is hereby granted, provided that the above
-    copyright notice and this permission notice appear in all copies.
-
-    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
-    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
-    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-//==============================================================================
-
-#include <ripple/app/main/Application.h>
-#include <ripple/basics/contract.h>
-#include <ripple/basics/Log.h>
-#include <ripple/app/misc/ValidatorList.h>
-#include <ripple/core/DatabaseCon.h>
-#include <ripple/overlay/impl/Manifest.h>
-#include <ripple/protocol/PublicKey.h>
-#include <ripple/protocol/Sign.h>
-#include <boost/regex.hpp>
-#include <stdexcept>
-
-namespace ripple {
-
-boost::optional<Manifest>
-make_Manifest (std::string s)
-{
-    try
-    {
-        STObject st (sfGeneric);
-        SerialIter sit (s.data (), s.size ());
-        st.set (sit);
-        auto const opt_pk = get<PublicKey>(st, sfPublicKey);
-        auto const opt_spk = get<PublicKey>(st, sfSigningPubKey);
-        auto const opt_seq = get (st, sfSequence);
-        auto const opt_sig = get (st, sfSignature);
-        auto const opt_msig = get (st, sfMasterSignature);
-        if (!opt_pk || !opt_spk || !opt_seq || !opt_sig || !opt_msig)
-        {
-            return boost::none;
-        }
-        return Manifest (std::move (s), *opt_pk, *opt_spk, *opt_seq);
-    }
-    catch (std::exception const&)
-    {
-        return boost::none;
-    }
-}
-
-template<class Stream>
-Stream&
-logMftAct (
-    Stream& s,
-    std::string const& action,
-    PublicKey const& pk,
-    std::uint32_t seq)
-{
-    s << "Manifest: " << action <<
-         ";Pk: " << toBase58 (TokenType::TOKEN_NODE_PUBLIC, pk) <<
-         ";Seq: " << seq << ";";
-    return s;
-}
-
-template<class Stream>
-Stream& logMftAct (
-    Stream& s,
-    std::string const& action,
-    PublicKey const& pk,
-    std::uint32_t seq,
-    std::uint32_t oldSeq)
-{
-    s << "Manifest: " << action <<
-         ";Pk: " << toBase58 (TokenType::TOKEN_NODE_PUBLIC, pk) <<
-         ";Seq: " << seq <<
-         ";OldSeq: " << oldSeq << ";";
-    return s;
-}
-
-Manifest::Manifest (std::string s,
-                    PublicKey pk,
-                    PublicKey spk,
-                    std::uint32_t seq)
-    : serialized (std::move (s))
-    , masterKey (std::move (pk))
-    , signingKey (std::move (spk))
-    , sequence (seq)
-{
-}
-
-bool Manifest::verify () const
-{
-    STObject st (sfGeneric);
-    SerialIter sit (serialized.data (), serialized.size ());
-    st.set (sit);
-    if (! ripple::verify (st, HashPrefix::manifest, signingKey))
-        return false;
-
-    return ripple::verify (
-        st, HashPrefix::manifest, masterKey, sfMasterSignature);
-}
-
-uint256 Manifest::hash () const
-{
-    STObject st (sfGeneric);
-    SerialIter sit (serialized.data (), serialized.size ());
-    st.set (sit);
-    return st.getHash (HashPrefix::manifest);
-}
-
-bool Manifest::revoked () const
-{
-    /*
-        The maximum possible sequence number means that the master key
-        has been revoked.
-    */
-    return sequence == std::numeric_limits<std::uint32_t>::max ();
-}
-
-Blob Manifest::getSignature () const
-{
-    STObject st (sfGeneric);
-    SerialIter sit (serialized.data (), serialized.size ());
-    st.set (sit);
-    return st.getFieldVL (sfSignature);
-}
-
-Blob Manifest::getMasterSignature () const
-{
-    STObject st (sfGeneric);
-    SerialIter sit (serialized.data (), serialized.size ());
-    st.set (sit);
-    return st.getFieldVL (sfMasterSignature);
-}
-
-bool
-ManifestCache::loadValidatorKeys(
-    Section const& keys,
-    beast::Journal journal)
-{
-    static boost::regex const re (
-        "[[:space:]]*"            // skip leading whitespace
-        "([[:alnum:]]+)"          // node identity
-        "(?:"                     // begin optional comment block
-        "[[:space:]]+"            // (skip all leading whitespace)
-        "(?:"                     // begin optional comment
-        "(.*[^[:space:]]+)"       // the comment
-        "[[:space:]]*"            // (skip all trailing whitespace)
-        ")?"                      // end optional comment
-        ")?"                      // end optional comment block
-    );
-
-    JLOG (journal.debug()) <<
-        "Loading configured validator keys";
-
-    std::size_t count = 0;
-
-    for (auto const& line : keys.lines())
-    {
-        boost::smatch match;
-
-        if (!boost::regex_match (line, match, re))
-        {
-            JLOG (journal.error()) <<
-                "Malformed entry: '" << line << "'";
-            return false;
-        }
-
-        auto const key = parseBase58<PublicKey>(
-            TokenType::TOKEN_NODE_PUBLIC, match[1]);
-
-        if (!key)
-        {
-            JLOG (journal.error()) <<
-                "Error decoding validator key: " << match[1];
-            return false;
-        }
-
-        if (publicKeyType(*key) != KeyType::ed25519)
-        {
-            JLOG (journal.error()) <<
-                "Validator key not using Ed25519: " << match[1];
-            return false;
-        }
-
-        JLOG (journal.debug()) << "Loaded key: " << match[1];
-
-        addTrustedKey (*key, match[2]);
-        ++count;
-    }
-
-    JLOG (journal.debug()) <<
-        "Loaded " << count << " entries";
-
-    return true;
-}
-
-void
-ManifestCache::configManifest (
-    Manifest m, ValidatorList& unl, beast::Journal journal)
-{
-    if (! m.verify())
-    {
-        Throw<std::runtime_error> ("Unverifiable manifest in config");
-    }
-
-    // Trust our own master public key
-    if (!trusted(m.masterKey) && !unl.trusted (m.masterKey))
-    {
-        addTrustedKey (m.masterKey, "");
-    }
-
-    auto const result = applyManifest (std::move(m), unl, journal);
-
-    if (result != ManifestDisposition::accepted)
-    {
-        Throw<std::runtime_error> ("Our own validation manifest was not accepted");
-    }
-}
-
-bool
-ManifestCache::trusted (PublicKey const& identity) const
-{
-    return map_.count(identity);
-}
-
-void
-ManifestCache::addTrustedKey (PublicKey const& pk, std::string comment)
-{
-    std::lock_guard<std::mutex> lock (mutex_);
-
-    auto& value = map_[pk];
-
-    if (value.m)
-    {
-        Throw<std::runtime_error> (
-            "New trusted validator key already has a manifest");
-    }
-
-    value.comment = std::move(comment);
-}
-
-std::size_t
-ManifestCache::size () const
-{
-    std::lock_guard <std::mutex> lock (mutex_);
-    return map_.size ();
-}
-
-ManifestDisposition
-ManifestCache::canApply (PublicKey const& pk, std::uint32_t seq,
-    beast::Journal journal) const
-{
-    auto const iter = map_.find(pk);
-
-    if (iter == map_.end())
-    {
-        /*
-            A manifest was received whose master key we don't trust.
-            Since rippled always sends all of its current manifests,
-            this will happen normally any time a peer connects.
-        */
-        if (auto stream = journal.debug())
-            logMftAct(stream, "Untrusted", pk, seq);
-        return ManifestDisposition::untrusted;
-    }
-
-    auto& old = iter->second.m;
-
-    if (old  &&  seq <= old->sequence)
-    {
-        /*
-            A manifest was received for a validator we're tracking, but
-            its sequence number is no higher than the one already stored.
-            This will happen normally when a peer without the latest gossip
-            connects.
-        */
-        if (auto stream = journal.debug())
-            logMftAct(stream, "Stale", pk, seq, old->sequence);
-        return ManifestDisposition::stale;  // not a newer manifest, ignore
-    }
-
-    return ManifestDisposition::accepted;
-}
-
-
-ManifestDisposition
-ManifestCache::applyManifest (
-    Manifest m, ValidatorList& unl, beast::Journal journal)
-{
-    /*
-        Move master public key from permanent trusted key list
-        to manifest cache.
-    */
-    if (auto unlComment = unl.member (m.masterKey))
-    {
-        addTrustedKey (m.masterKey, *unlComment);
-        unl.removePermanentKey (m.masterKey);
-    }
-
-    {
-        std::lock_guard<std::mutex> lock (mutex_);
-
-        /*
-            before we spend time checking the signature, make sure we trust the
-            master key and the sequence number is newer than any we have.
-        */
-        auto const chk = canApply(m.masterKey, m.sequence, journal);
-
-        if (chk != ManifestDisposition::accepted)
-        {
-            return chk;
-        }
-    }
-
-    if (! m.verify())
-    {
-        /*
-          A manifest's signature is invalid.
-          This shouldn't happen normally.
-        */
-        if (auto stream = journal.warn())
-            logMftAct(stream, "Invalid", m.masterKey, m.sequence);
-        return ManifestDisposition::invalid;
-    }
-
-    std::lock_guard<std::mutex> lock (mutex_);
-
-    /*
-        We released the lock above, so we have to check again, in case
-        another thread accepted a newer manifest.
-    */
-    auto const chk = canApply(m.masterKey, m.sequence, journal);
-
-    if (chk != ManifestDisposition::accepted)
-    {
-        return chk;
-    }
-
-    auto const iter = map_.find(m.masterKey);
-
-    auto& old = iter->second.m;
-
-    if (! old)
-    {
-        /*
-            This is the first received manifest for a trusted master key
-            (possibly our own).  This only happens once per validator per
-            run (and possibly not at all, if there's an obsolete entry in
-            [validator_keys] for a validator that no longer exists).
-        */
-        if (auto stream = journal.info())
-            logMftAct(stream, "AcceptedNew", m.masterKey, m.sequence);
-    }
-    else
-    {
-        if (m.revoked ())
-        {
-            /*
-               The MASTER key for this validator was revoked.  This is
-               expected, but should happen at most *very* rarely.
-            */
-            if (auto stream = journal.info())
-                logMftAct(stream, "Revoked",
-                    m.masterKey, m.sequence, old->sequence);
-        }
-        else
-        {
-            /*
-                An ephemeral key was revoked and superseded by a new key.
-                This is expected, but should happen infrequently.
-            */
-            if (auto stream = journal.info())
-                logMftAct(stream, "AcceptedUpdate",
-                          m.masterKey, m.sequence, old->sequence);
-        }
-
-        unl.removeEphemeralKey (old->signingKey);
-    }
-
-    if (m.revoked ())
-    {
-        // The master key is revoked -- don't insert the signing key
-        if (auto stream = journal.warn())
-            logMftAct(stream, "Revoked", m.masterKey, m.sequence);
-
-        /*
-            A validator master key has been compromised, so its manifests
-            are now untrustworthy.  In order to prevent us from accepting
-            a forged manifest signed by the compromised master key, store
-            this manifest, which has the highest possible sequence number
-            and therefore can't be superseded by a forged one.
-        */
-    }
-    else
-    {
-        unl.insertEphemeralKey (m.signingKey, iter->second.comment);
-    }
-
-    old = std::move(m);
-
-    return ManifestDisposition::accepted;
-}
-
-void ManifestCache::load (
-    DatabaseCon& dbCon, ValidatorList& unl, beast::Journal journal)
-{
-    static const char* const sql =
-        "SELECT RawData FROM ValidatorManifests;";
-    auto db = dbCon.checkoutDb ();
-    soci::blob sociRawData (*db);
-    soci::statement st =
-        (db->prepare << sql,
-             soci::into (sociRawData));
-    st.execute ();
-    while (st.fetch ())
-    {
-        std::string serialized;
-        convert (sociRawData, serialized);
-        if (auto mo = make_Manifest (std::move (serialized)))
-        {
-            if (!mo->verify())
-            {
-                JLOG(journal.warn())
-                    << "Unverifiable manifest in db";
-                continue;
-            }
-
-            if (trusted(mo->masterKey) || unl.trusted(mo->masterKey))
-            {
-                applyManifest (std::move(*mo), unl, journal);
-            }
-            else
-            {
-                JLOG(journal.info())
-                   << "Manifest in db is no longer trusted";
-            }
-        }
-        else
-        {
-            JLOG(journal.warn())
-                << "Malformed manifest in database";
-        }
-    }
-}
-
-void ManifestCache::save (DatabaseCon& dbCon) const
-{
-    auto db = dbCon.checkoutDb ();
-
-    soci::transaction tr(*db);
-    *db << "DELETE FROM ValidatorManifests";
-    static const char* const sql =
-        "INSERT INTO ValidatorManifests (RawData) VALUES (:rawData);";
-    for (auto const& v : map_)
-    {
-        if (!v.second.m)
-            continue;
-
-        // soci does not support bulk insertion of blob data
-        // Do not reuse blob because manifest ecdsa signatures vary in length
-        // but blob write length is expected to be >= the last write
-        soci::blob rawData(*db);
-        convert (v.second.m->serialized, rawData);
-        *db << sql,
-            soci::use (rawData);
-    }
-    tr.commit ();
-}
-}

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -20,7 +20,7 @@
 #include <BeastConfig.h>
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/NetworkOPs.h>
-#include <ripple/core/ConfigSections.h>
+#include <ripple/app/misc/ValidatorList.h>
 #include <ripple/core/DatabaseCon.h>
 #include <ripple/basics/contract.h>
 #include <ripple/basics/Log.h>
@@ -479,57 +479,6 @@ OverlayImpl::checkStopped ()
         stopped();
 }
 
-void
-OverlayImpl::setupValidatorKeyManifests (BasicConfig const& config,
-                                         DatabaseCon& db)
-{
-    auto const loaded = manifestCache_.loadValidatorKeys (
-        config.section (SECTION_VALIDATOR_KEYS),
-        journal_);
-
-    if (!loaded)
-        Throw<std::runtime_error> (
-            "Unable to load keys from [" SECTION_VALIDATOR_KEYS "]");
-
-    auto const& validation_manifest =
-        config.section (SECTION_VALIDATION_MANIFEST);
-
-    if (! validation_manifest.lines().empty())
-    {
-        std::string s;
-        s.reserve (Manifest::textLength);
-        for (auto const& line : validation_manifest.lines())
-            s += beast::rfc2616::trim(line);
-        if (auto mo = make_Manifest (beast::detail::base64_decode(s)))
-        {
-            manifestCache_.configManifest (
-                std::move (*mo),
-                app_.validators(),
-                journal_);
-        }
-        else
-        {
-            Throw<std::runtime_error> ("Malformed manifest in config");
-        }
-    }
-    else
-    {
-        JLOG(journal_.debug()) << "No [" SECTION_VALIDATION_MANIFEST <<
-            "] section in config";
-    }
-
-    manifestCache_.load (
-        db,
-        app_.validators(),
-        journal_);
-}
-
-void
-OverlayImpl::saveValidatorKeyManifests (DatabaseCon& db) const
-{
-    manifestCache_.save (db);
-}
-
 void
 OverlayImpl::onPrepare()
 {
@@ -713,21 +662,29 @@ OverlayImpl::onManifests (
     {
         auto& s = m->list ().Get (i).stobject ();
 
-        if (auto mo = make_Manifest (s))
+        if (auto mo = Manifest::make_Manifest (s))
         {
             uint256 const hash = mo->hash ();
             if (!hashRouter.addSuppressionPeer (hash, from->id ()))
                 continue;
 
-            auto const serialized = mo->serialized;
+            if (! app_.validators().listed (mo->masterKey))
-            auto const result = manifestCache_.applyManifest (
+            {
-                std::move(*mo),
+                JLOG(journal.info()) << "Untrusted manifest #" << i + 1;
-                app_.validators(),
+                app_.getOPs().pubManifest (*mo);
-                journal);
+                continue;
+            }
 
-            if (result == ManifestDisposition::accepted ||
+            auto const serialized = mo->serialized;
-                    result == ManifestDisposition::untrusted)
+
-                app_.getOPs().pubManifest (*make_Manifest(serialized));
+            auto const result = app_.validatorManifests ().applyManifest (
+                std::move(*mo));
+
+            if (result == ManifestDisposition::accepted)
+            {
+                app_.getOPs().pubManifest (
+                    *Manifest::make_Manifest(serialized));
+            }
 
             if (result == ManifestDisposition::accepted)
             {

src/ripple/overlay/impl/OverlayImpl.h

@@ -23,7 +23,6 @@
 #include <ripple/app/main/Application.h>
 #include <ripple/core/Job.h>
 #include <ripple/overlay/Overlay.h>
-#include <ripple/overlay/impl/Manifest.h>
 #include <ripple/overlay/impl/TrafficCount.h>
 #include <ripple/server/Handoff.h>
 #include <ripple/rpc/ServerHandler.h>
@@ -118,7 +117,6 @@ private:
     hash_map<Peer::id_t, std::weak_ptr<PeerImp>> ids_;
     Resolver& m_resolver;
     std::atomic <Peer::id_t> next_id_;
-    ManifestCache manifestCache_;
     int timer_count_;
 
     //--------------------------------------------------------------------------
@@ -152,12 +150,6 @@ public:
         return serverHandler_;
     }
 
-    ManifestCache const&
-    manifestCache() const
-    {
-        return manifestCache_;
-    }
-
     Setup const&
     setup() const
     {
@@ -195,15 +187,6 @@ public:
     relay (protocol::TMValidation& m,
         uint256 const& uid) override;
 
-    virtual
-    void
-    setupValidatorKeyManifests (BasicConfig const& config,
-                                DatabaseCon& db) override;
-
-    virtual
-    void
-    saveValidatorKeyManifests (DatabaseCon& db) const override;
-
     //--------------------------------------------------------------------------
     //
     // OverlayImpl

src/ripple/overlay/impl/PeerImp.cpp

@@ -703,8 +703,8 @@ PeerImp::doProtocolStart()
 
     protocol::TMManifests tm;
 
-    overlay_.manifestCache ().for_each_manifest (
+    app_.validatorManifests ().for_each_manifest (
-        [&tm](size_t s){tm.mutable_list()->Reserve(s);},
+        [&tm](std::size_t s){tm.mutable_list()->Reserve(s);},
         [&tm](Manifest const& manifest)
         {
             auto const& s = manifest.serialized;

src/ripple/protocol/JsonFields.h

@@ -116,10 +116,9 @@ JSS ( close_time_resolution );      // in: Application; out: LedgerToJson
 JSS ( closed );                     // out: NetworkOPs, LedgerToJson,
                                     //      handlers/Ledger
 JSS ( closed_ledger );              // out: NetworkOPs
-JSS ( cluster );                    // out: UniqueNodeList, PeerImp
+JSS ( cluster );                    // out: PeerImp
 JSS ( code );                       // out: errors
 JSS ( command );                    // in: RPCHandler
-JSS ( comment );                    // out: UnlList
 JSS ( complete );                   // out: NetworkOPs, InboundLedger
 JSS ( complete_ledgers );           // out: NetworkOPs, PeerImp
 JSS ( consensus );                  // out: NetworkOPs, LedgerConsensus
@@ -156,7 +155,6 @@ JSS ( enabled );                    // out: AmendmentTable
 JSS ( engine_result );              // out: NetworkOPs, TransactionSign, Submit
 JSS ( engine_result_code );         // out: NetworkOPs, TransactionSign, Submit
 JSS ( engine_result_message );      // out: NetworkOPs, TransactionSign, Submit
-JSS ( ephemeral );                  // out: UnlList
 JSS ( error );                      // out: error
 JSS ( error_code );                 // out: error
 JSS ( error_exception );            // out: Submit
@@ -416,6 +414,7 @@ JSS ( transactions );               // out: LedgerToJson,
 JSS ( transitions );                // out: NetworkOPs
 JSS ( treenode_cache_size );        // out: GetCounts
 JSS ( treenode_track_size );        // out: GetCounts
+JSS ( trusted );                    // out: UnlList
 JSS ( tx );                         // out: STTx, AccountTx*
 JSS ( tx_blob );                    // in/out: Submit,
                                     // in: TransactionSign, AccountTx*

src/ripple/rpc/handlers/UnlList.cpp

@@ -31,19 +31,16 @@ Json::Value doUnlList (RPC::Context& context)
     auto lock = make_lock(context.app.getMasterMutex());
     Json::Value obj (Json::objectValue);
 
-    context.app.validators().for_each (
+    context.app.validators().for_each_listed (
         [&unl = obj[jss::unl]](
             PublicKey const& publicKey,
-            std::string const& comment,
+            bool trusted)
-            bool ephemeral)
         {
             Json::Value node (Json::objectValue);
 
             node[jss::pubkey_validator] = toBase58(
                 TokenType::TOKEN_NODE_PUBLIC, publicKey);
-            node[jss::ephemeral] = ephemeral;
+            node[jss::trusted] = trusted;
-            if (!comment.empty())
-                node[jss::comment] = comment;
 
             unl.append (node);
         });

src/ripple/unity/app_misc.cpp

@@ -29,6 +29,7 @@
 #include <ripple/app/misc/impl/AccountTxPaging.cpp>
 #include <ripple/app/misc/impl/AmendmentTable.cpp>
 #include <ripple/app/misc/impl/LoadFeeTrack.cpp>
+#include <ripple/app/misc/impl/Manifest.cpp>
 #include <ripple/app/misc/impl/Transaction.cpp>
 #include <ripple/app/misc/impl/TxQ.cpp>
 #include <ripple/app/misc/impl/ValidatorList.cpp>

src/ripple/unity/overlay.cpp

@@ -21,7 +21,6 @@
 
 #include <ripple/overlay/impl/ConnectAttempt.cpp>
 #include <ripple/overlay/impl/Cluster.cpp>
-#include <ripple/overlay/impl/Manifest.cpp>
 #include <ripple/overlay/impl/Message.cpp>
 #include <ripple/overlay/impl/OverlayImpl.cpp>
 #include <ripple/overlay/impl/PeerImp.cpp>

src/test/app/AmendmentTable_test.cpp

@@ -402,7 +402,7 @@ public:
                 v->setFieldV256 (sfAmendments, field);
 
             v->setTrusted();
-            validations [calcNodeID(val)] = v;
+            validations [val] = v;
         }
 
         ourVotes = table.doValidation (enabled);

src/test/app/Manifest_test.cpp

@@ -17,23 +17,25 @@
 */
 //==============================================================================
 
-#include <BeastConfig.h>
+#include <ripple/app/misc/Manifest.h>
+#include <ripple/app/misc/ValidatorList.h>
 #include <ripple/basics/contract.h>
 #include <ripple/basics/StringUtilities.h>
-#include <test/jtx/TestSuite.h>
+#include <test/jtx.h>
-#include <ripple/overlay/impl/Manifest.h>
 #include <ripple/core/DatabaseCon.h>
 #include <ripple/app/main/DBInit.h>
 #include <ripple/protocol/SecretKey.h>
 #include <ripple/protocol/Sign.h>
 #include <ripple/protocol/STExchange.h>
+#include <beast/core/detail/base64.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/utility/in_place_factory.hpp>
 
 namespace ripple {
-namespace tests {
+namespace test {
 
-class manifest_test : public ripple::TestSuite
+class Manifest_test : public beast::unit_test::suite
 {
 private:
     static PublicKey randomNode ()
@@ -78,8 +80,9 @@ private:
     {
         return boost::filesystem::current_path () / "manifest_test_databases";
     }
+
 public:
-    manifest_test ()
+    Manifest_test ()
     {
         try
         {
@@ -89,7 +92,7 @@ public:
         {
         }
     }
-    ~manifest_test ()
+    ~Manifest_test ()
     {
         try
         {
@@ -100,6 +103,30 @@ public:
         }
     }
 
+    std::string
+    makeManifestString (
+        PublicKey const& pk,
+        SecretKey const& sk,
+        PublicKey const& spk,
+        SecretKey const& ssk,
+        int seq)
+    {
+        STObject st(sfGeneric);
+        st[sfSequence] = seq;
+        st[sfPublicKey] = pk;
+        st[sfSigningPubKey] = spk;
+
+        sign(st, HashPrefix::manifest, *publicKeyType(spk), ssk);
+        sign(st, HashPrefix::manifest, *publicKeyType(pk), sk,
+            sfMasterSignature);
+
+        Serializer s;
+        st.add(s);
+
+        return beast::detail::base64_encode (std::string(
+            static_cast<char const*> (s.data()), s.size()));
+    }
+
     Manifest
     make_Manifest
         (SecretKey const& sk, KeyType type, SecretKey const& ssk, KeyType stype,
@@ -129,10 +156,10 @@ public:
         st.add(s);
 
         std::string const m (static_cast<char const*> (s.data()), s.size());
-        if (auto r = ripple::make_Manifest (std::move (m)))
+        if (auto r = Manifest::make_Manifest (std::move (m)))
             return std::move (*r);
         Throw<std::runtime_error> ("Could not create a manifest");
-        return *ripple::make_Manifest(std::move(m)); // Silence compiler warning.
+        return *Manifest::make_Manifest(std::move(m)); // Silence compiler warning.
     }
 
     Manifest
@@ -141,105 +168,7 @@ public:
         return Manifest (m.serialized, m.masterKey, m.signingKey, m.sequence);
     }
 
-
+    void testLoadStore (ManifestCache& m)
-    void
-    testConfigLoad ()
-    {
-        testcase ("Config Load");
-
-        ManifestCache cache;
-        beast::Journal journal;
-
-        std::vector<PublicKey> network;
-        network.reserve(8);
-
-        while (network.size () != 8)
-            network.push_back (randomMasterKey());
-
-        auto format = [](
-            PublicKey const &publicKey,
-            char const* comment = nullptr)
-        {
-            auto ret = toBase58(
-                TokenType::TOKEN_NODE_PUBLIC,
-                publicKey);
-
-            if (comment)
-                ret += comment;
-
-            return ret;
-        };
-
-        Section s1;
-
-        // Correct (empty) configuration
-        BEAST_EXPECT(cache.loadValidatorKeys (s1, journal));
-        BEAST_EXPECT(cache.size() == 0);
-
-        // Correct configuration
-        s1.append (format (network[0]));
-        s1.append (format (network[1], " Comment"));
-        s1.append (format (network[2], " Multi Word Comment"));
-        s1.append (format (network[3], "    Leading Whitespace"));
-        s1.append (format (network[4], " Trailing Whitespace    "));
-        s1.append (format (network[5], "    Leading & Trailing Whitespace    "));
-        s1.append (format (network[6], "    Leading, Trailing & Internal    Whitespace    "));
-        s1.append (format (network[7], "    "));
-
-        BEAST_EXPECT(cache.loadValidatorKeys (s1, journal));
-
-        for (auto const& n : network)
-            BEAST_EXPECT(cache.trusted (n));
-
-        // Incorrect configurations:
-        Section s2;
-        s2.append ("NotAPublicKey");
-        BEAST_EXPECT(!cache.loadValidatorKeys (s2, journal));
-
-        Section s3;
-        s3.append (format (network[0], "!"));
-        BEAST_EXPECT(!cache.loadValidatorKeys (s3, journal));
-
-        Section s4;
-        s4.append (format (network[0], "!  Comment"));
-        BEAST_EXPECT(!cache.loadValidatorKeys (s4, journal));
-
-        // Check if we properly terminate when we encounter
-        // a malformed or unparseable entry:
-        auto const masterKey1 = randomMasterKey();
-        auto const masterKey2 = randomMasterKey ();
-
-        Section s5;
-        s5.append (format (masterKey1, "XXX"));
-        s5.append (format (masterKey2));
-        BEAST_EXPECT(!cache.loadValidatorKeys (s5, journal));
-        BEAST_EXPECT(!cache.trusted (masterKey1));
-        BEAST_EXPECT(!cache.trusted (masterKey2));
-
-        // Reject secp256k1 permanent validator keys
-        auto const node1 = randomNode ();
-        auto const node2 = randomNode ();
-
-        Section s6;
-        s6.append (format (node1));
-        s6.append (format (node2, " Comment"));
-        BEAST_EXPECT(!cache.loadValidatorKeys (s6, journal));
-        BEAST_EXPECT(!cache.trusted (node1));
-        BEAST_EXPECT(!cache.trusted (node2));
-
-        // Trust our own master public key from configured manifest
-        auto unl = std::make_unique<ValidatorList> (journal);
-
-        auto const sk = randomSecretKey();
-        auto const kp = randomKeyPair(KeyType::secp256k1);
-        auto const m  = make_Manifest (
-            sk, KeyType::ed25519, kp.second, KeyType::secp256k1, 0);
-
-        cache.configManifest (clone (m), *unl, journal);
-        BEAST_EXPECT(cache.trusted (m.masterKey));
-    }
-
-    void testLoadStore (ManifestCache const& m, ValidatorList& unl)
     {
         testcase ("load/store");
 
@@ -251,13 +180,6 @@ public:
             setup.dataDir = getDatabasePath ();
             DatabaseCon dbCon(setup, dbName, WalletDBInit, WalletDBCount);
 
-            if (!m.size ())
-                fail ();
-
-            m.save (dbCon);
-
-            beast::Journal journal;
-
             auto getPopulatedManifests =
                     [](ManifestCache const& cache) -> std::vector<Manifest const*>
                     {
@@ -279,21 +201,45 @@ public:
                     };
             std::vector<Manifest const*> const inManifests (
                 sort (getPopulatedManifests (m)));
+
+            beast::Journal journal;
+            jtx::Env env (*this);
+            auto unl = std::make_unique<ValidatorList> (
+                m, m, env.timeKeeper(), journal);
+
             {
-                // load should not load untrusted master keys from db
+                // save should not store untrusted master keys to db
+                m.save (dbCon, "ValidatorManifests",
+                    [&unl](PublicKey const& pubKey)
+                    {
+                        return unl->listed (pubKey);
+                    });
+
                 ManifestCache loaded;
 
-                loaded.load (dbCon, unl, journal);
+                loaded.load (dbCon, "ValidatorManifests");
-                BEAST_EXPECT(loaded.size() == 0);
+                for (auto const& man : inManifests)
+                    BEAST_EXPECT(
+                        loaded.getSigningKey (man->masterKey) == man->masterKey);
             }
             {
-                // load should load all trusted master keys from db
+                // save should store all trusted master keys to db
-                ManifestCache loaded;
+                PublicKey emptyLocalKey;
-
+                std::vector<std::string> s1;
+                std::vector<std::string> keys;
+                std::vector<std::string> cfgManifest;
                 for (auto const& man : inManifests)
-                    loaded.addTrustedKey (man->masterKey, "");
+                    s1.push_back (toBase58(
+                        TokenType::TOKEN_NODE_PUBLIC, man->masterKey));
+                unl->load (emptyLocalKey, s1, keys);
 
-                loaded.load (dbCon, unl, journal);
+                m.save (dbCon, "ValidatorManifests",
+                    [&unl](PublicKey const& pubKey)
+                    {
+                        return unl->listed (pubKey);
+                    });
+                ManifestCache loaded;
+                loaded.load (dbCon, "ValidatorManifests");
 
                 std::vector<Manifest const*> const loadedManifests (
                     sort (getPopulatedManifests (loaded)));
@@ -312,18 +258,23 @@ public:
                 }
             }
             {
-                // load should remove master key from permanent key list
+                // load config manifest
-                ManifestCache loaded;
+                std::vector<std::string> const badManifest ({"bad manifest"});
-                auto const iMan = inManifests.begin();
 
-                if (!*iMan)
+                ManifestCache loaded;
-                    fail ();
+                BEAST_EXPECT(! loaded.load (
-                BEAST_EXPECT(m.trusted((*iMan)->masterKey));
+                    dbCon, "ValidatorManifests", badManifest));
-                BEAST_EXPECT(unl.insertPermanentKey((*iMan)->masterKey, "trusted key"));
+
-                BEAST_EXPECT(unl.trusted((*iMan)->masterKey));
+                auto const sk  = randomSecretKey();
-                loaded.load (dbCon, unl, journal);
+                auto const pk  = derivePublicKey(KeyType::ed25519, sk);
-                BEAST_EXPECT(!unl.trusted((*iMan)->masterKey));
+                auto const kp = randomKeyPair(KeyType::secp256k1);
-                BEAST_EXPECT(loaded.trusted((*iMan)->masterKey));
+
+                std::vector<std::string> const cfgManifest ({
+                    makeManifestString (pk, sk, kp.first, kp.second, 0)
+                });
+
+                BEAST_EXPECT(loaded.load (
+                    dbCon, "ValidatorManifests", cfgManifest));
             }
         }
         boost::filesystem::remove (getDatabasePath () /
@@ -353,16 +304,76 @@ public:
         BEAST_EXPECT(strHex(masterSig) == strHex(m.getMasterSignature()));
     }
 
+    void testGetKeys()
+    {
+        testcase ("getKeys");
+
+        ManifestCache cache;
+        auto const sk  = randomSecretKey();
+        auto const pk  = derivePublicKey(KeyType::ed25519, sk);
+
+        // getSigningKey should return same key if there is no manifest
+        BEAST_EXPECT(cache.getSigningKey(pk) == pk);
+
+        // getSigningKey should return the ephemeral public key
+        // for the listed validator master public key
+        // getMasterKey should return the listed validator master key
+        // for that ephemeral public key
+        auto const kp0 = randomKeyPair(KeyType::secp256k1);
+        auto const m0  = make_Manifest (
+            sk, KeyType::ed25519, kp0.second, KeyType::secp256k1, 0);
+        BEAST_EXPECT(cache.applyManifest(clone (m0)) ==
+                ManifestDisposition::accepted);
+        BEAST_EXPECT(cache.getSigningKey(pk) == kp0.first);
+        BEAST_EXPECT(cache.getMasterKey(kp0.first) == pk);
+
+        // getSigningKey should return the latest ephemeral public key
+        // for the listed validator master public key
+        // getMasterKey should only return a master key for the latest
+        // ephemeral public key
+        auto const kp1 = randomKeyPair(KeyType::secp256k1);
+        auto const m1  = make_Manifest (
+            sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 1);
+        BEAST_EXPECT(cache.applyManifest(clone (m1)) ==
+                ManifestDisposition::accepted);
+        BEAST_EXPECT(cache.getSigningKey(pk) == kp1.first);
+        BEAST_EXPECT(cache.getMasterKey(kp1.first) == pk);
+        BEAST_EXPECT(cache.getMasterKey(kp0.first) == kp0.first);
+
+        // getSigningKey and getMasterKey should return the same keys if
+        // a new manifest is applied with the same signing key but a higher
+        // sequence
+        auto const m2  = make_Manifest (
+            sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 2);
+        BEAST_EXPECT(cache.applyManifest(clone (m2)) ==
+                ManifestDisposition::accepted);
+        BEAST_EXPECT(cache.getSigningKey(pk) == kp1.first);
+        BEAST_EXPECT(cache.getMasterKey(kp1.first) == pk);
+        BEAST_EXPECT(cache.getMasterKey(kp0.first) == kp0.first);
+
+        // getSigningKey should return boost::none for a
+        // revoked master public key
+        // getMasterKey should return boost::none for an ephemeral public key
+        // from a revoked master public key
+        auto const kpMax = randomKeyPair(KeyType::secp256k1);
+        auto const mMax = make_Manifest (
+            sk, KeyType::ed25519, kpMax.second, KeyType::secp256k1,
+            std::numeric_limits<std::uint32_t>::max ());
+        BEAST_EXPECT(cache.applyManifest(clone (mMax)) ==
+                ManifestDisposition::accepted);
+        BEAST_EXPECT(cache.revoked(pk));
+        BEAST_EXPECT(cache.getSigningKey(pk) == pk);
+        BEAST_EXPECT(cache.getMasterKey(kpMax.first) == kpMax.first);
+        BEAST_EXPECT(cache.getMasterKey(kp1.first) == kp1.first);
+    }
+
     void
     run() override
     {
         ManifestCache cache;
-        beast::Journal journal;
-        auto unl = std::make_unique<ValidatorList> (journal);
         {
             testcase ("apply");
             auto const accepted = ManifestDisposition::accepted;
-            auto const untrusted = ManifestDisposition::untrusted;
             auto const stale = ManifestDisposition::stale;
             auto const invalid = ManifestDisposition::invalid;
 
@@ -373,9 +384,11 @@ public:
                 sk_a, KeyType::ed25519, kp_a.second, KeyType::secp256k1, 0);
             auto const s_a1 = make_Manifest (
                 sk_a, KeyType::ed25519, kp_a.second, KeyType::secp256k1, 1);
+            auto const s_aMax = make_Manifest (
+                sk_a, KeyType::ed25519, kp_a.second, KeyType::secp256k1,
+                std::numeric_limits<std::uint32_t>::max ());
 
             auto const sk_b = randomSecretKey();
-            auto const pk_b = derivePublicKey(KeyType::ed25519, sk_b);
             auto const kp_b = randomKeyPair(KeyType::secp256k1);
             auto const s_b0 = make_Manifest (
                 sk_b, KeyType::ed25519, kp_b.second, KeyType::secp256k1, 0);
@@ -386,45 +399,39 @@ public:
                 true);  // broken
             auto const fake = s_b1.serialized + '\0';
 
-            BEAST_EXPECT(cache.applyManifest (clone (s_a0), *unl, journal) == untrusted);
+            // applyManifest should accept new manifests with
+            // higher sequence numbers
+            BEAST_EXPECT(cache.applyManifest (clone (s_a0)) == accepted);
+            BEAST_EXPECT(cache.applyManifest (clone (s_a0)) == stale);
 
-            cache.addTrustedKey (pk_a, "a");
+            BEAST_EXPECT(cache.applyManifest (clone (s_a1)) == accepted);
-            cache.addTrustedKey (pk_b, "b");
+            BEAST_EXPECT(cache.applyManifest (clone (s_a1)) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_a0)) == stale);
 
-            BEAST_EXPECT(cache.applyManifest (clone (s_a0), *unl, journal) == accepted);
+            // applyManifest should accept manifests with max sequence numbers
-            BEAST_EXPECT(cache.applyManifest (clone (s_a0), *unl, journal) == stale);
+            // that revoke the master public key
+            BEAST_EXPECT(!cache.revoked (pk_a));
+            BEAST_EXPECT(s_aMax.revoked ());
+            BEAST_EXPECT(cache.applyManifest (clone (s_aMax)) == accepted);
+            BEAST_EXPECT(cache.applyManifest (clone (s_aMax)) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_a1)) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_a0)) == stale);
+            BEAST_EXPECT(cache.revoked (pk_a));
 
-            BEAST_EXPECT(cache.applyManifest (clone (s_a1), *unl, journal) == accepted);
+            // applyManifest should reject manifests with invalid signatures
-            BEAST_EXPECT(cache.applyManifest (clone (s_a1), *unl, journal) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_b0)) == accepted);
-            BEAST_EXPECT(cache.applyManifest (clone (s_a0), *unl, journal) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_b0)) == stale);
 
-            BEAST_EXPECT(cache.applyManifest (clone (s_b0), *unl, journal) == accepted);
+            BEAST_EXPECT(!Manifest::make_Manifest(fake));
-            BEAST_EXPECT(cache.applyManifest (clone (s_b0), *unl, journal) == stale);
+            BEAST_EXPECT(cache.applyManifest (clone (s_b2)) == invalid);
-
-            BEAST_EXPECT(!ripple::make_Manifest(fake));
-            BEAST_EXPECT(cache.applyManifest (clone (s_b2), *unl, journal) == invalid);
-
-            // When trusted permanent key is found as manifest master key
-            // move to manifest cache
-            auto const sk_c = randomSecretKey();
-            auto const pk_c = derivePublicKey(KeyType::ed25519, sk_c);
-            auto const kp_c = randomKeyPair(KeyType::secp256k1);
-            auto const s_c0 = make_Manifest (
-                sk_c, KeyType::ed25519, kp_c.second, KeyType::secp256k1, 0);
-            BEAST_EXPECT(unl->insertPermanentKey(pk_c, "trusted key"));
-            BEAST_EXPECT(unl->trusted(pk_c));
-            BEAST_EXPECT(!cache.trusted(pk_c));
-            BEAST_EXPECT(cache.applyManifest(clone (s_c0), *unl, journal) == accepted);
-            BEAST_EXPECT(!unl->trusted(pk_c));
-            BEAST_EXPECT(cache.trusted(pk_c));
         }
-        testConfigLoad();
+        testLoadStore (cache);
-        testLoadStore (cache, *unl);
         testGetSignature ();
+        testGetKeys ();
     }
 };
 
-BEAST_DEFINE_TESTSUITE(manifest,overlay,ripple);
+BEAST_DEFINE_TESTSUITE(Manifest,app,ripple);
 
-} // tests
+} // test
 } // ripple

src/test/app/ValidatorList_test.cpp

@@ -17,48 +17,111 @@
 */
 //==============================================================================
 
-#include <BeastConfig.h>
+#include <beast/core/detail/base64.hpp>
 #include <ripple/basics/Slice.h>
-#include <test/jtx/TestSuite.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/app/misc/ValidatorList.h>
+#include <test/jtx.h>
+#include <ripple/protocol/digest.h>
+#include <ripple/protocol/HashPrefix.h>
+#include <ripple/protocol/PublicKey.h>
 #include <ripple/protocol/SecretKey.h>
+#include <ripple/protocol/Sign.h>
+
 
 namespace ripple {
-namespace tests {
+namespace test {
 
-class ValidatorList_test : public ripple::TestSuite
+class ValidatorList_test : public beast::unit_test::suite
 {
 private:
     static
     PublicKey
     randomNode ()
     {
-        return derivePublicKey (
+        return derivePublicKey (KeyType::secp256k1, randomSecretKey());
-            KeyType::secp256k1,
-            randomSecretKey());
     }
 
     static
     PublicKey
     randomMasterKey ()
     {
-        return derivePublicKey (
+        return derivePublicKey (KeyType::ed25519, randomSecretKey());
-            KeyType::ed25519,
-            randomSecretKey());
     }
 
-    static
+    std::string
-    bool
+    makeManifestString (
-    isPresent (
+        PublicKey const& pk,
-        std::vector<PublicKey> container,
+        SecretKey const& sk,
-        PublicKey const& item)
+        PublicKey const& spk,
+        SecretKey const& ssk,
+        int seq)
     {
-        auto found = std::find (
+        STObject st(sfGeneric);
-            std::begin (container),
+        st[sfSequence] = seq;
-            std::end (container),
+        st[sfPublicKey] = pk;
-            item);
+        st[sfSigningPubKey] = spk;
 
-        return (found != std::end (container));
+        sign(st, HashPrefix::manifest, *publicKeyType(spk), ssk);
+        sign(st, HashPrefix::manifest, *publicKeyType(pk), sk,
+            sfMasterSignature);
+
+        Serializer s;
+        st.add(s);
+
+        return std::string(static_cast<char const*> (s.data()), s.size());
+    }
+
+    std::string
+    makeList (
+        std::vector <PublicKey> const& validators,
+        std::size_t sequence,
+        std::size_t expiration)
+    {
+        std::string data =
+            "{\"sequence\":" + std::to_string(sequence) +
+            ",\"expiration\":" + std::to_string(expiration) +
+            ",\"validators\":[";
+
+        for (auto const& val : validators)
+        {
+            data += "{\"validation_public_key\":\"" + strHex(val) + "\"},";
+        }
+
+        data.pop_back();
+        data += "]}";
+        return beast::detail::base64_encode(data);
+    }
+
+    std::string
+    signList (
+        std::string const& blob,
+        std::pair<PublicKey, SecretKey> const& keys)
+    {
+        auto const data = beast::detail::base64_decode (blob);
+        return strHex(sign(
+            keys.first, keys.second, makeSlice(data)));
+    }
+
+    void
+    testGenesisQuorum ()
+    {
+        testcase ("Genesis Quorum");
+
+        beast::Journal journal;
+        ManifestCache manifests;
+        jtx::Env env (*this);
+        {
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
+            BEAST_EXPECT(trustedKeys->quorum () == 1);
+        }
+        {
+            std::size_t minQuorum = 0;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal, minQuorum);
+            BEAST_EXPECT(trustedKeys->quorum () == minQuorum);
+        }
     }
 
     void
@@ -66,21 +129,28 @@ private:
     {
         testcase ("Config Load");
 
-        auto validators = std::make_unique <ValidatorList> (beast::Journal ());
+        beast::Journal journal;
+        jtx::Env env (*this);
+        PublicKey emptyLocalKey;
+        std::vector<std::string> emptyCfgKeys;
+        std::vector<std::string> emptyCfgPublishers;
 
-        std::vector<PublicKey> network;
+        auto const localSigningKeys = randomKeyPair(KeyType::secp256k1);
-        network.reserve(8);
+        auto const localSigningPublic = localSigningKeys.first;
+        auto const localSigningSecret = localSigningKeys.second;
+        auto const localMasterSecret = randomSecretKey();
+        auto const localMasterPublic = derivePublicKey(
+            KeyType::ed25519, localMasterSecret);
 
-        while (network.size () != 8)
+        auto cfgManifest = makeManifestString (
-            network.push_back (randomNode());
+                localMasterPublic, localMasterSecret,
+                localSigningPublic, localSigningSecret, 1);
 
         auto format = [](
             PublicKey const &publicKey,
             char const* comment = nullptr)
         {
-            auto ret = toBase58(
+            auto ret = toBase58 (TokenType::TOKEN_NODE_PUBLIC, publicKey);
-                TokenType::TOKEN_NODE_PUBLIC,
-                publicKey);
 
             if (comment)
                 ret += comment;
@@ -88,213 +158,592 @@ private:
             return ret;
         };
 
-        Section s1;
+        std::vector<PublicKey> configList;
+        configList.reserve(8);
 
-        // Correct (empty) configuration
+        while (configList.size () != 8)
-        BEAST_EXPECT(validators->load (s1));
+            configList.push_back (randomNode());
-        BEAST_EXPECT(validators->size() == 0);
 
         // Correct configuration
-        s1.append (format (network[0]));
+        std::vector<std::string> cfgKeys ({
-        s1.append (format (network[1], " Comment"));
+            format (configList[0]),
-        s1.append (format (network[2], " Multi Word Comment"));
+            format (configList[1], " Comment"),
-        s1.append (format (network[3], "    Leading Whitespace"));
+            format (configList[2], " Multi Word Comment"),
-        s1.append (format (network[4], " Trailing Whitespace    "));
+            format (configList[3], "    Leading Whitespace"),
-        s1.append (format (network[5], "    Leading & Trailing Whitespace    "));
+            format (configList[4], " Trailing Whitespace    "),
-        s1.append (format (network[6], "    Leading, Trailing & Internal    Whitespace    "));
+            format (configList[5], "    Leading & Trailing Whitespace    "),
-        s1.append (format (network[7], "    "));
+            format (configList[6], "    Leading, Trailing & Internal    Whitespace    "),
+            format (configList[7], "    ")
+        });
 
-        BEAST_EXPECT(validators->load (s1));
+        {
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-        for (auto const& n : network)
+            // Correct (empty) configuration
-            BEAST_EXPECT(validators->trusted (n));
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, emptyCfgKeys, emptyCfgPublishers));
 
-        // Incorrect configurations:
+            // load local validator key with or without manifest
-        Section s2;
+            BEAST_EXPECT(trustedKeys->load (
-        s2.append ("NotAPublicKey");
+                localSigningPublic, emptyCfgKeys, emptyCfgPublishers));
-        BEAST_EXPECT(!validators->load (s2));
+            BEAST_EXPECT(trustedKeys->listed (localSigningPublic));
 
-        Section s3;
+            manifests.applyManifest (*Manifest::make_Manifest(cfgManifest));
-        s3.append (format (network[0], "!"));
+            BEAST_EXPECT(trustedKeys->load (
-        BEAST_EXPECT(!validators->load (s3));
+                localSigningPublic, emptyCfgKeys, emptyCfgPublishers));
 
-        Section s4;
+            BEAST_EXPECT(trustedKeys->listed (localMasterPublic));
-        s4.append (format (network[0], "!  Comment"));
+            BEAST_EXPECT(trustedKeys->listed (localSigningPublic));
-        BEAST_EXPECT(!validators->load (s4));
+        }
+        {
+            // load should add validator keys from config
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-        // Check if we properly terminate when we encounter
+            BEAST_EXPECT(trustedKeys->load (
-        // a malformed or unparseable entry:
+                emptyLocalKey, cfgKeys, emptyCfgPublishers));
-        auto const node1 = randomNode();
-        auto const node2 = randomNode ();
 
-        Section s5;
+            for (auto const& n : configList)
-        s5.append (format (node1, "XXX"));
+                BEAST_EXPECT(trustedKeys->listed (n));
-        s5.append (format (node2));
-        BEAST_EXPECT(!validators->load (s5));
-        BEAST_EXPECT(!validators->trusted (node1));
-        BEAST_EXPECT(!validators->trusted (node2));
 
-        // Add Ed25519 master public keys to permanent validators list
+            // load should accept Ed25519 master public keys
             auto const masterNode1 = randomMasterKey ();
             auto const masterNode2 = randomMasterKey ();
 
-        Section s6;
+            std::vector<std::string> cfgMasterKeys({
-        s6.append (format (masterNode1));
+                format (masterNode1),
-        s6.append (format (masterNode2, " Comment"));
+                format (masterNode2, " Comment")
-        BEAST_EXPECT(validators->load (s6));
+            });
-        BEAST_EXPECT(validators->trusted (masterNode1));
+            BEAST_EXPECT(trustedKeys->load (
-        BEAST_EXPECT(validators->trusted (masterNode2));
+                emptyLocalKey, cfgMasterKeys, emptyCfgPublishers));
+            BEAST_EXPECT(trustedKeys->listed (masterNode1));
+            BEAST_EXPECT(trustedKeys->listed (masterNode2));
+
+            // load should reject invalid config keys
+            std::vector<std::string> badKeys({"NotAPublicKey"});
+            BEAST_EXPECT(!trustedKeys->load (
+                emptyLocalKey, badKeys, emptyCfgPublishers));
+
+            badKeys[0] = format (randomNode(), "!");
+            BEAST_EXPECT(!trustedKeys->load (
+                emptyLocalKey, badKeys, emptyCfgPublishers));
+
+            badKeys[0] = format (randomNode(), "!  Comment");
+            BEAST_EXPECT(!trustedKeys->load (
+                emptyLocalKey, badKeys, emptyCfgPublishers));
+
+            // load terminates when encountering an invalid entry
+            auto const goodKey = randomNode();
+            badKeys.push_back (format (goodKey));
+            BEAST_EXPECT(!trustedKeys->load (
+                emptyLocalKey, badKeys, emptyCfgPublishers));
+            BEAST_EXPECT(!trustedKeys->listed (goodKey));
         }
-
-    void
-    testMembership ()
         {
-        // The servers on the permanentValidators
+            // local validator key on config list
-        std::vector<PublicKey> permanentValidators;
+            ManifestCache manifests;
-        std::vector<PublicKey> ephemeralValidators;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-        while (permanentValidators.size () != 64)
+            auto const localSigningPublic = parseBase58<PublicKey> (
-            permanentValidators.push_back (randomNode());
+                TokenType::TOKEN_NODE_PUBLIC, cfgKeys.front());
 
-        while (ephemeralValidators.size () != 64)
+            BEAST_EXPECT(trustedKeys->load (
-            ephemeralValidators.push_back (randomNode());
+                *localSigningPublic, cfgKeys, emptyCfgPublishers));
 
-        {
+            BEAST_EXPECT(trustedKeys->listed (*localSigningPublic));
-            testcase ("Membership: No Validators");
+            for (auto const& n : configList)
-
+                BEAST_EXPECT(trustedKeys->listed (n));
-            auto vl = std::make_unique <ValidatorList> (beast::Journal ());
-
-            for (auto const& v : permanentValidators)
-                BEAST_EXPECT(!vl->trusted (v));
-
-            for (auto const& v : ephemeralValidators)
-                BEAST_EXPECT(!vl->trusted (v));
         }
-
         {
-            testcase ("Membership: Non-Empty, Some Present, Some Not Present");
+            // local validator key not on config list
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-            std::vector<PublicKey> p (
+            auto const localSigningPublic = randomNode();
-                permanentValidators.begin (),
+            BEAST_EXPECT(trustedKeys->load (
-                permanentValidators.begin () + 16);
+                localSigningPublic, cfgKeys, emptyCfgPublishers));
 
-            while (p.size () != 32)
+            BEAST_EXPECT(trustedKeys->listed (localSigningPublic));
-                p.push_back (randomNode());
+            for (auto const& n : configList)
+                BEAST_EXPECT(trustedKeys->listed (n));
+        }
+        {
+            // local validator key (with manifest) not on config list
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-            std::vector<PublicKey> e (
+            manifests.applyManifest (*Manifest::make_Manifest(cfgManifest));
-                ephemeralValidators.begin (),
-                ephemeralValidators.begin () + 16);
 
-            while (e.size () != 32)
+            BEAST_EXPECT(trustedKeys->load (
-                e.push_back (randomNode());
+                localSigningPublic, cfgKeys, emptyCfgPublishers));
 
-            auto vl = std::make_unique <ValidatorList> (beast::Journal ());
+            BEAST_EXPECT(trustedKeys->listed (localSigningPublic));
+            BEAST_EXPECT(trustedKeys->listed (localMasterPublic));
+            for (auto const& n : configList)
+                BEAST_EXPECT(trustedKeys->listed (n));
+        }
+        {
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), journal);
 
-            for (auto const& v : p)
+            // load should reject invalid validator list signing keys
-                vl->insertPermanentKey (v, "");
+            std::vector<std::string> badPublishers(
+                {"NotASigningKey"});
+            BEAST_EXPECT(!trustedKeys->load (
+                emptyLocalKey, emptyCfgKeys, badPublishers));
 
-            for (auto const& v : e)
+            // load should reject validator list signing keys with invalid encoding
-                vl->insertEphemeralKey (v, "");
+            std::vector<PublicKey> keys ({
+                randomMasterKey(), randomMasterKey(), randomMasterKey()});
+            badPublishers.clear();
+            for (auto const& key : keys)
+                badPublishers.push_back (
+                    toBase58 (TokenType::TOKEN_NODE_PUBLIC, key));
 
-            for (auto const& v : p)
+            BEAST_EXPECT(! trustedKeys->load (
-                BEAST_EXPECT(vl->trusted (v));
+                emptyLocalKey, emptyCfgKeys, badPublishers));
+            for (auto const& key : keys)
+                BEAST_EXPECT(!trustedKeys->trustedPublisher (key));
 
-            for (auto const& v : e)
+            // load should accept valid validator list publisher keys
-                BEAST_EXPECT(vl->trusted (v));
+            std::vector<std::string> cfgPublishers;
+            for (auto const& key : keys)
+                cfgPublishers.push_back (strHex(key));
 
-            for (auto const& v : permanentValidators)
+            BEAST_EXPECT(trustedKeys->load (
-                BEAST_EXPECT(static_cast<bool>(vl->trusted (v)) == isPresent (p, v));
+                emptyLocalKey, emptyCfgKeys, cfgPublishers));
-
+            for (auto const& key : keys)
-            for (auto const& v : ephemeralValidators)
+                BEAST_EXPECT(trustedKeys->trustedPublisher (key));
-                BEAST_EXPECT(static_cast<bool>(vl->trusted (v)) == isPresent (e, v));
         }
     }
 
     void
-    testModification ()
+    testApplyList ()
     {
-        testcase ("Insertion and Removal");
+        testcase ("Apply list");
 
-        auto vl = std::make_unique <ValidatorList> (beast::Journal ());
+        beast::Journal journal;
+        ManifestCache manifests;
+        jtx::Env env (*this);
+        auto trustedKeys = std::make_unique<ValidatorList> (
+            manifests, manifests, env.app().timeKeeper(), journal);
 
-        auto const v = randomNode ();
+        auto const publisherSecret = randomSecretKey();
+        auto const publisherPublic =
+            derivePublicKey(KeyType::ed25519, publisherSecret);
+        auto const pubSigningKeys1 = randomKeyPair(KeyType::secp256k1);
+        auto const manifest1 = beast::detail::base64_encode(makeManifestString (
+            publisherPublic, publisherSecret,
+            pubSigningKeys1.first, pubSigningKeys1.second, 1));
 
-        // Inserting a new permanent key succeeds
+        std::vector<std::string> cfgKeys1({
-        BEAST_EXPECT(vl->insertPermanentKey (v, "Permanent"));
+            strHex(publisherPublic)});
-        {
+        PublicKey emptyLocalKey;
-            auto member = vl->member (v);
+        std::vector<std::string> emptyCfgKeys;
-            BEAST_EXPECT(static_cast<bool>(member));
-            BEAST_EXPECT(member->compare("Permanent") == 0);
-        }
-        // Inserting the same permanent key fails:
-        BEAST_EXPECT(!vl->insertPermanentKey (v, ""));
-        {
-            auto member = vl->member (v);
-            BEAST_EXPECT(static_cast<bool>(member));
-            BEAST_EXPECT(member->compare("Permanent") == 0);
-        }
-        // Inserting the same key as ephemeral fails:
-        BEAST_EXPECT(!vl->insertEphemeralKey (v, "Ephemeral"));
-        {
-            auto member = vl->member (v);
-            BEAST_EXPECT(static_cast<bool>(member));
-            BEAST_EXPECT(member->compare("Permanent") == 0);
-        }
-        // Removing the key as ephemeral fails:
-        BEAST_EXPECT(!vl->removeEphemeralKey (v));
-        {
-            auto member = vl->member (v);
-            BEAST_EXPECT(static_cast<bool>(member));
-            BEAST_EXPECT(member->compare("Permanent") == 0);
-        }
-        // Deleting the key as permanent succeeds:
-        BEAST_EXPECT(vl->removePermanentKey (v));
-        BEAST_EXPECT(!static_cast<bool>(vl->trusted (v)));
 
-        // Insert an ephemeral validator key
+        BEAST_EXPECT(trustedKeys->load (
-        BEAST_EXPECT(vl->insertEphemeralKey (v, "Ephemeral"));
+            emptyLocalKey, emptyCfgKeys, cfgKeys1));
-        {
+
-            auto member = vl->member (v);
+        auto constexpr listSize = 20;
-            BEAST_EXPECT(static_cast<bool>(member));
+        std::vector<PublicKey> list1;
-            BEAST_EXPECT(member->compare("Ephemeral") == 0);
+        list1.reserve (listSize);
+        while (list1.size () < listSize)
+            list1.push_back (randomNode());
+
+        std::vector<PublicKey> list2;
+        list2.reserve (listSize);
+        while (list2.size () < listSize)
+            list2.push_back (randomNode());
+
+        // do not apply expired list
+        auto const version = 1;
+        auto const sequence = 1;
+        auto const expiredblob = makeList (
+            list1, sequence, env.timeKeeper().now().time_since_epoch().count());
+        auto const expiredSig = signList (expiredblob, pubSigningKeys1);
+
+        BEAST_EXPECT(ListDisposition::stale ==
+            trustedKeys->applyList (
+                manifest1, expiredblob, expiredSig, version));
+
+        // apply single list
+        NetClock::time_point const expiration =
+            env.timeKeeper().now() + 3600s;
+        auto const blob1 = makeList (
+            list1, sequence, expiration.time_since_epoch().count());
+        auto const sig1 = signList (blob1, pubSigningKeys1);
+
+        BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList (
+            manifest1, blob1, sig1, version));
+
+        for (auto const& val : list1)
+            BEAST_EXPECT(trustedKeys->listed (val));
+
+        // do not use list from untrusted publisher
+        auto const untrustedManifest = beast::detail::base64_encode(
+            makeManifestString (
+                randomMasterKey(), publisherSecret,
+                pubSigningKeys1.first, pubSigningKeys1.second, 1));
+
+        BEAST_EXPECT(ListDisposition::untrusted == trustedKeys->applyList (
+            untrustedManifest, blob1, sig1, version));
+
+        // do not use list with unhandled version
+        auto const badVersion = 666;
+        BEAST_EXPECT(ListDisposition::unsupported_version ==
+            trustedKeys->applyList (
+                manifest1, blob1, sig1, badVersion));
+
+        // apply list with highest sequence number
+        auto const sequence2 = 2;
+        auto const blob2 = makeList (
+            list2, sequence2, expiration.time_since_epoch().count());
+        auto const sig2 = signList (blob2, pubSigningKeys1);
+
+        BEAST_EXPECT(ListDisposition::accepted ==
+            trustedKeys->applyList (
+                manifest1, blob2, sig2, version));
+
+        for (auto const& val : list1)
+            BEAST_EXPECT(! trustedKeys->listed (val));
+
+        for (auto const& val : list2)
+            BEAST_EXPECT(trustedKeys->listed (val));
+
+        // do not re-apply lists with past or current sequence numbers
+        BEAST_EXPECT(ListDisposition::stale ==
+            trustedKeys->applyList (
+                manifest1, blob1, sig1, version));
+
+        BEAST_EXPECT(ListDisposition::stale ==
+            trustedKeys->applyList (
+                manifest1, blob2, sig2, version));
+
+        // apply list with new publisher key updated by manifest
+        auto const pubSigningKeys2 = randomKeyPair(KeyType::secp256k1);
+        auto manifest2 = beast::detail::base64_encode(makeManifestString (
+            publisherPublic, publisherSecret,
+            pubSigningKeys2.first, pubSigningKeys2.second, 2));
+
+        auto const sequence3 = 3;
+        auto const blob3 = makeList (
+            list1, sequence3, expiration.time_since_epoch().count());
+        auto const sig3 = signList (blob3, pubSigningKeys2);
+
+        BEAST_EXPECT(ListDisposition::accepted ==
+            trustedKeys->applyList (
+                manifest2, blob3, sig3, version));
+
+        auto const sequence4 = 4;
+        auto const blob4 = makeList (
+            list1, sequence4, expiration.time_since_epoch().count());
+        auto const badSig = signList (blob4, pubSigningKeys1);
+        BEAST_EXPECT(ListDisposition::invalid ==
+            trustedKeys->applyList (
+                manifest1, blob4, badSig, version));
+
+        // do not apply list with revoked publisher key
+        // applied list is removed due to revoked publisher key
+        auto const signingKeysMax = randomKeyPair(KeyType::secp256k1);
+        auto maxManifest = beast::detail::base64_encode(makeManifestString (
+            publisherPublic, publisherSecret,
+            pubSigningKeys2.first, pubSigningKeys2.second,
+            std::numeric_limits<std::uint32_t>::max ()));
+
+        auto const sequence5 = 5;
+        auto const blob5 = makeList (
+            list1, sequence5, expiration.time_since_epoch().count());
+        auto const sig5 = signList (blob5, signingKeysMax);
+
+        BEAST_EXPECT(ListDisposition::untrusted ==
+            trustedKeys->applyList (
+                maxManifest, blob5, sig5, version));
+
+        BEAST_EXPECT(! trustedKeys->trustedPublisher(publisherPublic));
+        for (auto const& val : list1)
+            BEAST_EXPECT(! trustedKeys->listed (val));
     }
-        // Inserting the same ephemeral key fails
+
-        BEAST_EXPECT(!vl->insertEphemeralKey (v, ""));
+    void
+    testUpdate ()
     {
-            auto member = vl->member (v);
+        testcase ("Update");
-            BEAST_EXPECT(static_cast<bool>(member));
+
-            BEAST_EXPECT(member->compare("Ephemeral") == 0);
+        PublicKey emptyLocalKey;
-        }
+        ManifestCache manifests;
-        // Inserting the same key as permanent fails:
+        jtx::Env env (*this);
-        BEAST_EXPECT(!vl->insertPermanentKey (v, "Permanent"));
+        auto trustedKeys = std::make_unique <ValidatorList> (
+            manifests, manifests, env.timeKeeper(), beast::Journal ());
+
+        std::vector<std::string> cfgPublishers;
+        hash_set<PublicKey> activeValidators;
+
         {
-            auto member = vl->member (v);
+            std::vector<std::string> cfgKeys;
-            BEAST_EXPECT(static_cast<bool>(member));
+            cfgKeys.reserve(20);
-            BEAST_EXPECT(member->compare("Ephemeral") == 0);
+
-        }
+            while (cfgKeys.size () != 20)
-        // Deleting the key as permanent fails:
-        BEAST_EXPECT(!vl->removePermanentKey (v));
             {
-            auto member = vl->member (v);
+                auto const valKey = randomNode();
-            BEAST_EXPECT(static_cast<bool>(member));
+                cfgKeys.push_back (toBase58(
-            BEAST_EXPECT(member->compare("Ephemeral") == 0);
+                    TokenType::TOKEN_NODE_PUBLIC, valKey));
+                if (cfgKeys.size () <= 15)
+                    activeValidators.emplace (valKey);
+            }
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, cfgKeys, cfgPublishers));
+
+            // onConsensusStart should make all available configured
+            // validators trusted
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 12);
+            std::size_t i = 0;
+            for (auto const& val : cfgKeys)
+            {
+                if (auto const valKey = parseBase58<PublicKey>(
+                    TokenType::TOKEN_NODE_PUBLIC, val))
+                {
+                    BEAST_EXPECT(trustedKeys->listed (*valKey));
+                    if (i++ < activeValidators.size ())
+                        BEAST_EXPECT(trustedKeys->trusted (*valKey));
+                    else
+                        BEAST_EXPECT(!trustedKeys->trusted (*valKey));
+                }
+                else
+                    fail ();
+            }
+        }
+        {
+            // update with manifests
+            auto const masterPrivate  = randomSecretKey();
+            auto const masterPublic =
+                derivePublicKey(KeyType::ed25519, masterPrivate);
+
+            std::vector<std::string> cfgKeys ({
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, masterPublic)});
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, cfgKeys, cfgPublishers));
+
+            auto const signingKeys1 = randomKeyPair(KeyType::secp256k1);
+            auto const signingPublic1 = signingKeys1.first;
+            activeValidators.emplace (masterPublic);
+
+            // Should not trust ephemeral signing key if there is no manifest
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->listed (masterPublic));
+            BEAST_EXPECT(trustedKeys->trusted (masterPublic));
+            BEAST_EXPECT(!trustedKeys->listed (signingPublic1));
+            BEAST_EXPECT(!trustedKeys->trusted (signingPublic1));
+
+            // Should trust the ephemeral signing key from the applied manifest
+            auto m1 = Manifest::make_Manifest (makeManifestString (
+                masterPublic, masterPrivate,
+                signingPublic1, signingKeys1.second, 1));
+
+            BEAST_EXPECT(
+                manifests.applyManifest(std::move (*m1)) ==
+                    ManifestDisposition::accepted);
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 13);
+            BEAST_EXPECT(trustedKeys->listed (masterPublic));
+            BEAST_EXPECT(trustedKeys->trusted (masterPublic));
+            BEAST_EXPECT(trustedKeys->listed (signingPublic1));
+            BEAST_EXPECT(trustedKeys->trusted (signingPublic1));
+
+            // Should only trust the ephemeral signing key
+            // from the newest applied manifest
+            auto const signingKeys2 = randomKeyPair(KeyType::secp256k1);
+            auto const signingPublic2 = signingKeys2.first;
+            auto m2 = Manifest::make_Manifest (makeManifestString (
+                masterPublic, masterPrivate,
+                signingPublic2, signingKeys2.second, 2));
+
+            BEAST_EXPECT(
+                manifests.applyManifest(std::move (*m2)) ==
+                    ManifestDisposition::accepted);
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 13);
+            BEAST_EXPECT(trustedKeys->listed (masterPublic));
+            BEAST_EXPECT(trustedKeys->trusted (masterPublic));
+            BEAST_EXPECT(trustedKeys->listed (signingPublic2));
+            BEAST_EXPECT(trustedKeys->trusted (signingPublic2));
+            BEAST_EXPECT(!trustedKeys->listed (signingPublic1));
+            BEAST_EXPECT(!trustedKeys->trusted (signingPublic1));
+
+            // Should not trust keys from revoked master public key
+            auto const signingKeysMax = randomKeyPair(KeyType::secp256k1);
+            auto const signingPublicMax = signingKeysMax.first;
+            activeValidators.emplace (signingPublicMax);
+            auto mMax = Manifest::make_Manifest (makeManifestString (
+                masterPublic, masterPrivate,
+                signingPublicMax, signingKeysMax.second,
+                std::numeric_limits<std::uint32_t>::max ()));
+
+            BEAST_EXPECT(mMax->revoked ());
+            BEAST_EXPECT(
+                manifests.applyManifest(std::move (*mMax)) ==
+                    ManifestDisposition::accepted);
+            BEAST_EXPECT(manifests.getSigningKey (masterPublic) == masterPublic);
+            BEAST_EXPECT(manifests.revoked (masterPublic));
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 12);
+            BEAST_EXPECT(trustedKeys->listed (masterPublic));
+            BEAST_EXPECT(!trustedKeys->trusted (masterPublic));
+            BEAST_EXPECT(!trustedKeys->listed (signingPublicMax));
+            BEAST_EXPECT(!trustedKeys->trusted (signingPublicMax));
+            BEAST_EXPECT(!trustedKeys->listed (signingPublic2));
+            BEAST_EXPECT(!trustedKeys->trusted (signingPublic2));
+            BEAST_EXPECT(!trustedKeys->listed (signingPublic1));
+            BEAST_EXPECT(!trustedKeys->trusted (signingPublic1));
+        }
+        {
+            // Make quorum unattainable if lists from any publishers are unavailable
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), beast::Journal ());
+            auto const publisherSecret = randomSecretKey();
+            auto const publisherPublic =
+                derivePublicKey(KeyType::ed25519, publisherSecret);
+
+            std::vector<std::string> cfgPublishers({
+                strHex(publisherPublic)});
+            std::vector<std::string> emptyCfgKeys;
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, emptyCfgKeys, cfgPublishers));
+
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () ==
+                std::numeric_limits<std::size_t>::max());
+        }
+        {
+            // Trust all listed validators if none are active
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), beast::Journal ());
+
+            std::vector<PublicKey> keys ({ randomNode (), randomNode () });
+            hash_set<PublicKey> activeValidators;
+            std::vector<std::string> cfgKeys ({
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, keys[0]),
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, keys[1])});
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, cfgKeys, cfgPublishers));
+
+            trustedKeys->onConsensusStart (activeValidators);
+
+            BEAST_EXPECT(trustedKeys->quorum () == 2);
+            for (auto const& key : keys)
+                BEAST_EXPECT(trustedKeys->trusted (key));
+        }
+        {
+            // Should use custom minimum quorum
+            std::size_t const minQuorum = 0;
+            ManifestCache manifests;
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), beast::Journal (), minQuorum);
+
+            auto const node = randomNode ();
+            std::vector<std::string> cfgKeys ({
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, node)});
+            hash_set<PublicKey> activeValidators;
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, cfgKeys, cfgPublishers));
+
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == minQuorum);
+
+            activeValidators.emplace (node);
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 1);
+        }
+        {
+            // Increase quorum when running as an unlisted validator
+            auto trustedKeys = std::make_unique <ValidatorList> (
+                manifests, manifests, env.timeKeeper(), beast::Journal ());
+
+            std::vector<PublicKey> keys ({ randomNode (), randomNode () });
+            hash_set<PublicKey> activeValidators ({ keys[0] });
+            std::vector<std::string> cfgKeys ({
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, keys[0]),
+                toBase58 (TokenType::TOKEN_NODE_PUBLIC, keys[1])});
+
+            auto const localKey = randomNode ();
+            BEAST_EXPECT(trustedKeys->load (
+                localKey, cfgKeys, cfgPublishers));
+
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->quorum () == 3);
+
+            // local validator key is always trusted
+            BEAST_EXPECT(trustedKeys->trusted (localKey));
+        }
+        {
+            // Remove expired published list
+            auto trustedKeys = std::make_unique<ValidatorList> (
+                manifests, manifests, env.app().timeKeeper(), beast::Journal ());
+
+            PublicKey emptyLocalKey;
+            std::vector<std::string> emptyCfgKeys;
+            auto const publisherKeys = randomKeyPair(KeyType::secp256k1);
+            auto const pubSigningKeys = randomKeyPair(KeyType::secp256k1);
+            auto const manifest = beast::detail::base64_encode (
+                makeManifestString (
+                    publisherKeys.first, publisherKeys.second,
+                    pubSigningKeys.first, pubSigningKeys.second, 1));
+
+            std::vector<std::string> cfgKeys ({
+                strHex(publisherKeys.first)});
+
+            BEAST_EXPECT(trustedKeys->load (
+                emptyLocalKey, emptyCfgKeys, cfgKeys));
+
+            std::vector<PublicKey> list ({randomNode()});
+            hash_set<PublicKey> activeValidators ({ list[0] });
+
+            // do not apply expired list
+            auto const version = 1;
+            auto const sequence = 1;
+            NetClock::time_point const expiration =
+                env.timeKeeper().now() + 60s;
+            auto const blob = makeList (
+                list, sequence, expiration.time_since_epoch().count());
+            auto const sig = signList (blob, pubSigningKeys);
+
+            BEAST_EXPECT(ListDisposition::accepted ==
+                trustedKeys->applyList (
+                    manifest, blob, sig, version));
+
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(trustedKeys->trusted (list[0]));
+
+            env.timeKeeper().set(expiration);
+            trustedKeys->onConsensusStart (activeValidators);
+            BEAST_EXPECT(! trustedKeys->trusted (list[0]));
         }
-        // Deleting the key as ephemeral succeeds:
-        BEAST_EXPECT(vl->removeEphemeralKey (v));
-        BEAST_EXPECT(!vl->trusted(v));
     }
 
 public:
     void
     run() override
     {
-        testConfigLoad();
+        testGenesisQuorum ();
-        testMembership ();
+        testConfigLoad ();
-        testModification ();
+        testApplyList ();
+        testUpdate ();
     }
 };
 
 BEAST_DEFINE_TESTSUITE(ValidatorList, app, ripple);
 
-} // tests
+} // test
 } // ripple

src/test/core/Config_test.cpp

@@ -285,9 +285,9 @@ public:
     }
 };
 
-std::string valFileContents (boost::optional<int> const& quorum)
+std::string valFileContents ()
 {
-    static boost::format configContentsTemplate (R"rippleConfig(
+    std::string configContents (R"rippleConfig(
 [validators]
 n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7
 n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj
@@ -300,14 +300,11 @@ nHUhG1PgAG8H8myUENypM35JgfqXAKNQvRVVAFDRzJrny5eZN8d5
 nHBu9PTL9dn2GuZtdW4U2WzBwffyX9qsQCd9CNU4Z5YG3PQfViM8
 nHUPDdcdb2Y5DZAJne4c2iabFuAP3F34xZUgYQT2NH7qfkdapgnz
 
-%1%
+[validator_list_keys]
-
+03E74EE14CB525AFBB9F1B7D86CD58ECC4B91452294B42AB4E78F260BD905C091D
+030775A669685BD6ABCEBD80385921C7851783D991A8055FD21D2F3966C96F1B56
 )rippleConfig");
-
+    return configContents;
-    std::string quorumSection =
-        quorum ? "[validation_quorum]\n" + to_string(*quorum) : "";
-    return boost::str (
-        configContentsTemplate % quorumSection);
 }
 
 /**
@@ -321,7 +318,7 @@ private:
 public:
     ValidatorsTxtGuard (beast::unit_test::suite& test,
         path subDir, path const& validatorsFileName,
-            boost::optional<int> const& quorum, bool useCounter = true)
+            bool useCounter = true)
         : ConfigGuard (test, std::move (subDir), useCounter)
     {
         using namespace boost::filesystem;
@@ -332,7 +329,7 @@ public:
         if (!exists (validatorsFile_))
         {
             std::ofstream o (validatorsFile_.string ());
-            o << valFileContents (quorum);
+            o << valFileContents ();
         }
         else
         {
@@ -392,9 +389,6 @@ port_wss_admin
 
 [ssl_verify]
 0
-
-[validation_quorum]
-3
 )rippleConfig");
 
         c.loadFromString (toLoad);
@@ -499,9 +493,8 @@ port_wss_admin
         }
         {
             // load should throw for invalid [validators_file]
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", quorum);
+                *this, "test_cfg", "validators.cfg");
             path const invalidFile = current_path () / vtg.subdir ();
             boost::format cc ("[validators_file]\n%1%\n");
             std::string error;
@@ -517,7 +510,7 @@ port_wss_admin
             BEAST_EXPECT(error == expectedError);
         }
         {
-            // load validators and quorum from config
+            // load validators from config into single section
             Config c;
             std::string toLoad(R"rippleConfig(
 [validators]
@@ -528,53 +521,59 @@ n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C
 [validator_keys]
 nHUhG1PgAG8H8myUENypM35JgfqXAKNQvRVVAFDRzJrny5eZN8d5
 nHBu9PTL9dn2GuZtdW4U2WzBwffyX9qsQCd9CNU4Z5YG3PQfViM8
-
-[validation_quorum]
-4
 )rippleConfig");
             c.loadFromString (toLoad);
             BEAST_EXPECT(c.legacy ("validators_file").empty ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 3);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 2);
+        }
-            BEAST_EXPECT(c.VALIDATION_QUORUM == 4);
+        {
+            // load validator list keys from config
+            Config c;
+            std::string toLoad(R"rippleConfig(
+[validator_list_keys]
+021A99A537FDEBC34E4FCA03B39BEADD04299BB19E85097EC92B15A3518801E566
+)rippleConfig");
+            c.loadFromString (toLoad);
+            BEAST_EXPECT(
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 1);
+            BEAST_EXPECT(
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ()[0] ==
+                    "021A99A537FDEBC34E4FCA03B39BEADD04299BB19E85097EC92B15A3518801E566");
         }
         {
             // load from specified [validators_file] absolute path
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", quorum);
+                *this, "test_cfg", "validators.cfg");
             BEAST_EXPECT(vtg.validatorsFileExists ());
             Config c;
             boost::format cc ("[validators_file]\n%1%\n");
             c.loadFromString (boost::str (cc % vtg.validatorsFile ()));
             BEAST_EXPECT(c.legacy ("validators_file") == vtg.validatorsFile ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 8);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 2);
         }
         {
             // load from specified [validators_file] file name
             // in config directory
-            int const quorum = 3;
             std::string const valFileName = "validators.txt";
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", valFileName, quorum);
+                *this, "test_cfg", valFileName);
             detail::RippledCfgGuard const rcg (
                 *this, vtg.subdir (), "", valFileName, false);
             BEAST_EXPECT(vtg.validatorsFileExists ());
             BEAST_EXPECT(rcg.configFileExists ());
             auto const& c (rcg.config ());
             BEAST_EXPECT(c.legacy ("validators_file") == valFileName);
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 8);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 2);
         }
         {
             // load from specified [validators_file] relative path
             // to config directory
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.txt", quorum);
+                *this, "test_cfg", "validators.txt");
             auto const valFilePath = ".." / vtg.subdir() / "validators.txt";
             detail::RippledCfgGuard const rcg (
                 *this, vtg.subdir (), "", valFilePath, false);
@@ -582,63 +581,41 @@ nHBu9PTL9dn2GuZtdW4U2WzBwffyX9qsQCd9CNU4Z5YG3PQfViM8
             BEAST_EXPECT(rcg.configFileExists ());
             auto const& c (rcg.config ());
             BEAST_EXPECT(c.legacy ("validators_file") == valFilePath);
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 8);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 2);
         }
         {
             // load from validators file in default location
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.txt", quorum);
+                *this, "test_cfg", "validators.txt");
             detail::RippledCfgGuard const rcg (*this, vtg.subdir (),
                 "", "", false);
             BEAST_EXPECT(vtg.validatorsFileExists ());
             BEAST_EXPECT(rcg.configFileExists ());
             auto const& c (rcg.config ());
             BEAST_EXPECT(c.legacy ("validators_file").empty ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 8);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 2);
         }
         {
             // load from specified [validators_file] instead
             // of default location
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", quorum);
+                *this, "test_cfg", "validators.cfg");
             BEAST_EXPECT(vtg.validatorsFileExists ());
             detail::ValidatorsTxtGuard const vtgDefault (
-                *this, vtg.subdir (), "validators.txt", 4, false);
+                *this, vtg.subdir (), "validators.txt", false);
             BEAST_EXPECT(vtgDefault.validatorsFileExists ());
             detail::RippledCfgGuard const rcg (
                 *this, vtg.subdir (), "", vtg.validatorsFile (), false);
             BEAST_EXPECT(rcg.configFileExists ());
             auto const& c (rcg.config ());
             BEAST_EXPECT(c.legacy ("validators_file") == vtg.validatorsFile ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 8);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 2);
-        }
-        {
-            // do not load quorum from validators file if in config
-            boost::format cc (R"rippleConfig(
-[validators_file]
-%1%
-
-[validation_quorum]
-4
-)rippleConfig");
-            int const quorum = 3;
-            detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", quorum);
-            BEAST_EXPECT(vtg.validatorsFileExists ());
-            Config c;
-            c.loadFromString (boost::str (cc % vtg.validatorsFile ()));
-            BEAST_EXPECT(c.legacy ("validators_file") == vtg.validatorsFile ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 5);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 3);
-            BEAST_EXPECT(c.VALIDATION_QUORUM == 4);
         }
 
         {
@@ -658,51 +635,34 @@ n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA
 nHB1X37qrniVugfQcuBTAjswphC1drx7QjFFojJPZwKHHnt8kU7v
 nHUkAWDR4cB8AgPg7VXMX6et8xRTQb2KJfgv1aBEXozwrawRKgMB
 
+[validator_list_keys]
+021A99A537FDEBC34E4FCA03B39BEADD04299BB19E85097EC92B15A3518801E566
 )rippleConfig");
-            int const quorum = 3;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", quorum);
+                *this, "test_cfg", "validators.cfg");
             BEAST_EXPECT(vtg.validatorsFileExists ());
             Config c;
             c.loadFromString (boost::str (cc % vtg.validatorsFile ()));
             BEAST_EXPECT(c.legacy ("validators_file") == vtg.validatorsFile ());
-            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 10);
+            BEAST_EXPECT(c.section (SECTION_VALIDATORS).values ().size () == 15);
-            BEAST_EXPECT(c.section (SECTION_VALIDATOR_KEYS).values ().size () == 5);
+            BEAST_EXPECT(
-            BEAST_EXPECT(c.VALIDATION_QUORUM == quorum);
+                c.section (SECTION_VALIDATOR_LIST_KEYS).values ().size () == 3);
         }
         {
-            // load should throw if [validators] and [validator_keys] are
+            // load should throw if [validators], [validator_keys] and
-            // missing from rippled cfg and validators file
+            // [validator_list_keys] are missing from rippled cfg and
+            // validators file
+            Config c;
             boost::format cc ("[validators_file]\n%1%\n");
             std::string error;
             detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", boost::none);
+                *this, "test_cfg", "validators.cfg");
             BEAST_EXPECT(vtg.validatorsFileExists ());
             auto const expectedError =
                 "The file specified in [validators_file] does not contain a "
-                "[validators] or [validator_keys] section: " +
+                "[validators], [validator_keys] or [validator_list_keys] section: " +
                 vtg.validatorsFile ();
             std::ofstream o (vtg.validatorsFile ());
-            o << "[validation_quorum]\n3\n";
-            try {
-                Config c;
-                c.loadFromString (boost::str (cc % vtg.validatorsFile ()));
-            } catch (std::runtime_error& e) {
-                error = e.what();
-            }
-            BEAST_EXPECT(error == expectedError);
-        }
-        {
-            // load should throw if [validation_quorum] is
-            // missing from rippled cfg and validators file
-            boost::format cc ("[validators_file]\n%1%\n");
-            std::string error;
-            detail::ValidatorsTxtGuard const vtg (
-                *this, "test_cfg", "validators.cfg", boost::none);
-            BEAST_EXPECT(vtg.validatorsFileExists ());
-            auto const expectedError =
-                "The file specified in [validators_file] does not contain a "
-                "[validation_quorum] section: " + vtg.validatorsFile ();
             try {
                 Config c;
                 c.loadFromString (boost::str (cc % vtg.validatorsFile ()));

src/test/rpc/ServerInfo_test.cpp

@@ -57,10 +57,13 @@ public:
 
 [validation_seed]
 %2%
+
+[validators]
+%3%
 )rippleConfig");
 
         p->loadFromString (boost::str (
-            toLoad % validator::manifest % validator::seed));
+            toLoad % validator::manifest % validator::seed % validator::master_key));
 
         setupConfigForUnitTests(*p);
 

src/test/unity/app_test_unity.cpp

@@ -28,6 +28,7 @@
 #include <test/app/HashRouter_test.cpp>
 #include <test/app/LedgerLoad_test.cpp>
 #include <test/app/LoadFeeTrack_test.cpp>
+#include <test/app/Manifest_test.cpp>
 #include <test/app/MultiSign_test.cpp>
 #include <test/app/OfferStream_test.cpp>
 #include <test/app/Offer_test.cpp>

src/test/unity/overlay_test_unity.cpp

@@ -19,6 +19,5 @@
 //==============================================================================
 
 #include <test/overlay/cluster_test.cpp>
-#include <test/overlay/manifest_test.cpp>
 #include <test/overlay/short_read_test.cpp>
 #include <test/overlay/TMHello_test.cpp>