Clear old Validations during online delete (RIPD-870):

* Add Validations.LedgerSeq and .InitialSeq fields.
* Clean up logging.
* Lower online delete minimum for standalone mode.
* Unit tests of online_delete.

Builds/VisualStudio2015/RippleD.vcxproj

@@ -1531,6 +1531,10 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\app\tests\SHAMapStore_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple\app\tests\SusPay_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>

Builds/VisualStudio2015/RippleD.vcxproj.filters

@@ -2112,6 +2112,9 @@
     <ClCompile Include="..\..\src\ripple\app\tests\SetAuth_test.cpp">
       <Filter>ripple\app\tests</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\app\tests\SHAMapStore_test.cpp">
+      <Filter>ripple\app\tests</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple\app\tests\SusPay_test.cpp">
       <Filter>ripple\app\tests</Filter>
     </ClCompile>

src/ripple/app/ledger/Ledger.cpp

@@ -890,16 +890,6 @@ static bool saveValidatedLedger (
         "DELETE FROM AccountTransactions WHERE LedgerSeq = %u;");
     static boost::format deleteAcctTrans (
         "DELETE FROM AccountTransactions WHERE TransID = '%s';");
-    static boost::format transExists (
-        "SELECT Status FROM Transactions WHERE TransID = '%s';");
-    static boost::format updateTx (
-        "UPDATE Transactions SET LedgerSeq = %u, Status = '%c', TxnMeta = %s "
-        "WHERE TransID = '%s';");
-    static boost::format addLedger (
-        "INSERT OR REPLACE INTO Ledgers "
-        "(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,"
-        "CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash) VALUES "
-        "('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');");
 
     auto seq = ledger->info().seq;
 
@@ -1032,18 +1022,53 @@ static bool saveValidatedLedger (
     }
 
     {
+        static std::string addLedger(
+            R"sql(INSERT OR REPLACE INTO Ledgers
+                (LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,
+                CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash)
+            VALUES
+                (:ledgerHash,:ledgerSeq,:prevHash,:totalCoins,:closingTime,:prevClosingTime,
+                :closeTimeRes,:closeFlags,:accountSetHash,:transSetHash);)sql");
+        static std::string updateVal(
+            R"sql(UPDATE Validations SET LedgerSeq = :ledgerSeq, InitialSeq = :initialSeq
+                WHERE LedgerHash = :ledgerHash;)sql");
+
         auto db (app.getLedgerDB ().checkoutDb ());
 
-        // TODO(tom): ARG!
+        soci::transaction tr(*db);
-        *db << boost::str (
+
-            addLedger %
+        auto const hash = to_string (ledger->info().hash);
-            to_string (ledger->info().hash) % seq % to_string (ledger->info().parentHash) %
+        auto const parentHash = to_string (ledger->info().parentHash);
-            to_string (ledger->info().drops) %
+        auto const drops = to_string (ledger->info().drops);
-            ledger->info().closeTime.time_since_epoch().count() %
+        auto const closeTime =
-            ledger->info().parentCloseTime.time_since_epoch().count() %
+            ledger->info().closeTime.time_since_epoch().count();
-            ledger->info().closeTimeResolution.count() %
+        auto const parentCloseTime =
-            ledger->info().closeFlags % to_string (ledger->info().accountHash) %
+            ledger->info().parentCloseTime.time_since_epoch().count();
-            to_string (ledger->info().txHash));
+        auto const closeTimeResolution =
+            ledger->info().closeTimeResolution.count();
+        auto const closeFlags = ledger->info().closeFlags;
+        auto const accountHash = to_string (ledger->info().accountHash);
+        auto const txHash = to_string (ledger->info().txHash);
+
+        *db << addLedger,
+            soci::use(hash),
+            soci::use(seq),
+            soci::use(parentHash),
+            soci::use(drops),
+            soci::use(closeTime),
+            soci::use(parentCloseTime),
+            soci::use(closeTimeResolution),
+            soci::use(closeFlags),
+            soci::use(accountHash),
+            soci::use(txHash);
+
+
+        *db << updateVal,
+            soci::use(seq),
+            soci::use(seq),
+            soci::use(hash);
+
+        tr.commit();
     }
 
     // Clients can now trust the database for

src/ripple/app/ledger/LedgerHistory.cpp

@@ -499,7 +499,9 @@ void LedgerHistory::clearLedgerCachePrior (LedgerIndex seq)
 {
     for (LedgerHash it: m_ledgers_by_hash.getKeys())
     {
-        if (getLedgerByHash (it)->info().seq < seq)
+        auto const ledger = getLedgerByHash (it);
+        assert(ledger);
+        if (!ledger || ledger->info().seq < seq)
             m_ledgers_by_hash.del (it, false);
     }
 }

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -984,7 +984,7 @@ LedgerMaster::getLedgerHashForHistory (LedgerIndex index)
     if (! ret)
         ret = walkHashBySeq (index);
 
-    return *ret;
+    return ret;
 }
 
 bool

src/ripple/app/main/DBInit.cpp

@@ -90,14 +90,29 @@ const char* LedgerDBInit[] =
     );",
     "CREATE INDEX IF NOT EXISTS SeqLedger ON Ledgers(LedgerSeq);",
 
+    // InitialSeq field is the current ledger seq when the row
+    // is inserted. Only relevant during online delete
     "CREATE TABLE IF NOT EXISTS Validations   (                   \
+        LedgerSeq   BIGINT UNSIGNED,                \
+        InitialSeq  BIGINT UNSIGNED,                \
         LedgerHash  CHARACTER(64),                  \
         NodePubKey  CHARACTER(56),                  \
         SignTime    BIGINT UNSIGNED,                \
         RawData     BLOB                            \
     );",
+    // This will error out if the column already exists,
+    //  but DatabaseCon intentionally ignores errors.
+    "ALTER TABLE Validations                        \
+        ADD COLUMN LedgerSeq       BIGINT UNSIGNED;",
+    "ALTER TABLE Validations                        \
+        ADD COLUMN InitialSeq      BIGINT UNSIGNED;",
+
     "CREATE INDEX IF NOT EXISTS ValidationsByHash ON              \
         Validations(LedgerHash);",
+    "CREATE INDEX IF NOT EXISTS ValidationsBySeq ON              \
+        Validations(LedgerSeq);",
+    "CREATE INDEX IF NOT EXISTS ValidationsByInitialSeq ON              \
+        Validations(InitialSeq, LedgerSeq);",
     "CREATE INDEX IF NOT EXISTS ValidationsByTime ON              \
         Validations(SignTime);",
 

src/ripple/app/misc/SHAMapStore.h

@@ -41,6 +41,7 @@ class SHAMapStore
 public:
     struct Setup
     {
+        bool standalone = false;
         std::uint32_t deleteInterval = 0;
         bool advisoryDelete = false;
         std::uint32_t ledgerHistory = 0;

src/ripple/app/misc/SHAMapStoreImp.cpp

@@ -29,6 +29,7 @@
 #include <boost/format.hpp>
 #include <boost/optional.hpp>
 #include <memory>
+#include <chrono>
 
 namespace ripple {
 void SHAMapStoreImp::SavedStateDB::init (BasicConfig const& config,
@@ -179,21 +180,24 @@ SHAMapStoreImp::SHAMapStoreImp (
     , scheduler_ (scheduler)
     , journal_ (journal)
     , nodeStoreJournal_ (nodeStoreJournal)
+    , rotating_(false)
     , transactionMaster_ (transactionMaster)
     , canDelete_ (std::numeric_limits <LedgerIndex>::max())
 {
     if (setup_.deleteInterval)
     {
-        if (setup_.deleteInterval < minimumDeletionInterval_)
+        auto const minInterval = setup.standalone ?
+            minimumDeletionIntervalSA_ : minimumDeletionInterval_;
+        if (setup_.deleteInterval < minInterval)
         {
             Throw<std::runtime_error> ("online_delete must be at least " +
-                std::to_string (minimumDeletionInterval_));
+                std::to_string (minInterval));
         }
 
         if (setup_.ledgerHistory > setup_.deleteInterval)
         {
             Throw<std::runtime_error> (
-                "online_delete must be less than ledger_history (currently " +
+                "online_delete must not be less than ledger_history (currently " +
                 std::to_string (setup_.ledgerHistory) + ")");
         }
 
@@ -283,7 +287,8 @@ SHAMapStoreImp::run()
     while (1)
     {
         healthy_ = true;
-        validatedLedger_.reset();
+        std::shared_ptr<Ledger const> validatedLedger;
+        rotating_ = false;
 
         {
             std::unique_lock <std::mutex> lock (mutex_);
@@ -294,12 +299,15 @@ SHAMapStoreImp::run()
             }
             cond_.wait (lock);
             if (newLedger_)
-                validatedLedger_ = std::move (newLedger_);
+            {
+                rotating_ = true;
+                validatedLedger = std::move(newLedger_);
+            }
             else
                 continue;
         }
 
-        LedgerIndex validatedSeq = validatedLedger_->info().seq;
+        LedgerIndex validatedSeq = validatedLedger->info().seq;
         if (!lastRotated)
         {
             lastRotated = validatedSeq;
@@ -340,7 +348,7 @@ SHAMapStoreImp::run()
             }
 
             std::uint64_t nodeCount = 0;
-            validatedLedger_->stateMap().snapShot (
+            validatedLedger->stateMap().snapShot (
                     false)->visitNodes (
                     std::bind (&SHAMapStoreImp::copyNode, this,
                     std::ref(nodeCount), std::placeholders::_1));
@@ -504,7 +512,7 @@ SHAMapStoreImp::makeDatabaseRotating (std::string const& name,
             readThreads, writableBackend, archiveBackend, nodeStoreJournal_);
 }
 
-void
+bool
 SHAMapStoreImp::clearSql (DatabaseCon& database,
         LedgerIndex lastRotated,
         std::string const& minQuery,
@@ -517,12 +525,12 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
         boost::optional<std::uint64_t> m;
         *db << minQuery, soci::into(m);
         if (!m)
-            return;
+            return false;
         min = *m;
     }
 
-    if (health() != Health::ok)
+    if(min > lastRotated || health() != Health::ok)
-        return;
+        return false;
 
     boost::format formattedDeleteQuery (deleteQuery);
 
@@ -530,19 +538,19 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
         "start: " << deleteQuery << " from " << min << " to " << lastRotated;
     while (min < lastRotated)
     {
-        min = (min + setup_.deleteBatch >= lastRotated) ? lastRotated :
+        min = std::min(lastRotated, min + setup_.deleteBatch);
-            min + setup_.deleteBatch;
         {
             auto db =  database.checkoutDb ();
             *db << boost::str (formattedDeleteQuery % min);
         }
         if (health())
-            return;
+            return true;
         if (min < lastRotated)
             std::this_thread::sleep_for (
                     std::chrono::milliseconds (setup_.backOff));
     }
     JLOG(journal_.debug) << "finished: " << deleteQuery;
+    return true;
 }
 
 void
@@ -566,25 +574,10 @@ SHAMapStoreImp::freshenCaches()
 void
 SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
 {
-    ledgerMaster_->clearPriorLedgers (lastRotated);
     if (health())
         return;
 
-    // TODO This won't remove validations for ledgers that do not get
+    ledgerMaster_->clearPriorLedgers (lastRotated);
-    // validated. That will likely require inserting LedgerSeq into
-    // the validations table.
-    //
-    // This query has poor performance with large data sets.
-    // The schema needs to be redesigned to avoid the JOIN, or an
-    // RDBMS that supports concurrency should be used.
-    /*
-    clearSql (*ledgerDb_, lastRotated,
-        "SELECT MIN(LedgerSeq) FROM Ledgers;",
-        "DELETE FROM Validations WHERE LedgerHash IN "
-        "(SELECT Ledgers.LedgerHash FROM Validations JOIN Ledgers ON "
-        "Validations.LedgerHash=Ledgers.LedgerHash WHERE Ledgers.LedgerSeq < %u);");
-     */
-
     if (health())
         return;
 
@@ -594,6 +587,118 @@ SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
     if (health())
         return;
 
+    {
+        /*
+            Steps for migration:
+            Assume: online_delete = 100, lastRotated = 1000,
+                Last shutdown was at ledger # 1080.
+                The current network validated ledger is 1090.
+            Implies: Ledgers has entries from 900 to 1080.
+                Validations has entries for all 1080 ledgers,
+                including orphan validations that were not included
+                in a validated ledger.
+            1) Columns are created in Validations with default NULL values.
+            2) During syncing, Ledgers and Validations for 1080 - 1090
+               are received from the network. Records are created in
+               Validations with InitialSeq approximately 1080 (exact value
+               doesn't matter), and later validated with the matching
+               LedgerSeq value.
+            3) rippled participates in ledgers 1091-1100. Validations
+                received are created with InitialSeq in that range, and
+                appropriate LedgerSeqs. Maybe some of those ledgers are
+                not accepted, so LedgerSeq stays null.
+            4) At ledger 1100, this function is called with
+                lastRotated = 1000. The first query tries to delete
+                rows WHERE LedgerSeq < 1000. It finds none.
+            5) The second round of deletions does not run.
+            6) Ledgers continue to advance from 1100-1200 as described
+                in step 3.
+            7) At ledger 1200, this function is called again with
+                lastRotated = 1100. The first query again tries to delete
+                rows WHERE LedgerSeq < 1100. It finds the rows for 1080-1099.
+            8) The second round of deletions runs. It gets
+                WHERE v.LedgerSeq is NULL AND
+                (v.InitialSeq IS NULL OR v.InitialSeq < 1100)
+                The rows that are found include (a) ALL of the Validations
+                for the first 1080 ledgers. (b) Any orphan validations that
+                were created in step 3.
+            9) This continues. The next rotation cycle does the same as steps
+                7 & 8, except that none of the original Validations (8a) exist
+                anymore, and 8b gets the orphans from step 6.
+        */
+
+        static auto anyValDeleted = false;
+        auto const valDeleted = clearSql(*ledgerDb_, lastRotated,
+            "SELECT MIN(LedgerSeq) FROM Validations;",
+            "DELETE FROM Validations WHERE LedgerSeq < %u;");
+        anyValDeleted |= valDeleted;
+
+        if (health())
+            return;
+
+        if (anyValDeleted)
+        {
+            /* Delete the old NULL LedgerSeqs - the Validations that
+               aren't linked to a validated ledger - but only if we
+               deleted rows in the matching `clearSql` call, and only
+               for those created with an old InitialSeq.
+            */
+            using namespace std::chrono;
+            auto const deleteBatch = setup_.deleteBatch;
+            auto const continueLimit = (deleteBatch + 1) / 2;
+
+            std::string const deleteQuery(
+                R"sql(DELETE FROM Validations
+                WHERE LedgerHash IN
+                (
+                    SELECT v.LedgerHash
+                    FROM Validations v
+                    WHERE v.LedgerSeq is NULL AND
+                        (v.InitialSeq IS NULL OR v.InitialSeq < )sql" +
+                    std::to_string(lastRotated) +
+                    ") LIMIT " +
+                    std::to_string (deleteBatch) +
+                ");");
+
+            JLOG(journal_.debug) << "start: " << deleteQuery << " of "
+                << deleteBatch << " rows.";
+            long long totalRowsAffected = 0;
+            long long rowsAffected;
+            soci::statement st = [&]
+            {
+                auto db = ledgerDb_->checkoutDb();
+                return (db->prepare << deleteQuery);
+            }();
+            if (health())
+                return;
+            do
+            {
+                {
+                    auto db = ledgerDb_->checkoutDb();
+                    auto const start = high_resolution_clock::now();
+                    st.execute(true);
+                    rowsAffected = st.get_affected_rows();
+                    totalRowsAffected += rowsAffected;
+                    auto const ms = duration_cast<milliseconds>(
+                        high_resolution_clock::now() - start).count();
+                    JLOG(journal_.trace) << "step: deleted " << rowsAffected
+                        << " rows in " << ms << "ms.";
+                }
+                if (health())
+                    return;
+                if (rowsAffected >= continueLimit)
+                    std::this_thread::sleep_for(
+                        std::chrono::milliseconds(setup_.backOff));
+            }
+            while (rowsAffected && rowsAffected >= continueLimit);
+            JLOG(journal_.debug) << "finished: " << deleteQuery << ". Deleted "
+                << totalRowsAffected << " rows.";
+        }
+    }
+
+    if (health())
+        return;
+
     clearSql (*transactionDb_, lastRotated,
         "SELECT MIN(LedgerSeq) FROM Transactions;",
         "DELETE FROM Transactions WHERE LedgerSeq < %u;");
@@ -675,6 +780,8 @@ setup_SHAMapStore (Config const& c)
 {
     SHAMapStore::Setup setup;
 
+    setup.standalone = c.RUN_STANDALONE;
+
     // Get existing settings and add some default values if not specified:
     setup.nodeDatabase = c.section (ConfigSection::nodeDatabase ());
 

src/ripple/app/misc/SHAMapStoreImp.h

@@ -80,7 +80,9 @@ private:
     // check health/stop status as records are copied
     std::uint64_t const checkHealthInterval_ = 1000;
     // minimum # of ledgers to maintain for health of network
-    std::uint32_t minimumDeletionInterval_ = 256;
+    static std::uint32_t const minimumDeletionInterval_ = 256;
+    // minimum # of ledgers required for standalone mode.
+    static std::uint32_t const minimumDeletionIntervalSA_ = 8;
 
     Setup setup_;
     NodeStore::Scheduler& scheduler_;
@@ -94,7 +96,7 @@ private:
     mutable std::condition_variable cond_;
     mutable std::mutex mutex_;
     std::shared_ptr<Ledger const> newLedger_;
-    std::shared_ptr<Ledger const> validatedLedger_;
+    std::atomic<bool> rotating_;
     TransactionMaster& transactionMaster_;
     std::atomic <LedgerIndex> canDelete_;
     // these do not exist upon SHAMapStore creation, but do exist
@@ -106,6 +108,12 @@ private:
     DatabaseCon* transactionDb_ = nullptr;
     DatabaseCon* ledgerDb_ = nullptr;
 
+public:
+    bool rotating() const
+    {
+        return rotating_;
+    }
+
 public:
     SHAMapStoreImp (Application& app,
             Setup const& setup,
@@ -203,8 +211,10 @@ private:
     /** delete from sqlite table in batches to not lock the db excessively
      *  pause briefly to extend access time to other users
      *  call with mutex object unlocked
+     *  @return true if any deletable rows were found (though not
+     *      necessarily deleted.
      */
-    void clearSql (DatabaseCon& database, LedgerIndex lastRotated,
+    bool clearSql (DatabaseCon& database, LedgerIndex lastRotated,
                    std::string const& minQuery, std::string const& deleteQuery);
     void clearCaches (LedgerIndex validatedSeq);
     void freshenCaches();

src/ripple/app/misc/Validations.cpp

@@ -461,8 +461,10 @@ private:
     void doWrite ()
     {
         LoadEvent::autoptr event (app_.getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite"));
-        boost::format insVal ("INSERT INTO Validations "
+        std::string insVal ("INSERT INTO Validations "
-                              "(LedgerHash,NodePubKey,SignTime,RawData) VALUES ('%s','%s','%u',%s);");
+            "(InitialSeq, LedgerSeq, LedgerHash,NodePubKey,SignTime,RawData) "
+            "VALUES (:initialSeq, :ledgerSeq, :ledgerHash,:nodePubKey,:signTime,:rawData);");
+        std::string findSeq("SELECT LedgerSeq FROM Ledgers WHERE Ledgerhash=:ledgerHash;");
 
         ScopedLockType sl (mLock);
         assert (mWriting);
@@ -484,13 +486,34 @@ private:
                     {
                         s.erase ();
                         it->add (s);
-                        *db << boost::str (
+
-                            insVal % to_string (it->getLedgerHash ()) %
+                        auto const ledgerHash = to_string(it->getLedgerHash());
-                            toBase58(
+
+                        boost::optional<std::uint64_t> ledgerSeq;
+                        *db << findSeq, soci::use(ledgerHash),
+                            soci::into(ledgerSeq);
+
+                        auto const initialSeq = ledgerSeq.value_or(
+                            app_.getLedgerMaster().getCurrentLedgerIndex());
+                        auto const nodePubKey = toBase58(
                             TokenType::TOKEN_NODE_PUBLIC,
-                                it->getSignerPublic ()) %
+                            it->getSignerPublic());
-                            it->getSignTime().time_since_epoch().count() %
+                        auto const signTime =
-                            sqlEscape (s.peekData ()));
+                            it->getSignTime().time_since_epoch().count();
+
+                        soci::blob rawData(*db);
+                        rawData.append(reinterpret_cast<const char*>(
+                            s.peekData().data()), s.peekData().size());
+                        assert(rawData.get_len() == s.peekData().size());
+
+                        *db <<
+                            insVal,
+                            soci::use(initialSeq),
+                            soci::use(ledgerSeq),
+                            soci::use(ledgerHash),
+                            soci::use(nodePubKey),
+                            soci::use(signTime),
+                            soci::use(rawData);
                     }
 
                     tr.commit ();

src/ripple/app/tests/SHAMapStore_test.cpp

@@ -0,0 +1,654 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012-2015 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <BeastConfig.h>
+#include <ripple/app/main/Application.h>
+#include <ripple/app/misc/SHAMapStoreImp.h>
+#include <ripple/core/ConfigSections.h>
+#include <ripple/protocol/JsonFields.h>
+#include <ripple/test/jtx.h>
+
+namespace ripple {
+namespace test {
+
+class SHAMapStore_test : public beast::unit_test::suite
+{
+    static auto const deleteInterval = 8;
+
+    static
+    std::unique_ptr<Config>
+    makeConfig()
+    {
+        auto p = std::make_unique<Config>();
+        setupConfigForUnitTests(*p);
+        p->LEDGER_HISTORY = deleteInterval;
+        auto& section = p->section(ConfigSection::nodeDatabase());
+        section.set("online_delete", to_string(deleteInterval));
+        //section.set("age_threshold", "60");
+        return p;
+    }
+
+    static
+    std::unique_ptr<Config>
+    makeConfigAdvisory()
+    {
+        auto p = makeConfig();
+        auto& section = p->section(ConfigSection::nodeDatabase());
+        section.set("advisory_delete", "1");
+        return p;
+    }
+
+    bool goodLedger(jtx::Env& env, Json::Value const& json,
+        std::string ledgerID, bool checkDB = false)
+    {
+        auto good = json.isMember(jss::result)
+            && !RPC::contains_error(json[jss::result])
+            && json[jss::result][jss::ledger][jss::ledger_index] == ledgerID;
+        if (!good || !checkDB)
+            return good;
+
+        auto const seq = json[jss::result][jss::ledger_index].asUInt();
+        std::string outHash;
+        LedgerIndex outSeq;
+        std::string outParentHash;
+        std::string outDrops;
+        std::uint64_t outCloseTime;
+        std::uint64_t outParentCloseTime;
+        std::uint64_t outCloseTimeResolution;
+        std::uint64_t outCloseFlags;
+        std::string outAccountHash;
+        std::string outTxHash;
+
+        {
+            auto db = env.app().getLedgerDB().checkoutDb();
+
+            *db << "SELECT LedgerHash,LedgerSeq,PrevHash,TotalCoins, "
+                "ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags, "
+                "AccountSetHash,TransSetHash "
+                "FROM Ledgers "
+                "WHERE LedgerSeq = :seq",
+                soci::use(seq),
+                soci::into(outHash),
+                soci::into(outSeq),
+                soci::into(outParentHash),
+                soci::into(outDrops),
+                soci::into(outCloseTime),
+                soci::into(outParentCloseTime),
+                soci::into(outCloseTimeResolution),
+                soci::into(outCloseFlags),
+                soci::into(outAccountHash),
+                soci::into(outTxHash);
+        }
+
+        auto const& ledger = json[jss::result][jss::ledger];
+        return outHash == ledger[jss::hash].asString() &&
+            outSeq == seq &&
+            outParentHash == ledger[jss::parent_hash].asString() &&
+            outDrops == ledger[jss::total_coins].asString() &&
+            outCloseTime == ledger[jss::close_time].asUInt() &&
+            outParentCloseTime == ledger[jss::parent_close_time].asUInt() &&
+            outCloseTimeResolution == ledger[jss::close_time_resolution].asUInt() &&
+            outCloseFlags == ledger[jss::close_flags].asUInt() &&
+            outAccountHash == ledger[jss::account_hash].asString() &&
+            outTxHash == ledger[jss::transaction_hash].asString();
+    }
+
+    bool bad(Json::Value const& json, error_code_i error = rpcLGR_NOT_FOUND)
+    {
+        return json.isMember(jss::result)
+            && RPC::contains_error(json[jss::result])
+            && json[jss::result][jss::error_code] == error;
+    }
+
+    std::string getHash(Json::Value const& json)
+    {
+        expect(json.isMember(jss::result) &&
+            json[jss::result].isMember(jss::ledger) &&
+            json[jss::result][jss::ledger].isMember(jss::hash) &&
+            json[jss::result][jss::ledger][jss::hash].isString());
+        return json[jss::result][jss::ledger][jss::hash].asString();
+    }
+
+    void validationCheck(jtx::Env& env, int const expected)
+    {
+        auto db = env.app().getLedgerDB().checkoutDb();
+
+        int actual;
+        *db << "SELECT count(*) AS rows FROM Validations;",
+            soci::into(actual);
+
+        expect(actual == expected);
+
+    }
+
+    void ledgerCheck(jtx::Env& env, int const rows,
+        int const first)
+    {
+        auto db = env.app().getLedgerDB().checkoutDb();
+
+        int actualRows, actualFirst, actualLast;
+        *db << "SELECT count(*) AS rows, "
+            "min(LedgerSeq) as first, "
+            "max(LedgerSeq) as last "
+            "FROM Ledgers;",
+            soci::into(actualRows),
+            soci::into(actualFirst),
+            soci::into(actualLast);
+
+        expect(actualRows == rows);
+        expect(actualFirst == first);
+        expect(actualLast == first + rows - 1);
+
+    }
+
+    void transactionCheck(jtx::Env& env, int const rows)
+    {
+        auto db = env.app().getTxnDB().checkoutDb();
+
+        int actualRows;
+        *db << "SELECT count(*) AS rows "
+            "FROM Transactions;",
+            soci::into(actualRows);
+
+        expect(actualRows == rows);
+    }
+
+    void accountTransactionCheck(jtx::Env& env, int const rows)
+    {
+        auto db = env.app().getTxnDB().checkoutDb();
+
+        int actualRows;
+        *db << "SELECT count(*) AS rows "
+            "FROM AccountTransactions;",
+            soci::into(actualRows);
+
+        expect(actualRows == rows);
+    }
+
+    int waitForReady(jtx::Env& env)
+    {
+        using namespace std::chrono_literals;
+
+        auto& store = env.app().getSHAMapStore();
+
+        int ledgerSeq = 3;
+        while (!store.getLastRotated())
+        {
+            env.close();
+            std::this_thread::sleep_for(100ms);
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++)));
+        }
+        return ledgerSeq;
+    }
+
+public:
+    void testClear()
+    {
+        using namespace std::chrono_literals;
+
+        testcase("clearPrior");
+        using namespace jtx;
+
+        Env env(*this, makeConfig());
+
+        auto store = dynamic_cast<SHAMapStoreImp*>(
+            &env.app().getSHAMapStore());
+        expect(store);
+        env.fund(XRP(10000), noripple("alice"));
+
+        validationCheck(env, 0);
+        ledgerCheck(env, 1, 2);
+        transactionCheck(env, 0);
+        accountTransactionCheck(env, 0);
+
+        std::map<std::uint32_t, Json::Value const> ledgers;
+
+        auto ledgerTmp = env.rpc("ledger", "0");
+        expect(bad(ledgerTmp));
+
+        ledgers.emplace(std::make_pair(1, env.rpc("ledger", "1")));
+        expect(goodLedger(env, ledgers[1], "1"));
+
+        ledgers.emplace(std::make_pair(2, env.rpc("ledger", "2")));
+        expect(goodLedger(env, ledgers[2], "2"));
+
+        ledgerTmp = env.rpc("ledger", "current");
+        expect(goodLedger(env, ledgerTmp, "3"));
+
+        ledgerTmp = env.rpc("ledger", "4");
+        expect(bad(ledgerTmp));
+
+        ledgerTmp = env.rpc("ledger", "100");
+        expect(bad(ledgerTmp));
+
+        for (auto i = 4; i < deleteInterval + 4; ++i)
+        {
+            env.fund(XRP(10000), noripple("test" + to_string(i)));
+            env.close();
+
+            ledgerTmp = env.rpc("ledger", "current");
+            expect(goodLedger(env, ledgerTmp, to_string(i)));
+        }
+        assert(store->getLastRotated() == 3);
+
+        for (auto i = 3; i < deleteInterval + 3; ++i)
+        {
+            ledgers.emplace(std::make_pair(i,
+                env.rpc("ledger", to_string(i))));
+            expect(goodLedger(env, ledgers[i], to_string(i), true) &&
+                getHash(ledgers[i]).length());
+        }
+
+        validationCheck(env, 0);
+        ledgerCheck(env, deleteInterval + 1, 2);
+        transactionCheck(env, deleteInterval + 1);
+        accountTransactionCheck(env, 2*(deleteInterval + 1));
+
+        {
+            // Since standalone doesn't _do_ validations, manually
+            // insert some into the table. Create some with the
+            // hashes from our real ledgers, and some with fake
+            // hashes to represent validations that never ended up
+            // in a validated ledger.
+            char lh[65];
+            memset(lh, 'a', 64);
+            lh[64] = '\0';
+            std::vector<std::string> preSeqLedgerHashes({
+                lh
+            });
+            std::vector<std::string> badLedgerHashes;
+            std::vector<LedgerIndex> badLedgerSeqs;
+            std::vector<std::string> ledgerHashes;
+            std::vector<LedgerIndex> ledgerSeqs;
+            for (auto const& lgr : ledgers)
+            {
+                ledgerHashes.emplace_back(getHash(lgr.second));
+                ledgerSeqs.emplace_back(lgr.second[jss::result][jss::ledger_index].asUInt());
+            }
+            for (auto i = 0; i < 10; ++i)
+            {
+                ++lh[30];
+                preSeqLedgerHashes.emplace_back(lh);
+                ++lh[20];
+                badLedgerHashes.emplace_back(lh);
+                badLedgerSeqs.emplace_back(i + 1);
+            }
+
+            auto db = env.app().getLedgerDB().checkoutDb();
+
+            // Pre-migration validation - no sequence numbers.
+            *db << "INSERT INTO Validations "
+                "(LedgerHash) "
+                "VALUES "
+                "(:ledgerHash);",
+                soci::use(preSeqLedgerHashes);
+            // Post-migration orphan validation - InitalSeq,
+            // but no LedgerSeq
+            *db << "INSERT INTO Validations "
+                "(LedgerHash, InitialSeq) "
+                "VALUES "
+                "(:ledgerHash, :initialSeq);",
+                soci::use(badLedgerHashes),
+                soci::use(badLedgerSeqs);
+            // Post-migration validated ledger.
+            *db << "INSERT INTO Validations "
+                "(LedgerHash, LedgerSeq) "
+                "VALUES "
+                "(:ledgerHash, :ledgerSeq);",
+                soci::use(ledgerHashes),
+                soci::use(ledgerSeqs);
+        }
+
+        validationCheck(env, deleteInterval + 23);
+        ledgerCheck(env, deleteInterval + 1, 2);
+        transactionCheck(env, deleteInterval + 1);
+        accountTransactionCheck(env, 2 * (deleteInterval + 1));
+
+        {
+            // Closing one more ledger triggers a rotate
+            env.close();
+
+            auto ledger = env.rpc("ledger", "current");
+            expect(goodLedger(env, ledger, to_string(deleteInterval + 4)));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        expect(store->getLastRotated() == deleteInterval + 3);
+        auto const lastRotated = store->getLastRotated();
+        expect(lastRotated == 11, to_string(lastRotated));
+
+        // That took care of the fake hashes
+        validationCheck(env, deleteInterval + 8);
+        ledgerCheck(env, deleteInterval + 1, 3);
+        transactionCheck(env, deleteInterval + 1);
+        accountTransactionCheck(env, 2 * (deleteInterval + 1));
+
+        // The last iteration of this loop should trigger a rotate
+        for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1; ++i)
+        {
+            validationCheck(env, deleteInterval + i + 1 - lastRotated + 8);
+
+            env.close();
+
+            ledgerTmp = env.rpc("ledger", "current");
+            expect(goodLedger(env, ledgerTmp, to_string(i + 3)));
+
+            ledgers.emplace(std::make_pair(i,
+                env.rpc("ledger", to_string(i))));
+            expect(store->getLastRotated() == lastRotated ||
+                i == lastRotated + deleteInterval - 2, "lastRotated");
+            expect(goodLedger(env, ledgers[i], to_string(i), true) &&
+                getHash(ledgers[i]).length(), to_string(ledgers[i]));
+
+            std::vector<std::string> ledgerHashes({
+                getHash(ledgers[i])
+            });
+            std::vector<LedgerIndex> ledgerSeqs({
+                ledgers[i][jss::result][jss::ledger_index].asUInt()
+            });
+            auto db = env.app().getLedgerDB().checkoutDb();
+
+            *db << "INSERT INTO Validations "
+                "(LedgerHash, LedgerSeq) "
+                "VALUES "
+                "(:ledgerHash, :ledgerSeq);",
+                soci::use(ledgerHashes),
+                soci::use(ledgerSeqs);
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        expect(store->getLastRotated() == deleteInterval + lastRotated);
+
+        validationCheck(env, deleteInterval - 1);
+        ledgerCheck(env, deleteInterval + 1, lastRotated);
+        transactionCheck(env, 0);
+        accountTransactionCheck(env, 0);
+
+    }
+
+    void testAutomatic()
+    {
+        testcase("automatic online_delete");
+        using namespace jtx;
+        using namespace std::chrono_literals;
+
+        Env env(*this, makeConfig());
+        auto store = dynamic_cast<SHAMapStoreImp*>(
+            &env.app().getSHAMapStore());
+        expect(store);
+
+        auto ledgerSeq = waitForReady(env);
+        auto lastRotated = ledgerSeq - 1;
+        expect(store->getLastRotated() == lastRotated,
+            to_string(store->getLastRotated()));
+        expect(lastRotated != 2);
+
+        // Because advisory_delete is unset,
+        // "can_delete" is disabled.
+        auto const canDelete = env.rpc("can_delete");
+        expect(bad(canDelete, rpcNOT_ENABLED));
+
+        // Close ledgers without triggering a rotate
+        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+        {
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while(store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        // The database will always have back to ledger 2,
+        // regardless of lastRotated.
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - 2, 2);
+        expect(lastRotated == store->getLastRotated());
+
+        {
+            // Closing one more ledger triggers a rotate
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+        expect(lastRotated != store->getLastRotated());
+
+        lastRotated = store->getLastRotated();
+
+        // Close enough ledgers to trigger another rotate
+        for (; ledgerSeq < lastRotated + deleteInterval + 1; ++ledgerSeq)
+        {
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, deleteInterval + 1, lastRotated);
+        expect(lastRotated != store->getLastRotated());
+    }
+
+    void testCanDelete()
+    {
+        testcase("online_delete with advisory_delete");
+        using namespace jtx;
+        using namespace std::chrono_literals;
+
+        // Same config with advisory_delete enabled
+        Env env(*this, makeConfigAdvisory());
+        auto store = dynamic_cast<SHAMapStoreImp*>(
+            &env.app().getSHAMapStore());
+        expect(store);
+
+        auto ledgerSeq = waitForReady(env);
+        auto lastRotated = ledgerSeq - 1;
+        expect(store->getLastRotated() == lastRotated,
+            to_string(store->getLastRotated()));
+        expect(lastRotated != 2);
+
+        auto canDelete = env.rpc("can_delete");
+        expect(!RPC::contains_error(canDelete[jss::result]));
+        expect(canDelete[jss::result][jss::can_delete] == 0);
+
+        canDelete = env.rpc("can_delete", "never");
+        expect(!RPC::contains_error(canDelete[jss::result]));
+        expect(canDelete[jss::result][jss::can_delete] == 0);
+
+        auto const firstBatch = deleteInterval + ledgerSeq;
+        for (; ledgerSeq < firstBatch; ++ledgerSeq)
+        {
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - 2, 2);
+        expect(lastRotated == store->getLastRotated());
+
+        // This does not kick off a cleanup
+        canDelete = env.rpc("can_delete", to_string(
+            ledgerSeq + deleteInterval / 2));
+        expect(!RPC::contains_error(canDelete[jss::result]));
+        expect(canDelete[jss::result][jss::can_delete] ==
+            ledgerSeq + deleteInterval / 2);
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - 2, 2);
+        expect(store->getLastRotated() == lastRotated);
+
+        {
+            // This kicks off a cleanup, but it stays small.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+        expect(store->getLastRotated() == ledgerSeq - 1);
+        lastRotated = ledgerSeq - 1;
+
+        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+        {
+            // No cleanups in this loop.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        expect(store->getLastRotated() == lastRotated);
+
+        {
+            // This kicks off another cleanup.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
+
+        expect(store->getLastRotated() == ledgerSeq - 1);
+        lastRotated = ledgerSeq - 1;
+
+        // This does not kick off a cleanup
+        canDelete = env.rpc("can_delete", "always");
+        expect(!RPC::contains_error(canDelete[jss::result]));
+        expect(canDelete[jss::result][jss::can_delete] ==
+            std::numeric_limits <unsigned int>::max());
+
+        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+        {
+            // No cleanups in this loop.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        expect(store->getLastRotated() == lastRotated);
+
+        {
+            // This kicks off another cleanup.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+        expect(store->getLastRotated() == ledgerSeq - 1);
+        lastRotated = ledgerSeq - 1;
+
+        // This does not kick off a cleanup
+        canDelete = env.rpc("can_delete", "now");
+        expect(!RPC::contains_error(canDelete[jss::result]));
+        expect(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
+
+        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+        {
+            // No cleanups in this loop.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        expect(store->getLastRotated() == lastRotated);
+
+        {
+            // This kicks off another cleanup.
+            env.close();
+
+            auto ledger = env.rpc("ledger", "validated");
+            expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+        }
+
+        while (store->rotating())
+            std::this_thread::sleep_for(1ms);
+
+        validationCheck(env, 0);
+        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+        expect(store->getLastRotated() == ledgerSeq - 1);
+        lastRotated = ledgerSeq - 1;
+    }
+
+    void run()
+    {
+        testClear();
+        testAutomatic();
+        testCanDelete();
+    }
+};
+
+BEAST_DEFINE_TESTSUITE(SHAMapStore,app,ripple);
+
+}
+}

src/ripple/core/impl/DatabaseCon.cpp

@@ -46,7 +46,9 @@ DatabaseCon::DatabaseCon (
     {
         try
         {
-            session_ << initStrings[i];
+            soci::statement st = session_.prepare <<
+                initStrings[i];
+            st.execute(true);
         }
         catch (soci::soci_error&)
         {

src/ripple/rpc/handlers/CanDelete.cpp

@@ -70,7 +70,8 @@ Json::Value doCanDelete (RPC::Context& context)
             {
                 canDeleteSeq = context.app.getSHAMapStore().getLastRotated();
                 if (!canDeleteSeq)
-                    return RPC::make_error (rpcNOT_READY);            }
+                    return RPC::make_error (rpcNOT_READY);
+            }
             else if (canDeleteStr.size() == 64 &&
                 canDeleteStr.find_first_not_of("0123456789abcdef") ==
                 std::string::npos)

src/ripple/unity/app_tests.cpp

@@ -29,6 +29,7 @@
 #include <ripple/app/tests/Offer.test.cpp>
 #include <ripple/app/tests/Path_test.cpp>
 #include <ripple/app/tests/Regression_test.cpp>
+#include <ripple/app/tests/SHAMapStore_test.cpp>
 #include <ripple/app/tests/SusPay_test.cpp>
 #include <ripple/app/tests/SetAuth_test.cpp>
 #include <ripple/app/tests/OversizeMeta_test.cpp>