Rearrange sources (#4997)

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -0,0 +1,736 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/misc/SHAMapStoreImp.h>
+
+#include <ripple/app/ledger/TransactionMaster.h>
+#include <ripple/app/misc/NetworkOPs.h>
+#include <ripple/app/rdb/State.h>
+#include <ripple/app/rdb/backend/SQLiteDatabase.h>
+#include <ripple/beast/core/CurrentThreadName.h>
+#include <ripple/core/ConfigSections.h>
+#include <ripple/core/Pg.h>
+#include <ripple/nodestore/Scheduler.h>
+#include <ripple/nodestore/impl/DatabaseRotatingImp.h>
+#include <ripple/shamap/SHAMapMissingNode.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+
+namespace ripple {
+void
+SHAMapStoreImp::SavedStateDB::init(
+    BasicConfig const& config,
+    std::string const& dbName)
+{
+    std::lock_guard lock(mutex_);
+    initStateDB(sqlDb_, config, dbName);
+}
+
+LedgerIndex
+SHAMapStoreImp::SavedStateDB::getCanDelete()
+{
+    std::lock_guard lock(mutex_);
+
+    return ripple::getCanDelete(sqlDb_);
+}
+
+LedgerIndex
+SHAMapStoreImp::SavedStateDB::setCanDelete(LedgerIndex canDelete)
+{
+    std::lock_guard lock(mutex_);
+
+    return ripple::setCanDelete(sqlDb_, canDelete);
+}
+
+SavedState
+SHAMapStoreImp::SavedStateDB::getState()
+{
+    std::lock_guard lock(mutex_);
+
+    return ripple::getSavedState(sqlDb_);
+}
+
+void
+SHAMapStoreImp::SavedStateDB::setState(SavedState const& state)
+{
+    std::lock_guard lock(mutex_);
+    ripple::setSavedState(sqlDb_, state);
+}
+
+void
+SHAMapStoreImp::SavedStateDB::setLastRotated(LedgerIndex seq)
+{
+    std::lock_guard lock(mutex_);
+    ripple::setLastRotated(sqlDb_, seq);
+}
+
+//------------------------------------------------------------------------------
+
+SHAMapStoreImp::SHAMapStoreImp(
+    Application& app,
+    NodeStore::Scheduler& scheduler,
+    beast::Journal journal)
+    : app_(app)
+    , scheduler_(scheduler)
+    , journal_(journal)
+    , working_(true)
+    , canDelete_(std::numeric_limits<LedgerIndex>::max())
+{
+    Config& config{app.config()};
+
+    Section& section{config.section(ConfigSection::nodeDatabase())};
+    if (section.empty())
+    {
+        Throw<std::runtime_error>(
+            "Missing [" + ConfigSection::nodeDatabase() +
+            "] entry in configuration file");
+    }
+
+    // RocksDB only. Use sensible defaults if no values specified.
+    if (boost::iequals(get(section, "type"), "RocksDB"))
+    {
+        if (!section.exists("cache_mb"))
+        {
+            section.set(
+                "cache_mb",
+                std::to_string(config.getValueFor(SizedItem::hashNodeDBCache)));
+        }
+
+        if (!section.exists("filter_bits") && (config.NODE_SIZE >= 2))
+            section.set("filter_bits", "10");
+    }
+
+    get_if_exists(section, "online_delete", deleteInterval_);
+
+    if (deleteInterval_)
+    {
+        if (app_.config().reporting())
+        {
+            Throw<std::runtime_error>(
+                "Reporting does not support online_delete. Remove "
+                "online_delete info from config");
+        }
+
+        // Configuration that affects the behavior of online delete
+        get_if_exists(section, "delete_batch", deleteBatch_);
+        std::uint32_t temp;
+        if (get_if_exists(section, "back_off_milliseconds", temp) ||
+            // Included for backward compaibility with an undocumented setting
+            get_if_exists(section, "backOff", temp))
+        {
+            backOff_ = std::chrono::milliseconds{temp};
+        }
+        if (get_if_exists(section, "age_threshold_seconds", temp))
+            ageThreshold_ = std::chrono::seconds{temp};
+        if (get_if_exists(section, "recovery_wait_seconds", temp))
+            recoveryWaitTime_ = std::chrono::seconds{temp};
+
+        get_if_exists(section, "advisory_delete", advisoryDelete_);
+
+        auto const minInterval = config.standalone()
+            ? minimumDeletionIntervalSA_
+            : minimumDeletionInterval_;
+        if (deleteInterval_ < minInterval)
+        {
+            Throw<std::runtime_error>(
+                "online_delete must be at least " +
+                std::to_string(minInterval));
+        }
+
+        if (config.LEDGER_HISTORY > deleteInterval_)
+        {
+            Throw<std::runtime_error>(
+                "online_delete must not be less than ledger_history "
+                "(currently " +
+                std::to_string(config.LEDGER_HISTORY) + ")");
+        }
+
+        state_db_.init(config, dbName_);
+        dbPaths();
+    }
+}
+
+std::unique_ptr<NodeStore::Database>
+SHAMapStoreImp::makeNodeStore(int readThreads)
+{
+    auto nscfg = app_.config().section(ConfigSection::nodeDatabase());
+
+    // Provide default values:
+    if (!nscfg.exists("cache_size"))
+        nscfg.set(
+            "cache_size",
+            std::to_string(app_.config().getValueFor(
+                SizedItem::treeCacheSize, std::nullopt)));
+
+    if (!nscfg.exists("cache_age"))
+        nscfg.set(
+            "cache_age",
+            std::to_string(app_.config().getValueFor(
+                SizedItem::treeCacheAge, std::nullopt)));
+
+    std::unique_ptr<NodeStore::Database> db;
+
+    if (deleteInterval_)
+    {
+        if (app_.config().reporting())
+        {
+            Throw<std::runtime_error>(
+                "Reporting does not support online_delete. Remove "
+                "online_delete info from config");
+        }
+        SavedState state = state_db_.getState();
+        auto writableBackend = makeBackendRotating(state.writableDb);
+        auto archiveBackend = makeBackendRotating(state.archiveDb);
+        if (!state.writableDb.size())
+        {
+            state.writableDb = writableBackend->getName();
+            state.archiveDb = archiveBackend->getName();
+            state_db_.setState(state);
+        }
+
+        // Create NodeStore with two backends to allow online deletion of
+        // data
+        auto dbr = std::make_unique<NodeStore::DatabaseRotatingImp>(
+            scheduler_,
+            readThreads,
+            std::move(writableBackend),
+            std::move(archiveBackend),
+            nscfg,
+            app_.logs().journal(nodeStoreName_));
+        fdRequired_ += dbr->fdRequired();
+        dbRotating_ = dbr.get();
+        db.reset(dynamic_cast<NodeStore::Database*>(dbr.release()));
+    }
+    else
+    {
+        db = NodeStore::Manager::instance().make_Database(
+            megabytes(
+                app_.config().getValueFor(SizedItem::burstSize, std::nullopt)),
+            scheduler_,
+            readThreads,
+            nscfg,
+            app_.logs().journal(nodeStoreName_));
+        fdRequired_ += db->fdRequired();
+    }
+    return db;
+}
+
+void
+SHAMapStoreImp::onLedgerClosed(std::shared_ptr<Ledger const> const& ledger)
+{
+    {
+        std::lock_guard lock(mutex_);
+        newLedger_ = ledger;
+        working_ = true;
+    }
+    cond_.notify_one();
+}
+
+void
+SHAMapStoreImp::rendezvous() const
+{
+    if (!working_)
+        return;
+
+    std::unique_lock<std::mutex> lock(mutex_);
+    rendezvous_.wait(lock, [&] { return !working_; });
+}
+
+int
+SHAMapStoreImp::fdRequired() const
+{
+    return fdRequired_;
+}
+
+bool
+SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
+{
+    // Copy a single record from node to dbRotating_
+    dbRotating_->fetchNodeObject(
+        node.getHash().as_uint256(),
+        0,
+        NodeStore::FetchType::synchronous,
+        true);
+    if (!(++nodeCount % checkHealthInterval_))
+    {
+        if (healthWait() == stopping)
+            return false;
+    }
+
+    return true;
+}
+
+void
+SHAMapStoreImp::run()
+{
+    if (app_.config().reporting())
+    {
+        assert(false);
+        Throw<std::runtime_error>(
+            "Reporting does not support online_delete. Remove "
+            "online_delete info from config");
+    }
+    beast::setCurrentThreadName("SHAMapStore");
+    LedgerIndex lastRotated = state_db_.getState().lastRotated;
+    netOPs_ = &app_.getOPs();
+    ledgerMaster_ = &app_.getLedgerMaster();
+    fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache(0));
+    treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache(0));
+
+    if (advisoryDelete_)
+        canDelete_ = state_db_.getCanDelete();
+
+    while (true)
+    {
+        healthy_ = true;
+        std::shared_ptr<Ledger const> validatedLedger;
+
+        {
+            std::unique_lock<std::mutex> lock(mutex_);
+            working_ = false;
+            rendezvous_.notify_all();
+            if (stop_)
+            {
+                return;
+            }
+            cond_.wait(lock);
+            if (newLedger_)
+            {
+                validatedLedger = std::move(newLedger_);
+            }
+            else
+                continue;
+        }
+
+        LedgerIndex const validatedSeq = validatedLedger->info().seq;
+        if (!lastRotated)
+        {
+            lastRotated = validatedSeq;
+            state_db_.setLastRotated(lastRotated);
+        }
+
+        bool const readyToRotate =
+            validatedSeq >= lastRotated + deleteInterval_ &&
+            canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
+
+        // Make sure we don't delete ledgers currently being
+        // imported into the ShardStore
+        bool const waitForImport = readyToRotate && [this, lastRotated] {
+            if (auto shardStore = app_.getShardStore())
+            {
+                if (auto sequence = shardStore->getDatabaseImportSequence())
+                    return sequence <= lastRotated - 1;
+            }
+
+            return false;
+        }();
+
+        if (waitForImport)
+        {
+            JLOG(journal_.info())
+                << "NOT rotating validatedSeq " << validatedSeq
+                << " as rotation would interfere with ShardStore import";
+        }
+
+        // will delete up to (not including) lastRotated
+        if (readyToRotate && !waitForImport)
+        {
+            JLOG(journal_.warn())
+                << "rotating  validatedSeq " << validatedSeq << " lastRotated "
+                << lastRotated << " deleteInterval " << deleteInterval_
+                << " canDelete_ " << canDelete_ << " state "
+                << app_.getOPs().strOperatingMode(false) << " age "
+                << ledgerMaster_->getValidatedLedgerAge().count() << 's';
+
+            clearPrior(lastRotated);
+            if (healthWait() == stopping)
+                return;
+
+            JLOG(journal_.debug()) << "copying ledger " << validatedSeq;
+            std::uint64_t nodeCount = 0;
+
+            try
+            {
+                validatedLedger->stateMap().snapShot(false)->visitNodes(
+                    std::bind(
+                        &SHAMapStoreImp::copyNode,
+                        this,
+                        std::ref(nodeCount),
+                        std::placeholders::_1));
+            }
+            catch (SHAMapMissingNode const& e)
+            {
+                JLOG(journal_.error())
+                    << "Missing node while copying ledger before rotate: "
+                    << e.what();
+                continue;
+            }
+
+            if (healthWait() == stopping)
+                return;
+            // Only log if we completed without a "health" abort
+            JLOG(journal_.debug()) << "copied ledger " << validatedSeq
+                                   << " nodecount " << nodeCount;
+
+            JLOG(journal_.debug()) << "freshening caches";
+            freshenCaches();
+            if (healthWait() == stopping)
+                return;
+            // Only log if we completed without a "health" abort
+            JLOG(journal_.debug()) << validatedSeq << " freshened caches";
+
+            JLOG(journal_.trace()) << "Making a new backend";
+            auto newBackend = makeBackendRotating();
+            JLOG(journal_.debug())
+                << validatedSeq << " new backend " << newBackend->getName();
+
+            clearCaches(validatedSeq);
+            if (healthWait() == stopping)
+                return;
+
+            lastRotated = validatedSeq;
+
+            dbRotating_->rotateWithLock(
+                [&](std::string const& writableBackendName) {
+                    SavedState savedState;
+                    savedState.writableDb = newBackend->getName();
+                    savedState.archiveDb = writableBackendName;
+                    savedState.lastRotated = lastRotated;
+                    state_db_.setState(savedState);
+
+                    clearCaches(validatedSeq);
+
+                    return std::move(newBackend);
+                });
+
+            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
+        }
+    }
+}
+
+void
+SHAMapStoreImp::dbPaths()
+{
+    Section section{app_.config().section(ConfigSection::nodeDatabase())};
+    boost::filesystem::path dbPath = get(section, "path");
+
+    if (boost::filesystem::exists(dbPath))
+    {
+        if (!boost::filesystem::is_directory(dbPath))
+        {
+            journal_.error()
+                << "node db path must be a directory. " << dbPath.string();
+            Throw<std::runtime_error>("node db path must be a directory.");
+        }
+    }
+    else
+    {
+        boost::filesystem::create_directories(dbPath);
+    }
+
+    SavedState state = state_db_.getState();
+
+    {
+        auto update = [&dbPath](std::string& sPath) {
+            if (sPath.empty())
+                return false;
+
+            // Check if configured "path" matches stored directory path
+            using namespace boost::filesystem;
+            auto const stored{path(sPath)};
+            if (stored.parent_path() == dbPath)
+                return false;
+
+            sPath = (dbPath / stored.filename()).string();
+            return true;
+        };
+
+        if (update(state.writableDb))
+        {
+            update(state.archiveDb);
+            state_db_.setState(state);
+        }
+    }
+
+    bool writableDbExists = false;
+    bool archiveDbExists = false;
+
+    std::vector<boost::filesystem::path> pathsToDelete;
+    for (boost::filesystem::directory_iterator it(dbPath);
+         it != boost::filesystem::directory_iterator();
+         ++it)
+    {
+        if (!state.writableDb.compare(it->path().string()))
+            writableDbExists = true;
+        else if (!state.archiveDb.compare(it->path().string()))
+            archiveDbExists = true;
+        else if (!dbPrefix_.compare(it->path().stem().string()))
+            pathsToDelete.push_back(it->path());
+    }
+
+    if ((!writableDbExists && state.writableDb.size()) ||
+        (!archiveDbExists && state.archiveDb.size()) ||
+        (writableDbExists != archiveDbExists) ||
+        state.writableDb.empty() != state.archiveDb.empty())
+    {
+        boost::filesystem::path stateDbPathName =
+            app_.config().legacy("database_path");
+        stateDbPathName /= dbName_;
+        stateDbPathName += "*";
+
+        journal_.error()
+            << "state db error:\n"
+            << "  writableDbExists " << writableDbExists << " archiveDbExists "
+            << archiveDbExists << '\n'
+            << "  writableDb '" << state.writableDb << "' archiveDb '"
+            << state.archiveDb << "\n\n"
+            << "The existing data is in a corrupted state.\n"
+            << "To resume operation, remove the files matching "
+            << stateDbPathName.string() << " and contents of the directory "
+            << get(section, "path") << '\n'
+            << "Optionally, you can move those files to another\n"
+            << "location if you wish to analyze or back up the data.\n"
+            << "However, there is no guarantee that the data in its\n"
+            << "existing form is usable.";
+
+        Throw<std::runtime_error>("state db error");
+    }
+
+    // The necessary directories exist. Now, remove any others.
+    for (boost::filesystem::path& p : pathsToDelete)
+        boost::filesystem::remove_all(p);
+}
+
+std::unique_ptr<NodeStore::Backend>
+SHAMapStoreImp::makeBackendRotating(std::string path)
+{
+    Section section{app_.config().section(ConfigSection::nodeDatabase())};
+    boost::filesystem::path newPath;
+
+    if (path.size())
+    {
+        newPath = path;
+    }
+    else
+    {
+        boost::filesystem::path p = get(section, "path");
+        p /= dbPrefix_;
+        p += ".%%%%";
+        newPath = boost::filesystem::unique_path(p);
+    }
+    section.set("path", newPath.string());
+
+    auto backend{NodeStore::Manager::instance().make_Backend(
+        section,
+        megabytes(
+            app_.config().getValueFor(SizedItem::burstSize, std::nullopt)),
+        scheduler_,
+        app_.logs().journal(nodeStoreName_))};
+    backend->open();
+    return backend;
+}
+
+void
+SHAMapStoreImp::clearSql(
+    LedgerIndex lastRotated,
+    std::string const& TableName,
+    std::function<std::optional<LedgerIndex>()> const& getMinSeq,
+    std::function<void(LedgerIndex)> const& deleteBeforeSeq)
+{
+    assert(deleteInterval_);
+    LedgerIndex min = std::numeric_limits<LedgerIndex>::max();
+
+    {
+        JLOG(journal_.trace())
+            << "Begin: Look up lowest value of: " << TableName;
+        auto m = getMinSeq();
+        JLOG(journal_.trace()) << "End: Look up lowest value of: " << TableName;
+        if (!m)
+            return;
+        min = *m;
+    }
+
+    if (min > lastRotated || healthWait() == stopping)
+        return;
+    if (min == lastRotated)
+    {
+        // Micro-optimization mainly to clarify logs
+        JLOG(journal_.trace()) << "Nothing to delete from " << TableName;
+        return;
+    }
+
+    JLOG(journal_.debug()) << "start deleting in: " << TableName << " from "
+                           << min << " to " << lastRotated;
+    while (min < lastRotated)
+    {
+        min = std::min(lastRotated, min + deleteBatch_);
+        JLOG(journal_.trace())
+            << "Begin: Delete up to " << deleteBatch_
+            << " rows with LedgerSeq < " << min << " from: " << TableName;
+        deleteBeforeSeq(min);
+        JLOG(journal_.trace())
+            << "End: Delete up to " << deleteBatch_ << " rows with LedgerSeq < "
+            << min << " from: " << TableName;
+        if (healthWait() == stopping)
+            return;
+        if (min < lastRotated)
+            std::this_thread::sleep_for(backOff_);
+        if (healthWait() == stopping)
+            return;
+    }
+    JLOG(journal_.debug()) << "finished deleting from: " << TableName;
+}
+
+void
+SHAMapStoreImp::clearCaches(LedgerIndex validatedSeq)
+{
+    ledgerMaster_->clearLedgerCachePrior(validatedSeq);
+    fullBelowCache_->clear();
+}
+
+void
+SHAMapStoreImp::freshenCaches()
+{
+    if (freshenCache(*treeNodeCache_))
+        return;
+    if (freshenCache(app_.getMasterTransaction().getCache()))
+        return;
+}
+
+void
+SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
+{
+    if (app_.config().reporting())
+    {
+        assert(false);
+        Throw<std::runtime_error>(
+            "Reporting does not support online_delete. Remove "
+            "online_delete info from config");
+    }
+    // Do not allow ledgers to be acquired from the network
+    // that are about to be deleted.
+    minimumOnline_ = lastRotated + 1;
+    JLOG(journal_.trace()) << "Begin: Clear internal ledgers up to "
+                           << lastRotated;
+    ledgerMaster_->clearPriorLedgers(lastRotated);
+    JLOG(journal_.trace()) << "End: Clear internal ledgers up to "
+                           << lastRotated;
+    if (healthWait() == stopping)
+        return;
+
+    SQLiteDatabase* const db =
+        dynamic_cast<SQLiteDatabase*>(&app_.getRelationalDatabase());
+
+    if (!db)
+        Throw<std::runtime_error>("Failed to get relational database");
+
+    clearSql(
+        lastRotated,
+        "Ledgers",
+        [db]() -> std::optional<LedgerIndex> { return db->getMinLedgerSeq(); },
+        [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); });
+    if (healthWait() == stopping)
+        return;
+
+    if (!app_.config().useTxTables())
+        return;
+
+    clearSql(
+        lastRotated,
+        "Transactions",
+        [&db]() -> std::optional<LedgerIndex> {
+            return db->getTransactionsMinLedgerSeq();
+        },
+        [&db](LedgerIndex min) -> void {
+            db->deleteTransactionsBeforeLedgerSeq(min);
+        });
+    if (healthWait() == stopping)
+        return;
+
+    clearSql(
+        lastRotated,
+        "AccountTransactions",
+        [&db]() -> std::optional<LedgerIndex> {
+            return db->getAccountTransactionsMinLedgerSeq();
+        },
+        [&db](LedgerIndex min) -> void {
+            db->deleteAccountTransactionsBeforeLedgerSeq(min);
+        });
+    if (healthWait() == stopping)
+        return;
+}
+
+SHAMapStoreImp::HealthResult
+SHAMapStoreImp::healthWait()
+{
+    auto age = ledgerMaster_->getValidatedLedgerAge();
+    OperatingMode mode = netOPs_->getOperatingMode();
+    std::unique_lock lock(mutex_);
+    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
+    {
+        lock.unlock();
+        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
+                              << "s for node to stabilize. state: "
+                              << app_.getOPs().strOperatingMode(mode, false)
+                              << ". age " << age.count() << 's';
+        std::this_thread::sleep_for(recoveryWaitTime_);
+        age = ledgerMaster_->getValidatedLedgerAge();
+        mode = netOPs_->getOperatingMode();
+        lock.lock();
+    }
+
+    return stop_ ? stopping : keepGoing;
+}
+
+void
+SHAMapStoreImp::stop()
+{
+    if (thread_.joinable())
+    {
+        {
+            std::lock_guard lock(mutex_);
+            stop_ = true;
+            cond_.notify_one();
+        }
+        thread_.join();
+    }
+}
+
+std::optional<LedgerIndex>
+SHAMapStoreImp::minimumOnline() const
+{
+    // minimumOnline_ with 0 value is equivalent to unknown/not set.
+    // Don't attempt to acquire ledgers if that value is unknown.
+    if (deleteInterval_ && minimumOnline_)
+        return minimumOnline_.load();
+    return app_.getLedgerMaster().minSqlSeq();
+}
+
+//------------------------------------------------------------------------------
+
+std::unique_ptr<SHAMapStore>
+make_SHAMapStore(
+    Application& app,
+    NodeStore::Scheduler& scheduler,
+    beast::Journal journal)
+{
+    return std::make_unique<SHAMapStoreImp>(app, scheduler, journal);
+}
+
+}  // namespace ripple