Expand SQLite potential storage capacity:

Increase page size for SQLite transaction database upon creation
Provide diagnostics for transaction db page usage.
Shut down rippled gracefullly if transaction db is running out of pages.
Add new rippled maintenance command line option to cause new page size
to take effect.

src/ripple/app/main/Application.cpp

@@ -58,6 +58,7 @@
 #include <ripple/beast/asio/io_latency_probe.h>
 #include <ripple/beast/core/LexicalCast.h>
 #include <boost/asio/steady_timer.hpp>
+#include <boost/system/error_code.hpp>
 #include <fstream>
 #include <sstream>
 #include <iostream>
@@ -824,7 +825,7 @@ public:
         assert (mWalletDB.get () == nullptr);
 
         DatabaseCon::Setup setup = setup_DatabaseCon (*config_);
-        mTxnDB = std::make_unique <DatabaseCon> (setup, "transaction.db",
+        mTxnDB = std::make_unique <DatabaseCon> (setup, TxnDBName,
                 TxnDBInit, TxnDBCount);
         mLedgerDB = std::make_unique <DatabaseCon> (setup, "ledger.db",
                 LedgerDBInit, LedgerDBCount);
@@ -1034,6 +1035,54 @@ public:
                     << "Remaining free disk space is less than 512MB";
                 signalStop ();
             }
+
+            DatabaseCon::Setup dbSetup = setup_DatabaseCon(*config_);
+            boost::filesystem::path dbPath = dbSetup.dataDir / TxnDBName;
+            boost::system::error_code ec;
+            boost::optional<std::uint64_t> dbSize = boost::filesystem::file_size(dbPath, ec);
+            if (ec)
+            {
+                JLOG(m_journal.error())
+                    << "Error checking transaction db file size: "
+                    << ec.message();
+                dbSize.reset();
+            }
+
+            auto db = mTxnDB->checkoutDb();
+            static auto const pageSize = [&]{
+                std::uint32_t ps;
+                *db << "PRAGMA page_size;", soci::into(ps);
+                return ps;
+            }();
+            static auto const maxPages = [&]{
+                std::uint32_t mp;
+                *db << "PRAGMA max_page_count;" , soci::into(mp);
+                return mp;
+            }();
+            std::uint32_t pageCount;
+            *db << "PRAGMA page_count;", soci::into(pageCount);
+            std::uint32_t freePages = maxPages - pageCount;
+            std::uint64_t freeSpace =
+                static_cast<std::uint64_t>(freePages) * pageSize;
+            JLOG(m_journal.info())
+               << "Transaction DB pathname: " << dbPath.string()
+               << "; file size: " << dbSize.value_or(-1) << " bytes"
+               << "; SQLite page size: " << pageSize  << " bytes"
+               << "; Free pages: " << freePages
+               << "; Free space: " << freeSpace << " bytes; "
+               << "Note that this does not take into account available disk "
+                  "space.";
+
+            if (freeSpace < bytes512M)
+            {
+                JLOG(m_journal.fatal())
+                    << "Free SQLite space for transaction db is less than "
+                       "512MB. To fix this, rippled must be executed with the "
+                       "vacuum <sqlitetmpdir> parameter before restarting. "
+                       "Note that this activity can take multiple days, "
+                       "depending on database size.";
+                signalStop();
+            }
         }
 
         // VFALCO NOTE Does the order of calls matter?

src/ripple/app/main/DBInit.cpp

@@ -23,8 +23,10 @@
 namespace ripple {
 
 // Transaction database holds transactions and public keys
+const char* TxnDBName = "transaction.db";
 const char* TxnDBInit[] =
 {
+    "PRAGMA page_size=4096;",
     "PRAGMA synchronous=NORMAL;",
     "PRAGMA journal_mode=WAL;",
     "PRAGMA journal_size_limit=1582080;",

src/ripple/app/main/DBInit.h

@@ -23,6 +23,7 @@
 namespace ripple {
 
 // VFALCO TODO Tidy these up into a class with functions and return types.
+extern const char* TxnDBName;
 extern const char* TxnDBInit[];
 extern const char* LedgerDBInit[];
 extern const char* WalletDBInit[];

src/ripple/app/main/Main.cpp

@@ -21,11 +21,13 @@
 #include <ripple/basics/Log.h>
 #include <ripple/protocol/digest.h>
 #include <ripple/app/main/Application.h>
+#include <ripple/app/main/DBInit.h>
 #include <ripple/basics/contract.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/basics/Sustain.h>
 #include <ripple/core/Config.h>
 #include <ripple/core/ConfigSections.h>
+#include <ripple/core/DatabaseCon.h>
 #include <ripple/core/TerminateHandler.h>
 #include <ripple/core/TimeKeeper.h>
 #include <ripple/crypto/csprng.h>
@@ -46,8 +48,10 @@
 
 #include <google/protobuf/stubs/common.h>
 
+#include <boost/filesystem.hpp>
 #include <boost/process.hpp>
 #include <boost/program_options.hpp>
+#include <boost/system/error_code.hpp>
 #include <boost/version.hpp>
 
 #include <cstdlib>
@@ -342,6 +346,9 @@ int run (int argc, char** argv)
     ("nodetoshard", "Import node store into shards")
     ("replay","Replay a ledger close.")
     ("start", "Start from a fresh Ledger.")
+    ("vacuum", po::value<std::string>(),
+        "VACUUM the transaction db. Mandatory string argument specifies "
+        "temporary directory path.")
     ("valid", "Consider the initial ledger a valid network ledger.")
     ("validateShards", shardsText.c_str ())
     ;
@@ -488,6 +495,81 @@ int run (int argc, char** argv)
             crypto_prng().load_state(entropy.string ());
     }
 
+    if (vm.count("vacuum"))
+    {
+        DatabaseCon::Setup dbSetup = setup_DatabaseCon(*config);
+        if (dbSetup.standAlone)
+        {
+            std::cerr << "vacuum not applicable in standalone mode.\n";
+            return -1;
+        }
+        boost::filesystem::path dbPath = dbSetup.dataDir / TxnDBName;
+        auto txnDB = std::make_unique<DatabaseCon> (dbSetup, TxnDBName,
+            TxnDBInit, TxnDBCount);
+        if (txnDB.get() == nullptr)
+        {
+            std::cerr << "Cannot create connection to " << dbPath.string() <<
+                '\n';
+            return -1;
+        }
+        boost::system::error_code ec;
+        uintmax_t dbSize = boost::filesystem::file_size(dbPath, ec);
+        if (ec)
+        {
+            std::cerr << "Error checking size of " << dbPath.string() << ": "
+                << ec.message() << '\n';
+            return -1;
+        }
+
+        assert(dbSize != static_cast<uintmax_t>(-1));
+
+        std::string tmpDir = vm["vacuum"].as<std::string>();
+        boost::filesystem::path tmpPath = tmpDir;
+        if (boost::filesystem::space(tmpPath, ec).available < dbSize)
+        {
+            if (ec)
+            {
+                std::cerr << "Error checking status of " << tmpPath.string()
+                    << ": " << ec.message() << '\n';
+            }
+            else
+            {
+                std::cerr << "A valid directory for vacuuming must be "
+                             "specified on a filesystem with at least "
+                             "as much free space as the size of " <<
+                             dbPath.string() << ", which is " <<
+                             dbSize << " bytes. The filesystem for " <<
+                             tmpPath.string() << " only has " <<
+                             boost::filesystem::space(tmpPath, ec).available
+                             << " bytes.\n";
+            }
+            return -1;
+        }
+
+        auto db = txnDB->checkoutDb();
+        std::uint32_t pageSize;
+        try
+        {
+            *db << "PRAGMA page_size;", soci::into(pageSize);
+            std::cout << "VACUUM beginning. page_size: " << pageSize
+                << std::endl;
+            *db << "PRAGMA journal_mode=OFF;";
+            *db << "PRAGMA temp_store_directory=\"" << tmpPath.string()
+                << "\";";
+            *db << "VACUUM;";
+            *db << "PRAGMA journal_mode=WAL;";
+            *db << "PRAGMA page_size;", soci::into(pageSize);
+        }
+        catch (soci::soci_error const& e)
+        {
+            std::cerr << "SQLite error: " << e.what() << '\n';
+            return 1;
+        }
+        std::cout << "VACUUM finished. page_size: " << pageSize << std::endl;
+
+        return 0;
+    }
+
     if (vm.count ("start"))
         config->START_UP = Config::FRESH;