Replace some boost::thread with beast::thread

TODO.txt

@@ -40,6 +40,10 @@ David Feature:
 
 --------------------------------------------------------------------------------
 
+- Consolidate databases
+
+- Figure out why we need WAL sqlite mode if we no longer use sqlite for the node store
+
 - Add "skipped" field to beginTestCase() to disable a test but still record
   that it was skipped in the output. Like for mdb import.
 

modules/ripple_app/data/ripple_SqliteDatabase.cpp

@@ -33,8 +33,11 @@ SqliteStatement::~SqliteStatement ()
     sqlite3_finalize (statement);
 }
 
+//------------------------------------------------------------------------------
+
 SqliteDatabase::SqliteDatabase (const char* host)
     : Database (host)
+    , m_thread ("sqlitedb")
     , mWalQ (NULL)
     , walRunning (false)
 {
@@ -290,9 +293,13 @@ void SqliteDatabase::doHook (const char* db, int pages)
     }
 
     if (mWalQ)
+    {
         mWalQ->addJob (jtWAL, std::string ("WAL:") + mHost, BIND_TYPE (&SqliteDatabase::runWal, this));
+    }
     else
-        boost::thread (BIND_TYPE (&SqliteDatabase::runWal, this)).detach ();
+    {
+        m_thread.call (&SqliteDatabase::runWal, this);
+    }
 }
 
 void SqliteDatabase::runWal ()

modules/ripple_app/data/ripple_SqliteDatabase.h

@@ -57,6 +57,8 @@ public:
     int getKBUsedAll ();
 
 private:
+    ThreadWithCallQueue m_thread;
+
     sqlite3* mConnection;
     // VFALCO TODO Why do we need an "aux" connection? Should just use a second SqliteDatabase object.
     sqlite3* mAuxConnection;

modules/ripple_app/main/ripple_Application.cpp

@@ -44,6 +44,13 @@ public:
             // the dtor of m_threads will block until each thread exits.
         }
 
+        // TEMPORARY HACK for compatibility with old code
+        void runExtraThreads ()
+        {
+            for (int i = 0; i < m_threads.size (); ++i)
+                m_threads [i]->start ();
+        }
+
         // Run on the callers thread.
         // This will block until stop is issued.
         void run ()
@@ -92,7 +99,7 @@ public:
                 : Thread (name)
                 , m_service (service)
             {
-                startThread ();
+                //startThread ();
             }
 
             ~ServiceThread ()
@@ -102,6 +109,11 @@ public:
                 stopThread (-1); // wait forever
             }
 
+            void start ()
+            {
+                startThread ();
+            }
+
             void run ()
             {
                 m_service.run ();
@@ -149,7 +161,6 @@ public:
         , mTempNodeCache ("NodeCache", 16384, 90)
         , mSLECache ("LedgerEntryCache", 4096, 120)
         , mSNTPClient (m_auxService)
-        , mJobQueue (m_mainService)
         // VFALCO New stuff
         , m_nodeStore (NodeStore::New (
             getConfig ().nodeDatabase,
@@ -360,119 +371,49 @@ public:
     {
         return mShutdown;
     }
-    void setup ();
-    void run ();
-    void stop ();
-    void sweep ();
-    void doSweep (Job&);
 
-private:
+    //--------------------------------------------------------------------------
-    void updateTables ();
-    void startNewLedger ();
-    bool loadOldLedger (const std::string&, bool);
 
-private:
+    static DatabaseCon* openDatabaseCon (const char* fileName,
-    IoServiceThread m_mainService;
+                                         const char* dbInit[],
-    IoServiceThread m_auxService;
+                                         int dbCount)
+    {
+        return new DatabaseCon (fileName, dbInit, dbCount);
+    }
 
-    //boost::asio::io_service mIOService;
+    void initSqliteDb (int index)
-    //boost::asio::io_service mAuxService;
+    {
-    //boost::asio::io_service::work mIOWork;
+        switch (index)
+        {
+        case 0: mRpcDB = openDatabaseCon ("rpc.db", RpcDBInit, RpcDBCount); break;
+        case 1: mTxnDB = openDatabaseCon ("transaction.db", TxnDBInit, TxnDBCount); break;
+        case 2: mLedgerDB = openDatabaseCon ("ledger.db", LedgerDBInit, LedgerDBCount); break;
+        case 3: mWalletDB = openDatabaseCon ("wallet.db", WalletDBInit, WalletDBCount); break;
+        };
+    }
 
-    MasterLockType mMasterLock;
+    // VFALCO TODO Is it really necessary to init the dbs in parallel?
+    void initSqliteDbs ()
+    {
+        int const count = 4;
 
-    LocalCredentials   m_localCredentials;
+        ThreadGroup threadGroup (count);
-    LedgerMaster       mLedgerMaster;
+        ParallelFor (threadGroup).loop (count, &ApplicationImp::initSqliteDb, this);
-    InboundLedgers     m_inboundLedgers;
+    }
-    TransactionMaster  mMasterTransaction;
-    ScopedPointer <NetworkOPs> mNetOps;
-    RPCServerHandler   m_rpcServerHandler;
-    NodeCache          mTempNodeCache;
-    SLECache           mSLECache;
-    SNTPClient         mSNTPClient;
-    JobQueue           mJobQueue;
-    TXQueue            mTxnQueue;
-    OrderBookDB        mOrderBookDB;
-
-    // VFALCO Clean stuff
-    ScopedPointer <NodeStore> m_nodeStore;
-    ScopedPointer <Validators> m_validators;
-    ScopedPointer <IFeatures> mFeatures;
-    ScopedPointer <IFeeVote> mFeeVote;
-    ScopedPointer <ILoadFeeTrack> mFeeTrack;
-    ScopedPointer <IHashRouter> mHashRouter;
-    ScopedPointer <IValidations> mValidations;
-    ScopedPointer <UniqueNodeList> mUNL;
-    ScopedPointer <IProofOfWorkFactory> mProofOfWorkFactory;
-    ScopedPointer <IPeers> mPeers;
-    ScopedPointer <ILoadManager> m_loadManager;
-    // VFALCO End Clean stuff
-
-    DatabaseCon* mRpcDB;
-    DatabaseCon* mTxnDB;
-    DatabaseCon* mLedgerDB;
-    DatabaseCon* mWalletDB;
-
-    ScopedPointer <PeerDoor> mPeerDoor;
-    ScopedPointer <RPCDoor>  mRPCDoor;
-    ScopedPointer <WSDoor>   mWSPublicDoor;
-    ScopedPointer <WSDoor>   mWSPrivateDoor;
-
-    boost::asio::deadline_timer mSweepTimer;
-
-    bool volatile mShutdown;
-};
-
-// VFALCO TODO Why do we even have this function?
-//             It could just be handled in the destructor.
-//
-void ApplicationImp::stop ()
-{
-    WriteLog (lsINFO, Application) << "Received shutdown request";
-    StopSustain ();
-    mShutdown = true;
-    m_mainService.stop ();
-    m_nodeStore = nullptr;
-    mValidations->flush ();
-    m_auxService.stop ();
-    mJobQueue.shutdown ();
-
-    //WriteLog (lsINFO, Application) << "Stopped: " << mIOService.stopped 
-
-    mShutdown = false;
-}
-
-static void InitDB (DatabaseCon** dbCon, const char* fileName, const char* dbInit[], int dbCount)
-{
-    *dbCon = new DatabaseCon (fileName, dbInit, dbCount);
-}
 
 #ifdef SIGINT
-void sigIntHandler (int)
+    static void sigIntHandler (int)
-{
+    {
         doShutdown = true;
-}
+    }
 #endif
 
-// VFALCO TODO Figure this out it looks like the wrong tool
+    // VFALCO TODO Break this function up into many small initialization segments.
-static void runAux (boost::asio::io_service& svc)
+    //             Or better yet refactor these initializations into RAII classes
-{
+    //             which are members of the Application object.
-    setCallingThreadName ("aux");
+    //
-    svc.run ();
+    void setup ()
-}
+    {
-
-static void runIO (boost::asio::io_service& io)
-{
-    setCallingThreadName ("io");
-    io.run ();
-}
-
-// VFALCO TODO Break this function up into many small initialization segments.
-//             Or better yet refactor these initializations into RAII classes
-//             which are members of the Application object.
-//
-void ApplicationImp::setup ()
-{
         // VFALCO NOTE: 0 means use heuristics to determine the thread count.
         mJobQueue.setThreadCount (0, getConfig ().RUN_STANDALONE);
 
@@ -481,19 +422,19 @@ void ApplicationImp::setup ()
 
         m_loadManager->startThread ();
 
-#if ! BEAST_WIN32
+    #if ! BEAST_WIN32
-#ifdef SIGINT
+    #ifdef SIGINT
 
         if (!getConfig ().RUN_STANDALONE)
         {
             struct sigaction sa;
             memset (&sa, 0, sizeof (sa));
-        sa.sa_handler = sigIntHandler;
+            sa.sa_handler = &ApplicationImp::sigIntHandler;
             sigaction (SIGINT, &sa, NULL);
         }
 
-#endif
+    #endif
-#endif
+    #endif
 
         assert (mTxnDB == NULL);
 
@@ -509,17 +450,7 @@ void ApplicationImp::setup ()
         if (!getConfig ().RUN_STANDALONE)
             mSNTPClient.init (getConfig ().SNTP_SERVERS);
 
-    //
+        initSqliteDbs ();
-    // Construct databases.
-    //
-    boost::thread t1 (BIND_TYPE (&InitDB, &mRpcDB, "rpc.db", RpcDBInit, RpcDBCount));
-    boost::thread t2 (BIND_TYPE (&InitDB, &mTxnDB, "transaction.db", TxnDBInit, TxnDBCount));
-    boost::thread t3 (BIND_TYPE (&InitDB, &mLedgerDB, "ledger.db", LedgerDBInit, LedgerDBCount));
-    boost::thread t4 (BIND_TYPE (&InitDB, &mWalletDB, "wallet.db", WalletDBInit, WalletDBCount));
-    t1.join ();
-    t2.join ();
-    t3.join ();
-    t4.join ();
 
         leveldb::Options options;
         options.create_if_missing = true;
@@ -642,7 +573,7 @@ void ApplicationImp::setup ()
         {
             try
             {
-            mWSPrivateDoor  = WSDoor::createWSDoor (getConfig ().WEBSOCKET_IP, getConfig ().WEBSOCKET_PORT, false);
+                mWSPrivateDoor  = new WSDoor (getConfig ().WEBSOCKET_IP, getConfig ().WEBSOCKET_PORT, false);
             }
             catch (const std::exception& e)
             {
@@ -664,7 +595,7 @@ void ApplicationImp::setup ()
         {
             try
             {
-            mWSPublicDoor   = WSDoor::createWSDoor (getConfig ().WEBSOCKET_PUBLIC_IP, getConfig ().WEBSOCKET_PUBLIC_PORT, true);
+                mWSPublicDoor   = new WSDoor (getConfig ().WEBSOCKET_PUBLIC_IP, getConfig ().WEBSOCKET_PUBLIC_PORT, true);
             }
             catch (const std::exception& e)
             {
@@ -697,10 +628,98 @@ void ApplicationImp::setup ()
             //
             mNetOps->setStateTimer ();
         }
+    }
+
+
+    void run ();
+    void stop ();
+    void sweep ();
+    void doSweep (Job&);
+
+private:
+    void updateTables ();
+    void startNewLedger ();
+    bool loadOldLedger (const std::string&, bool);
+
+private:
+    IoServiceThread m_mainService;
+    IoServiceThread m_auxService;
+
+    //boost::asio::io_service mIOService;
+    //boost::asio::io_service mAuxService;
+    //boost::asio::io_service::work mIOWork;
+
+    MasterLockType mMasterLock;
+
+    LocalCredentials   m_localCredentials;
+    LedgerMaster       mLedgerMaster;
+    InboundLedgers     m_inboundLedgers;
+    TransactionMaster  mMasterTransaction;
+    ScopedPointer <NetworkOPs> mNetOps;
+    RPCServerHandler   m_rpcServerHandler;
+    NodeCache          mTempNodeCache;
+    SLECache           mSLECache;
+    SNTPClient         mSNTPClient;
+    JobQueue           mJobQueue;
+    TXQueue            mTxnQueue;
+    OrderBookDB        mOrderBookDB;
+
+    // VFALCO Clean stuff
+    ScopedPointer <NodeStore> m_nodeStore;
+    ScopedPointer <Validators> m_validators;
+    ScopedPointer <IFeatures> mFeatures;
+    ScopedPointer <IFeeVote> mFeeVote;
+    ScopedPointer <ILoadFeeTrack> mFeeTrack;
+    ScopedPointer <IHashRouter> mHashRouter;
+    ScopedPointer <IValidations> mValidations;
+    ScopedPointer <UniqueNodeList> mUNL;
+    ScopedPointer <IProofOfWorkFactory> mProofOfWorkFactory;
+    ScopedPointer <IPeers> mPeers;
+    ScopedPointer <ILoadManager> m_loadManager;
+    // VFALCO End Clean stuff
+
+    DatabaseCon* mRpcDB;
+    DatabaseCon* mTxnDB;
+    DatabaseCon* mLedgerDB;
+    DatabaseCon* mWalletDB;
+
+    ScopedPointer <PeerDoor> mPeerDoor;
+    ScopedPointer <RPCDoor>  mRPCDoor;
+    ScopedPointer <WSDoor>   mWSPublicDoor;
+    ScopedPointer <WSDoor>   mWSPrivateDoor;
+
+    boost::asio::deadline_timer mSweepTimer;
+
+    bool volatile mShutdown;
+};
+
+// VFALCO TODO Why do we even have this function?
+//             It could just be handled in the destructor.
+//
+void ApplicationImp::stop ()
+{
+    WriteLog (lsINFO, Application) << "Received shutdown request";
+    StopSustain ();
+    mShutdown = true;
+    m_mainService.stop ();
+    m_nodeStore = nullptr;
+    mValidations->flush ();
+    m_auxService.stop ();
+    mJobQueue.shutdown ();
+
+    //WriteLog (lsINFO, Application) << "Stopped: " << mIOService.stopped 
+
+    mShutdown = false;
 }
 
 void ApplicationImp::run ()
 {
+    // VFALCO TODO The unit tests crash if we try to
+    //             run these threads in the IoService constructor
+    //             so this hack makes them start later.
+    //
+    m_mainService.runExtraThreads ();
+
     if (!getConfig ().RUN_STANDALONE)
     {
         // VFALCO NOTE This seems unnecessary. If we properly refactor the load

modules/ripple_app/network/WSDoor.cpp

@@ -28,6 +28,7 @@ WSDoor::WSDoor (std::string const& strIp, int iPort, bool bPublic)
     , mIp (strIp)
     , mPort (iPort)
 {
+    startThread ();
 }
 
 WSDoor::~WSDoor ()
@@ -39,8 +40,8 @@ WSDoor::~WSDoor ()
             m_endpoint->stop ();
     }
 
-    m_thread.signalThreadShouldExit ();
+    signalThreadShouldExit ();
-    m_thread.waitForThreadToExit ();
+    waitForThreadToExit ();
 }
 
 void WSDoor::run ()
@@ -110,6 +111,6 @@ void WSDoor::stop ()
             m_endpoint->stop ();
     }
 
-    m_thread.signalThreadShouldExit ();
+    signalThreadShouldExit ();
-    m_thread.waitForThreadToExit ();
+    waitForThreadToExit ();
 }

modules/ripple_app/network/WSDoor.h

@@ -20,7 +20,7 @@ private:
     void run ();
 
 private:
-    ScopedPointer <websocketpp::server_autotls*> m_endpoint;
+    ScopedPointer <websocketpp::server_autotls> m_endpoint;
     CriticalSection m_endpointLock;
     bool                            mPublic;
     std::string                     mIp;

modules/ripple_app/ripple_app.cpp

@@ -60,8 +60,6 @@
 #include <boost/unordered_set.hpp>
 #include <boost/weak_ptr.hpp>
 
-//#include "../ripple_sqlite/ripple_sqlite.h" // for SqliteDatabase.cpp
-
 #include "../ripple_core/ripple_core.h"
 
 #include "beast/modules/beast_db/beast_db.h"

modules/ripple_core/functional/ripple_JobQueue.cpp

@@ -195,7 +195,7 @@ void JobQueue::setThreadCount (int c, bool const standaloneMode)
     }
     else if (c == 0)
     {
-        c = boost::thread::hardware_concurrency ();
+        c = SystemStats::getNumCpus ();
 
         // VFALCO NOTE According to boost, hardware_concurrency cannot return
         //             negative numbers/