Prevent DatabaseRotateImp crash on shutdown (RIPD-1392):

The DatabaseImp holds threads that access DatabaseRotateImp.  But
the DatabaseRotateImp's destructor runs before the DatabaseImp
destructor.  The DatabaseRotateImp now assures that the
DatabaseImp threads are stopped before the DatabaseRotateImp
destructor completes.

src/ripple/nodestore/impl/DatabaseImp.h

@@ -92,17 +92,15 @@ public:
             fdlimit_ = m_backend->fdlimit();
     }
 
-    ~DatabaseImp ()
+    ~DatabaseImp () override
     {
-        {
-            std::lock_guard <std::mutex> lock (m_readLock);
-            m_readShut = true;
-            m_readCondVar.notify_all ();
-            m_readGenCondVar.notify_all ();
-        }
-
-        for (auto& e : m_readThreads)
-            e.join();
+        // NOTE!
+        // Any derived class should call the stopThreads() method in its
+        // destructor.  Otherwise, occasionally, the derived class may
+        // crash during shutdown when its members are accessed by one of
+        // these threads after the derived class is destroyed but before
+        // this base class is destroyed.
+        stopThreads();
     }
 
     std::string
@@ -442,6 +440,23 @@ public:
         return fdlimit_;
     }
 
+protected:
+    void stopThreads ()
+    {
+        {
+            std::lock_guard <std::mutex> lock (m_readLock);
+            if (m_readShut) // Only stop threads once.
+                return;
+
+            m_readShut = true;
+            m_readCondVar.notify_all ();
+            m_readGenCondVar.notify_all ();
+        }
+
+        for (auto& e : m_readThreads)
+            e.join();
+    }
+
 private:
     std::atomic <std::uint32_t> m_storeCount;
     std::atomic <std::uint32_t> m_fetchTotalCount;

src/ripple/nodestore/impl/DatabaseRotatingImp.h

@@ -63,6 +63,12 @@ public:
             , archiveBackend_ (archiveBackend)
     {}
 
+    ~DatabaseRotatingImp () override
+    {
+        // Stop threads before data members are destroyed.
+        DatabaseImp::stopThreads ();
+    }
+
     std::shared_ptr <Backend> const& getWritableBackend() const override
     {
         std::lock_guard <std::mutex> lock (rotateMutex_);