From 4b1155bf32bef52eaac10d940ab49295d6e6475e Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Mon, 6 Jan 2014 20:52:27 -0800 Subject: [PATCH] Improved diagnostic for RocksDB error codes Conflicts: Builds/VisualStudio2012/RippleD.vcxproj.filters --- .../rocksdb/rocksdb/include/rocksdb/status.h | 2 ++ src/ripple_app/main/Application.cpp | 6 +++-- src/ripple_app/node/SqliteFactory.cpp | 3 ++- src/ripple_app/node/SqliteFactory.h | 3 ++- src/ripple_core/nodestore/api/Database.h | 1 + src/ripple_core/nodestore/api/Factory.h | 3 ++- src/ripple_core/nodestore/api/Types.h | 4 +++- .../nodestore/backend/HyperDBFactory.cpp | 3 ++- .../nodestore/backend/HyperDBFactory.h | 3 ++- .../nodestore/backend/KeyvaDBFactory.cpp | 3 ++- .../nodestore/backend/KeyvaDBFactory.h | 3 ++- .../nodestore/backend/LevelDBFactory.cpp | 3 ++- .../nodestore/backend/LevelDBFactory.h | 3 ++- .../nodestore/backend/MemoryFactory.cpp | 3 ++- .../nodestore/backend/MemoryFactory.h | 3 ++- .../nodestore/backend/NullFactory.cpp | 3 ++- .../nodestore/backend/NullFactory.h | 3 ++- .../nodestore/backend/RocksDBFactory.cpp | 15 ++++++++----- src/ripple_core/nodestore/impl/DatabaseImp.h | 21 ++++++++++++------ .../nodestore/tests/BackendTests.cpp | 8 +++++-- .../nodestore/tests/DatabaseTests.cpp | 22 +++++++++++++------ .../nodestore/tests/TimingTests.cpp | 5 ++++- 22 files changed, 85 insertions(+), 38 deletions(-) diff --git a/src/ripple/rocksdb/rocksdb/include/rocksdb/status.h b/src/ripple/rocksdb/rocksdb/include/rocksdb/status.h index 5f2bbbf4b..162472e2b 100644 --- a/src/ripple/rocksdb/rocksdb/include/rocksdb/status.h +++ b/src/ripple/rocksdb/rocksdb/include/rocksdb/status.h @@ -105,10 +105,12 @@ class Status { kIncomplete = 7 }; +public: Code code() const { return (state_ == nullptr) ? kOk : static_cast(state_[4]); } +private: Status(Code code, const Slice& msg, const Slice& msg2); static const char* CopyState(const char* s); }; diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index c56e9172b..4f29f1ab0 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -127,7 +127,8 @@ public: , m_nodeStoreScheduler (*m_jobQueue, *m_jobQueue) , m_nodeStore (NodeStore::Database::New ("NodeStore.main", m_nodeStoreScheduler, - getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase)) + LogPartition::getJournal (), + getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase)) , m_sntpClient (SNTPClient::New (*this)) @@ -1246,7 +1247,8 @@ void ApplicationImp::updateTables () { NodeStore::DummyScheduler scheduler; ScopedPointer source (NodeStore::Database::New ( - "NodeStore.import", scheduler, getConfig ().importNodeDatabase)); + "NodeStore.import", scheduler, LogPartition::getJournal (), + getConfig ().importNodeDatabase)); WriteLog (lsWARNING, NodeObject) << "Node import from '" << source->getName () << "' to '" diff --git a/src/ripple_app/node/SqliteFactory.cpp b/src/ripple_app/node/SqliteFactory.cpp index 0e969e0a0..4667838e9 100644 --- a/src/ripple_app/node/SqliteFactory.cpp +++ b/src/ripple_app/node/SqliteFactory.cpp @@ -245,7 +245,8 @@ String SqliteFactory::getName () const NodeStore::Backend* SqliteFactory::createInstance ( size_t keyBytes, NodeStore::Parameters const& keyValues, - NodeStore::Scheduler& scheduler) + NodeStore::Scheduler& scheduler, + Journal) { return new BackendImp (keyBytes, keyValues ["path"].toStdString (), scheduler); } diff --git a/src/ripple_app/node/SqliteFactory.h b/src/ripple_app/node/SqliteFactory.h index 1a587a407..7e862313e 100644 --- a/src/ripple_app/node/SqliteFactory.h +++ b/src/ripple_app/node/SqliteFactory.h @@ -39,7 +39,8 @@ public: NodeStore::Backend* createInstance (size_t keyBytes, NodeStore::Parameters const& keyValues, - NodeStore::Scheduler& scheduler); + NodeStore::Scheduler& scheduler, + Journal journal); }; #endif diff --git a/src/ripple_core/nodestore/api/Database.h b/src/ripple_core/nodestore/api/Database.h index 0327a85fd..041fd84ac 100644 --- a/src/ripple_core/nodestore/api/Database.h +++ b/src/ripple_core/nodestore/api/Database.h @@ -64,6 +64,7 @@ public: */ static Database* New (char const* name, Scheduler& scheduler, + Journal journal, Parameters const& backendParameters, Parameters fastBackendParameters = Parameters ()); diff --git a/src/ripple_core/nodestore/api/Factory.h b/src/ripple_core/nodestore/api/Factory.h index 4a5d7bef9..5902cd9c6 100644 --- a/src/ripple_core/nodestore/api/Factory.h +++ b/src/ripple_core/nodestore/api/Factory.h @@ -42,7 +42,8 @@ public: */ virtual Backend* createInstance (size_t keyBytes, Parameters const& parameters, - Scheduler& scheduler) = 0; + Scheduler& scheduler, + Journal journal) = 0; }; } diff --git a/src/ripple_core/nodestore/api/Types.h b/src/ripple_core/nodestore/api/Types.h index db514c0f6..947ab5637 100644 --- a/src/ripple_core/nodestore/api/Types.h +++ b/src/ripple_core/nodestore/api/Types.h @@ -37,7 +37,9 @@ enum Status ok, notFound, dataCorrupt, - unknown + unknown, + + customCode = 100 }; /** A batch of NodeObjects to write at once. */ diff --git a/src/ripple_core/nodestore/backend/HyperDBFactory.cpp b/src/ripple_core/nodestore/backend/HyperDBFactory.cpp index 502aa3567..9c599e01e 100644 --- a/src/ripple_core/nodestore/backend/HyperDBFactory.cpp +++ b/src/ripple_core/nodestore/backend/HyperDBFactory.cpp @@ -249,7 +249,8 @@ String HyperDBFactory::getName () const Backend* HyperDBFactory::createInstance ( size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler) + Scheduler& scheduler, + Journal journal) { return new HyperDBFactory::BackendImp (keyBytes, keyValues, scheduler); } diff --git a/src/ripple_core/nodestore/backend/HyperDBFactory.h b/src/ripple_core/nodestore/backend/HyperDBFactory.h index 347557607..989e427fd 100644 --- a/src/ripple_core/nodestore/backend/HyperDBFactory.h +++ b/src/ripple_core/nodestore/backend/HyperDBFactory.h @@ -44,7 +44,8 @@ public: NodeStore::Backend* createInstance (size_t keyBytes, Parameters const& keyValues, - NodeStore::Scheduler& scheduler); + NodeStore::Scheduler& scheduler, + Journal journal); }; } diff --git a/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp b/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp index f0f64e517..dd4854b98 100644 --- a/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp +++ b/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp @@ -184,7 +184,8 @@ String KeyvaDBFactory::getName () const Backend* KeyvaDBFactory::createInstance ( size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler) + Scheduler& scheduler, + Journal journal) { return new KeyvaDBFactory::BackendImp (keyBytes, keyValues, scheduler); } diff --git a/src/ripple_core/nodestore/backend/KeyvaDBFactory.h b/src/ripple_core/nodestore/backend/KeyvaDBFactory.h index be9e511f1..60437b76b 100644 --- a/src/ripple_core/nodestore/backend/KeyvaDBFactory.h +++ b/src/ripple_core/nodestore/backend/KeyvaDBFactory.h @@ -42,7 +42,8 @@ public: Backend* createInstance (size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler); + Scheduler& scheduler, + Journal journal); }; } diff --git a/src/ripple_core/nodestore/backend/LevelDBFactory.cpp b/src/ripple_core/nodestore/backend/LevelDBFactory.cpp index c2080f88b..f8f4c8e88 100644 --- a/src/ripple_core/nodestore/backend/LevelDBFactory.cpp +++ b/src/ripple_core/nodestore/backend/LevelDBFactory.cpp @@ -258,7 +258,8 @@ String LevelDBFactory::getName () const Backend* LevelDBFactory::createInstance ( size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler) + Scheduler& scheduler, + Journal journal) { return new LevelDBFactory::BackendImp (keyBytes, keyValues, scheduler); } diff --git a/src/ripple_core/nodestore/backend/LevelDBFactory.h b/src/ripple_core/nodestore/backend/LevelDBFactory.h index dd9aed9a2..ecc084669 100644 --- a/src/ripple_core/nodestore/backend/LevelDBFactory.h +++ b/src/ripple_core/nodestore/backend/LevelDBFactory.h @@ -42,7 +42,8 @@ public: Backend* createInstance (size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler); + Scheduler& scheduler, + Journal journal); private: void* m_lruCache; diff --git a/src/ripple_core/nodestore/backend/MemoryFactory.cpp b/src/ripple_core/nodestore/backend/MemoryFactory.cpp index 17847f268..f624bf667 100644 --- a/src/ripple_core/nodestore/backend/MemoryFactory.cpp +++ b/src/ripple_core/nodestore/backend/MemoryFactory.cpp @@ -121,7 +121,8 @@ String MemoryFactory::getName () const Backend* MemoryFactory::createInstance ( size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler) + Scheduler& scheduler, + Journal journal) { return new MemoryFactory::BackendImp (keyBytes, keyValues, scheduler); } diff --git a/src/ripple_core/nodestore/backend/MemoryFactory.h b/src/ripple_core/nodestore/backend/MemoryFactory.h index 1e6490950..12e62b3f4 100644 --- a/src/ripple_core/nodestore/backend/MemoryFactory.h +++ b/src/ripple_core/nodestore/backend/MemoryFactory.h @@ -42,7 +42,8 @@ public: Backend* createInstance (size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler); + Scheduler& scheduler, + Journal journal); }; } diff --git a/src/ripple_core/nodestore/backend/NullFactory.cpp b/src/ripple_core/nodestore/backend/NullFactory.cpp index ca64a95ab..f7dabec62 100644 --- a/src/ripple_core/nodestore/backend/NullFactory.cpp +++ b/src/ripple_core/nodestore/backend/NullFactory.cpp @@ -86,7 +86,8 @@ String NullFactory::getName () const Backend* NullFactory::createInstance ( size_t, Parameters const&, - Scheduler& scheduler) + Scheduler& scheduler, + Journal journal) { return new NullFactory::BackendImp (scheduler); } diff --git a/src/ripple_core/nodestore/backend/NullFactory.h b/src/ripple_core/nodestore/backend/NullFactory.h index 84739c750..6b449bda7 100644 --- a/src/ripple_core/nodestore/backend/NullFactory.h +++ b/src/ripple_core/nodestore/backend/NullFactory.h @@ -44,7 +44,8 @@ public: Backend* createInstance (size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler); + Scheduler& scheduler, + Journal journal); }; } diff --git a/src/ripple_core/nodestore/backend/RocksDBFactory.cpp b/src/ripple_core/nodestore/backend/RocksDBFactory.cpp index 1a86803bd..9bfc7a2e6 100644 --- a/src/ripple_core/nodestore/backend/RocksDBFactory.cpp +++ b/src/ripple_core/nodestore/backend/RocksDBFactory.cpp @@ -86,8 +86,10 @@ public: BackendImp (int keyBytes, Parameters const& keyValues, - Scheduler& scheduler) - : m_keyBytes (keyBytes) + Scheduler& scheduler, + Journal journal) + : m_journal (journal) + , m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_batch (*this, scheduler) , m_name (keyValues ["path"].toStdString ()) @@ -200,7 +202,9 @@ public: } else { - status = unknown; + status = Status (customCode + getStatus.code()); + + m_journal.error << getStatus.ToString (); } } } @@ -285,6 +289,7 @@ public: } private: + Journal m_journal; size_t const m_keyBytes; Scheduler& m_scheduler; BatchWriter m_batch; @@ -323,10 +328,10 @@ public: Backend* createInstance ( size_t keyBytes, Parameters const& keyValues, - Scheduler& scheduler) + Scheduler& scheduler, Journal journal) { return new RocksDBFactory::BackendImp ( - keyBytes, keyValues, scheduler); + keyBytes, keyValues, scheduler, journal); } }; diff --git a/src/ripple_core/nodestore/impl/DatabaseImp.h b/src/ripple_core/nodestore/impl/DatabaseImp.h index fdcb18ca5..d097ca1ee 100644 --- a/src/ripple_core/nodestore/impl/DatabaseImp.h +++ b/src/ripple_core/nodestore/impl/DatabaseImp.h @@ -28,11 +28,13 @@ public: DatabaseImp (char const* name, Scheduler& scheduler, Parameters const& backendParameters, - Parameters const& fastBackendParameters) - : m_scheduler (scheduler) - , m_backend (createBackend (backendParameters, scheduler)) + Parameters const& fastBackendParameters, + Journal journal) + : m_journal (journal) + , m_scheduler (scheduler) + , m_backend (createBackend (backendParameters, scheduler, journal)) , m_fastBackend ((fastBackendParameters.size () > 0) - ? createBackend (fastBackendParameters, scheduler) : nullptr) + ? createBackend (fastBackendParameters, scheduler, journal) : nullptr) , m_cache ("NodeStore", 16384, 300) { } @@ -259,7 +261,8 @@ public: fatal_error ("Your rippled.cfg is missing a [node_db] entry, please see the rippled-example.cfg file!"); } - static Backend* createBackend (Parameters const& parameters, Scheduler& scheduler) + static Backend* createBackend (Parameters const& parameters, + Scheduler& scheduler, Journal journal) { Backend* backend = nullptr; @@ -271,7 +274,8 @@ public: if (factory != nullptr) { - backend = factory->createInstance (NodeObject::keyBytes, parameters, scheduler); + backend = factory->createInstance ( + NodeObject::keyBytes, parameters, scheduler, journal); } else { @@ -289,6 +293,8 @@ public: //------------------------------------------------------------------------------ private: + Journal m_journal; + Scheduler& m_scheduler; // Persistent key/value storage. @@ -333,11 +339,12 @@ void Database::addAvailableBackends () Database* Database::New (char const* name, Scheduler& scheduler, + Journal journal, Parameters const& backendParameters, Parameters fastBackendParameters) { return new DatabaseImp (name, - scheduler, backendParameters, fastBackendParameters); + scheduler, backendParameters, fastBackendParameters, journal); } } diff --git a/src/ripple_core/nodestore/tests/BackendTests.cpp b/src/ripple_core/nodestore/tests/BackendTests.cpp index fbc26fb27..e30123a51 100644 --- a/src/ripple_core/nodestore/tests/BackendTests.cpp +++ b/src/ripple_core/nodestore/tests/BackendTests.cpp @@ -40,9 +40,12 @@ public: Batch batch; createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + Journal j ((journal ())); + { // Open the backend - ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + ScopedPointer backend (DatabaseImp::createBackend ( + params, scheduler, j)); // Write the batch storeBatch (*backend, batch); @@ -65,7 +68,8 @@ public: { // Re-open the backend - ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + ScopedPointer backend (DatabaseImp::createBackend ( + params, scheduler, j)); // Read it back in Batch copy; diff --git a/src/ripple_core/nodestore/tests/DatabaseTests.cpp b/src/ripple_core/nodestore/tests/DatabaseTests.cpp index 73e5c12a1..437a354a9 100644 --- a/src/ripple_core/nodestore/tests/DatabaseTests.cpp +++ b/src/ripple_core/nodestore/tests/DatabaseTests.cpp @@ -47,9 +47,12 @@ public: Batch batch; createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + Journal j ((journal ())); + // Write to source db { - ScopedPointer src (Database::New ("test", scheduler, srcParams)); + ScopedPointer src (Database::New ( + "test", scheduler, j, srcParams)); storeBatch (*src, batch); } @@ -57,7 +60,8 @@ public: { // Re-open the db - ScopedPointer src (Database::New ("test", scheduler, srcParams)); + ScopedPointer src (Database::New ( + "test", scheduler, j, srcParams)); // Set up the destination database File const dest_db (File::createTempFile ("dest_db")); @@ -65,7 +69,8 @@ public: destParams.set ("type", destBackendType); destParams.set ("path", dest_db.getFullPathName ()); - ScopedPointer dest (Database::New ("test", scheduler, destParams)); + ScopedPointer dest (Database::New ( + "test", scheduler, j, destParams)); beginTestCase (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); @@ -80,7 +85,6 @@ public: std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); expect (areBatchesEqual (batch, copy), "Should be equal"); - } //-------------------------------------------------------------------------- @@ -117,9 +121,12 @@ public: Batch batch; createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + Journal j ((journal ())); + { // Open the database - ScopedPointer db (Database::New ("test", scheduler, nodeParams, tempParams)); + ScopedPointer db (Database::New ("test", scheduler, + j, nodeParams, tempParams)); // Write the batch storeBatch (*db, batch); @@ -144,7 +151,8 @@ public: { { // Re-open the database without the ephemeral DB - ScopedPointer db (Database::New ("test", scheduler, nodeParams)); + ScopedPointer db (Database::New ( + "test", scheduler, j, nodeParams)); // Read it back in Batch copy; @@ -160,7 +168,7 @@ public: { // Verify the ephemeral db ScopedPointer db (Database::New ("test", - scheduler, tempParams, StringPairArray ())); + scheduler, j, tempParams, StringPairArray ())); // Read it back in Batch copy; diff --git a/src/ripple_core/nodestore/tests/TimingTests.cpp b/src/ripple_core/nodestore/tests/TimingTests.cpp index 34f745def..b072e376d 100644 --- a/src/ripple_core/nodestore/tests/TimingTests.cpp +++ b/src/ripple_core/nodestore/tests/TimingTests.cpp @@ -77,8 +77,11 @@ public: NodeStore::Batch batch2; createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); + Journal j ((journal ())); + // Open the backend - ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + ScopedPointer backend (DatabaseImp::createBackend ( + params, scheduler, j)); Stopwatch t;