diff --git a/src/Application.cpp b/src/Application.cpp index ebca2d4524..f85992d93b 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -37,7 +37,7 @@ DatabaseCon::~DatabaseCon() Application::Application() : mUNL(mIOService), - mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), + mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), mHashedObjectStore(16384, 300), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), mHashNodeDB(NULL), mNetNodeDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { diff --git a/src/Application.h b/src/Application.h index 629976dadb..4bdde4c19a 100644 --- a/src/Application.h +++ b/src/Application.h @@ -49,6 +49,7 @@ class Application NodeCache mNodeCache; ValidationCollection mValidations; SuppressionTable mSuppressions; + HashedObjectStore mHashedObjectStore; DatabaseCon *mTxnDB, *mLedgerDB, *mWalletDB, *mHashNodeDB, *mNetNodeDB; @@ -80,6 +81,7 @@ public: LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } NodeCache& getNodeCache() { return mNodeCache; } + HashedObjectStore& getHashedObjectStore() { return mHashedObjectStore; } ValidationCollection& getValidations() { return mValidations; } bool suppress(const uint256& s) { return mSuppressions.addSuppression(s); } bool suppress(const uint160& s) { return mSuppressions.addSuppression(s); } diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index d51b0cd545..63ff9f633f 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -7,7 +7,6 @@ #include "Application.h" #include "Log.h" - bool HashedObject::checkHash() const { uint256 hash = Serializer::getSHA512Half(mData); @@ -27,18 +26,19 @@ void HashedObject::setHash() mHash = Serializer::getSHA512Half(mData); } -// FIXME: Stores should be added to a queue that's services by an auxilliary thread or from an +// FIXME: Stores should be added to a queue that's serviced by an auxilliary thread or from an // auxilliary thread pool. These should be tied into a cache, since you need one to handle // an immedate read back (before the write completes) -bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector& data, - const uint256& hash) +bool HashedObjectStore::store(HashedObjectType type, uint32 index, + const std::vector& data, const uint256& hash) { if (!theApp->getHashNodeDB()) return true; -#ifdef DEBUG - Serializer s(data); - assert(hash == s.getSHA512Half()); -#endif + HashedObject::pointer object = boost::make_shared(type, index, data); + object->setHash(); + if (object->getHash() != hash) + throw std::runtime_error("Object added to store doesn't have valid hash"); + std::string sql = "INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('"; sql.append(hash.GetHex()); switch(type) @@ -51,7 +51,6 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< } sql.append(boost::lexical_cast(index)); sql.append("',"); - std::string obj; theApp->getHashNodeDB()->getDB()->escape(&(data.front()), data.size(), obj); sql.append(obj); @@ -61,22 +60,23 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< boost::str(boost::format("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';") % hash.GetHex()); ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); + if (mCache.canonicalize(hash, object)) + return false; Database* db = theApp->getHashNodeDB()->getDB(); if (SQL_EXISTS(db, exists)) return false; return db->executeSQL(sql); } -bool HashedObject::store() const +HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) { -#ifdef DEBUG - assert(checkHash()); -#endif - return store(mType, mLedgerIndex, mData, mHash); -} + HashedObject::pointer obj; + { + ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); + obj = mCache.fetch(hash); + if (obj) return obj; + } -HashedObject::pointer HashedObject::retrieve(const uint256& hash) -{ if (!theApp || !theApp->getHashNodeDB()) return HashedObject::pointer(); std::string sql = "SELECT * FROM CommittedObjects WHERE Hash='"; sql.append(hash.GetHex()); @@ -102,34 +102,37 @@ HashedObject::pointer HashedObject::retrieve(const uint256& hash) data.resize(size); db->getBinary("Object", &(data.front()), size); db->endIterRows(); - } - HashedObjectType htype = UNKNOWN; - switch(type[0]) - { - case 'L': htype = LEDGER; break; - case 'T': htype = TRANSACTION; break; - case 'A': htype = ACCOUNT_NODE; break; - case 'N': htype = TRANSACTION_NODE; break; - default: - Log(lsERROR) << "Invalid hashed object"; - return HashedObject::pointer(); - } + HashedObjectType htype = UNKNOWN; + switch(type[0]) + { + case 'L': htype = LEDGER; break; + case 'T': htype = TRANSACTION; break; + case 'A': htype = ACCOUNT_NODE; break; + case 'N': htype = TRANSACTION_NODE; break; + default: + Log(lsERROR) << "Invalid hashed object"; + return HashedObject::pointer(); + } - HashedObject::pointer obj = boost::make_shared(htype, index, data); - obj->mHash = hash; + obj = boost::make_shared(htype, index, data); + obj->mHash = hash; + mCache.canonicalize(hash, obj); + } #ifdef DEBUG assert(obj->checkHash()); #endif return obj; } -HashedObjectBulkWriter::HashedObjectBulkWriter() : sl(theApp->getHashNodeDB()->getDBLock()) +ScopedLock HashedObjectStore::beginBulk() { + ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); theApp->getHashNodeDB()->getDB()->executeSQL("BEGIN TRANSACTION;"); + return sl; } -HashedObjectBulkWriter::~HashedObjectBulkWriter() +void HashedObjectStore::endBulk() { theApp->getHashNodeDB()->getDB()->executeSQL("END TRANSACTION;"); } diff --git a/src/HashedObject.h b/src/HashedObject.h index 6a0fa28693..0f66c1509e 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -6,6 +6,7 @@ #include "types.h" #include "uint256.h" #include "ScopedLock.h" +#include "TaggedCache.h" enum HashedObjectType { @@ -34,29 +35,41 @@ public: void setHash(); const std::vector& getData() { return mData; } + const uint256& getHash() { return mHash; } +}; - bool store() const; +class HashedObjectStore +{ +protected: + TaggedCache mCache; - static bool store(HashedObjectType type, uint32 index, const std::vector& data, +public: + + HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge) { ; } + + bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); - static HashedObject::pointer retrieve(const uint256& hash); + HashedObject::pointer retrieve(const uint256& hash); + + ScopedLock beginBulk(); + void endBulk(); }; class HashedObjectBulkWriter { protected: + HashedObjectStore& mStore; ScopedLock sl; public: + HashedObjectBulkWriter(HashedObjectStore& ostore) : mStore(ostore), sl(mStore.beginBulk()) { ; } + ~HashedObjectBulkWriter() { mStore.endBulk(); } - HashedObjectBulkWriter(); - ~HashedObjectBulkWriter(); + bool store(HashedObjectType type, uint32 index, const std::vector& data, + const uint256& hash) { return mStore.store(type, index, data, hash); } - bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) - { return HashedObject::store(type, index, data, hash); } - HashedObject::pointer retrieve(const uint256& hash) - { return HashedObject::retrieve(hash); } + HashedObject::pointer retrieve(const uint256& hash) { return mStore.retrieve(hash); } }; #endif diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 987a088d31..e3961f5901 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -623,7 +623,7 @@ void SHAMapItem::dump() bool SHAMap::fetchNode(const uint256& hash, std::vector& data) { - HashedObject::pointer obj(HashedObject::retrieve(hash)); + HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); if(!obj) return false; data = obj->getData(); return true; @@ -642,7 +642,7 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) if(mDirtyNodes) { - HashedObjectBulkWriter bw; + HashedObjectBulkWriter bw(theApp->getHashedObjectStore()); boost::unordered_map& dirtyNodes = *mDirtyNodes; boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end())