From 5df387017b670fff71013ef0a588825f658c210f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Wed, 15 May 2013 21:07:01 +0000 Subject: [PATCH 01/15] Normalize line endings --- rippled.1 | Bin 3121 -> 3043 bytes src/cpp/json/LICENSE | Bin 67 -> 66 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/rippled.1 b/rippled.1 index c94248ae6baa9f963da886168721cd17b2df630f..8617000b89c97f6bc8e02bacaa3ca09c7bb5f982 100644 GIT binary patch delta 614 zcmZvY!E4iS6vt_rwArufJlM_+Cr?{cv`c9QDnlxjr8*dGu~Kx1;g{vtEs&-)$<(74 z5%IF_tB42v1H9<&9z2N${{le}JP4k=?q=H9%FxsM@;=}9-j}y~^!v*+pKJ+1 zRX6A@4ep#DJl6-GP7Hn)FrBDmnAQkrnfv!04{M29cj}NJNd{MIarCdTYl|8_jwi1#PiY{4EBbQ*ZfA#I zqv37i9vStv*{)tU%jl}da^?RaJwo$%svgH1wT?aY+J({GCa2O?;Fp3;?u$~#afRve zu3)v5Rl_Wk{WtO0j?7f?NX+$pd>j)$rb>t62v=NMupPbulg*&QXk2qKHe9B)HmPM8 xR9$T~2=7>9C)2Dk?WRwh%2cG0w1#q|)+dUS{-kZCtbAp~@h#Ob0WWtCk_vk}TO4$l97_I`HIQ zpz}NHMFc@Sc=6&)T}Nw+;R?GtOtlOJjtZxkI7f z?FOD(X8f&iADw-n0lb$w58O*#2EIw%1pZ0Q04GEvGw5Udffx^ZCQbw2iLw2gIITd` zL$brI*}o+jO$CaV$!_ZcD*q0+n!62w-=GhxRox$ z{WKjXJ_Kct6jz)WiAe4 zr}(E#3G7EkI~rZFX*005lI*2`jwV-}#PM%*C#xdxRkl$;W3xo@!Hmabn2lCLUu#tv zujM9-)ni317N_I`%?P-in-GZ@-_4l_{gS%{%*&Ifn6k9IVzjE{j&1|J^88TI{#UR% zI^WAz#tyP?1XOW-EXU6(1xS6FfQS6d;SoUait2_z`byoxQ3Az+^Ru#QLR!avrz(}> hhXwA$JfRySYLhmsIwNY++^9N-;5+#R1V7K;{R_fi!xjJl diff --git a/src/cpp/json/LICENSE b/src/cpp/json/LICENSE index d20fb29a7e23828c15487438c93ae790d2045707..1c37b1181031ce5347aa15666b380884e819d6d8 100644 GIT binary patch delta 6 NcmZ>En&8OD1po!{0b&3E delta 7 OcmZ>Ap5VyD%LM=jWdU#i From 1bef09b0ef1786d498b55f1b1421ef2efa1eac3b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:53:49 -0700 Subject: [PATCH 02/15] Operation to extract transaction ID from CanonicalTXSet entries. --- src/cpp/ripple/CanonicalTXSet.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/CanonicalTXSet.h b/src/cpp/ripple/CanonicalTXSet.h index fead540db..a28b004e5 100644 --- a/src/cpp/ripple/CanonicalTXSet.h +++ b/src/cpp/ripple/CanonicalTXSet.h @@ -23,6 +23,8 @@ public: bool operator==(const CanonicalTXKey& k) const { return mTXid == k.mTXid; } bool operator!=(const CanonicalTXKey& k) const { return mTXid != k.mTXid; } + + const uint256& getTXID() const { return mTXid; } }; class CanonicalTXSet From a690ca658d29b735bf807211b52ac0eb3e2dd6e8 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:54:30 -0700 Subject: [PATCH 03/15] Cache signature state inside SerializedTransaction. --- src/cpp/ripple/SerializedTransaction.cpp | 25 +++++++++++++++++++----- src/cpp/ripple/SerializedTransaction.h | 6 ++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/SerializedTransaction.cpp b/src/cpp/ripple/SerializedTransaction.cpp index 0d7dd34f2..ccd5f073e 100644 --- a/src/cpp/ripple/SerializedTransaction.cpp +++ b/src/cpp/ripple/SerializedTransaction.cpp @@ -11,7 +11,8 @@ SETUP_LOG(); DECLARE_INSTANCE(SerializedTransaction); -SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sfTransaction), mType(type) +SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sfTransaction), mType(type), + mSigGood(false), mSigBad(false) { mFormat = TransactionFormat::getTxnFormat(type); if (mFormat == NULL) @@ -23,7 +24,8 @@ SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sf setFieldU16(sfTransactionType, mFormat->t_type); } -SerializedTransaction::SerializedTransaction(const STObject& object) : STObject(object) +SerializedTransaction::SerializedTransaction(const STObject& object) : STObject(object), + mSigGood(false), mSigBad(false) { mType = static_cast(getFieldU16(sfTransactionType)); mFormat = TransactionFormat::getTxnFormat(mType); @@ -38,7 +40,8 @@ SerializedTransaction::SerializedTransaction(const STObject& object) : STObject( } } -SerializedTransaction::SerializedTransaction(SerializerIterator& sit) : STObject(sfTransaction) +SerializedTransaction::SerializedTransaction(SerializerIterator& sit) : STObject(sfTransaction), + mSigGood(false), mSigBad(false) { int length = sit.getBytesLeft(); if ((length < TransactionMinLen) || (length > TransactionMaxLen)) @@ -156,16 +159,28 @@ void SerializedTransaction::sign(const RippleAddress& naAccountPrivate) bool SerializedTransaction::checkSign() const { + if (mSigGood) + return true; + + if (mSigBad) + return false; + try { RippleAddress n; n.setAccountPublic(getFieldVL(sfSigningPubKey)); - return checkSign(n); + if (checkSign(n)) + { + mSigGood = true; + return true; + } } catch (...) { - return false; + ; } + mSigBad = true; + return false; } bool SerializedTransaction::checkSign(const RippleAddress& naAccountPublic) const diff --git a/src/cpp/ripple/SerializedTransaction.h b/src/cpp/ripple/SerializedTransaction.h index 5dfc24f27..6b9dde135 100644 --- a/src/cpp/ripple/SerializedTransaction.h +++ b/src/cpp/ripple/SerializedTransaction.h @@ -32,6 +32,8 @@ protected: SerializedTransaction* duplicate() const { return new SerializedTransaction(*this); } + mutable bool mSigGood, mSigBad; + public: SerializedTransaction(SerializerIterator& sit); SerializedTransaction(TransactionType type); @@ -69,6 +71,10 @@ public: void sign(const RippleAddress& naAccountPrivate); bool checkSign(const RippleAddress& naAccountPublic) const; bool checkSign() const; + bool isKnownGood() const { return mSigGood; } + bool isKnownBad() const { return mSigBad; } + void setGood() const { mSigGood = true; } + void setBad() const { mSigBad = true; } // SQL Functions static std::string getSQLValueHeader(); From 285ef764d3ede0490c5cae9128fb151ba7ebc768 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:54:52 -0700 Subject: [PATCH 04/15] Don't stash the key when we don't need it. --- src/cpp/ripple/NetworkOPs.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 3ec5d8a93..4b09dde55 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -184,8 +184,7 @@ void NetworkOPs::submitTransaction(Job&, SerializedTransaction::pointer iTrans, { try { - RippleAddress fromPubKey = RippleAddress::createAccountPublic(trans->getSigningPubKey()); - if (!trans->checkSign(fromPubKey)) + if (!trans->checkSign()) { cLog(lsWARNING) << "Submitted transaction has bad signature"; theApp->isNewFlag(suppress, SF_BAD); From 586c6aee3192679958e40ca1984a9e06fa321588 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:55:23 -0700 Subject: [PATCH 05/15] Bypass signature check if it has already been done. --- src/cpp/ripple/LedgerConsensus.cpp | 2 ++ src/cpp/ripple/LedgerMaster.cpp | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index a312d30ff..556ea0de9 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -1043,6 +1043,8 @@ int LedgerConsensus::applyTransaction(TransactionEngine& engine, SerializedTrans TransactionEngineParams parms = openLedger ? tapOPEN_LEDGER : tapNONE; if (retryAssured) parms = static_cast(parms | tapRETRY); + if (theApp->isNewFlag(txn->getTransactionID(), SF_SIGGOOD)) + parms = static_cast(parms | tapNO_CHECK_SIGN); cLog(lsDEBUG) << "TXN " << txn->getTransactionID() << (openLedger ? " open" : " closed") diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 23d2c7ab7..dd93c393e 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -101,8 +101,13 @@ Ledger::pointer LedgerMaster::closeLedger(bool recover) { try { + TransactionEngineParams tepFlags = tapOPEN_LEDGER; + + if (theApp->isNew(it->first.getTXID(), SF_SIGGOOD)); + tepFlags = static_cast(tepFlags | tapNO_CHECK_SIGN); + bool didApply; - mEngine.applyTransaction(*it->second, tapOPEN_LEDGER, didApply); + mEngine.applyTransaction(*it->second, tepFlags, didApply); if (didApply) ++recovers; } From b00ac6494164058094d6b99977afd32eb0a204e5 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:55:35 -0700 Subject: [PATCH 06/15] Update the signature check flags if needed. --- src/cpp/ripple/Transactor.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/Transactor.cpp b/src/cpp/ripple/Transactor.cpp index 82b8ab9bc..daa6171b3 100644 --- a/src/cpp/ripple/Transactor.cpp +++ b/src/cpp/ripple/Transactor.cpp @@ -160,7 +160,6 @@ TER Transactor::checkSeq() // check stuff before you bother to lock the ledger TER Transactor::preCheck() { - mTxnAccountID = mTxn.getSourceAccount().getAccountID(); if (!mTxnAccountID) { @@ -177,11 +176,15 @@ TER Transactor::preCheck() mSigningPubKey = RippleAddress::createAccountPublic(mTxn.getSigningPubKey()); // Consistency: really signed. - if ( !isSetBit(mParams, tapNO_CHECK_SIGN) && !mTxn.checkSign(mSigningPubKey)) + if (!mTxn.isKnownGood()) { - cLog(lsWARNING) << "applyTransaction: Invalid transaction: bad signature"; - - return temINVALID; + if (mTxn.isKnownBad() || (!isSetBit(mParams, tapNO_CHECK_SIGN) && !mTxn.checkSign(mSigningPubKey))) + { + mTxn.setGood(); + cLog(lsWARNING) << "applyTransaction: Invalid transaction: bad signature"; + return temINVALID; + } + mTxn.isKnownGood(); } return tesSUCCESS; From afbaa469243de6a70ae08465cef502aeb4ef9a20 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 17:32:44 -0700 Subject: [PATCH 07/15] Upgrade LevelDB to 1.10.0, mostly for better write stall logging. --- src/cpp/leveldb/db/db_impl.cc | 17 +++++++++++---- src/cpp/leveldb/db/db_test.cc | 31 ++++++++++++++++++++++++++++ src/cpp/leveldb/db/dbformat.cc | 2 +- src/cpp/leveldb/include/leveldb/db.h | 2 +- src/cpp/leveldb/table/block.cc | 13 ++++++------ src/cpp/leveldb/table/table.cc | 1 - src/cpp/leveldb/table/table_test.cc | 30 +++++++++++++++++++++++++++ src/cpp/leveldb/util/cache.cc | 5 +---- src/cpp/leveldb/util/env_posix.cc | 4 ++-- src/cpp/leveldb/util/hash.cc | 11 ++++++++-- 10 files changed, 95 insertions(+), 21 deletions(-) diff --git a/src/cpp/leveldb/db/db_impl.cc b/src/cpp/leveldb/db/db_impl.cc index c9de169f2..cd765c523 100644 --- a/src/cpp/leveldb/db/db_impl.cc +++ b/src/cpp/leveldb/db/db_impl.cc @@ -310,16 +310,24 @@ Status DBImpl::Recover(VersionEdit* edit) { if (!s.ok()) { return s; } + std::set expected; + versions_->AddLiveFiles(&expected); uint64_t number; FileType type; std::vector logs; for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type) - && type == kLogFile - && ((number >= min_log) || (number == prev_log))) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) logs.push_back(number); } } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } // Recover in the order in which the logs were generated std::sort(logs.begin(), logs.end()); @@ -1268,10 +1276,11 @@ Status DBImpl::MakeRoomForWrite(bool force) { } else if (imm_ != NULL) { // We have filled up the current memtable, but the previous // one is still being compacted, so we wait. + Log(options_.info_log, "Current memtable full; waiting...\n"); bg_cv_.Wait(); } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { // There are too many level-0 files. - Log(options_.info_log, "waiting...\n"); + Log(options_.info_log, "Too many L0 files; waiting...\n"); bg_cv_.Wait(); } else { // Attempt to switch to a new memtable and trigger compaction of old diff --git a/src/cpp/leveldb/db/db_test.cc b/src/cpp/leveldb/db/db_test.cc index 684ea3bdb..2f51296be 100644 --- a/src/cpp/leveldb/db/db_test.cc +++ b/src/cpp/leveldb/db/db_test.cc @@ -461,6 +461,20 @@ class DBTest { } return result; } + + bool DeleteAnSSTFile() { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number))); + return true; + } + } + return false; + } }; TEST(DBTest, Empty) { @@ -1567,6 +1581,23 @@ TEST(DBTest, ManifestWriteError) { } } +TEST(DBTest, MissingSSTFile) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + + ASSERT_TRUE(DeleteAnSSTFile()); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) + << s.ToString(); +} + TEST(DBTest, FilesDeletedAfterCompaction) { ASSERT_OK(Put("foo", "v2")); Compact("a", "z"); diff --git a/src/cpp/leveldb/db/dbformat.cc b/src/cpp/leveldb/db/dbformat.cc index 28e11b398..20a7ca446 100644 --- a/src/cpp/leveldb/db/dbformat.cc +++ b/src/cpp/leveldb/db/dbformat.cc @@ -26,7 +26,7 @@ std::string ParsedInternalKey::DebugString() const { (unsigned long long) sequence, int(type)); std::string result = "'"; - result += user_key.ToString(); + result += EscapeString(user_key.ToString()); result += buf; return result; } diff --git a/src/cpp/leveldb/include/leveldb/db.h b/src/cpp/leveldb/include/leveldb/db.h index 29d367447..a37c097d1 100644 --- a/src/cpp/leveldb/include/leveldb/db.h +++ b/src/cpp/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 9; +static const int kMinorVersion = 10; struct Options; struct ReadOptions; diff --git a/src/cpp/leveldb/table/block.cc b/src/cpp/leveldb/table/block.cc index ab83c1112..79ea9d9ee 100644 --- a/src/cpp/leveldb/table/block.cc +++ b/src/cpp/leveldb/table/block.cc @@ -16,7 +16,7 @@ namespace leveldb { inline uint32_t Block::NumRestarts() const { - assert(size_ >= 2*sizeof(uint32_t)); + assert(size_ >= sizeof(uint32_t)); return DecodeFixed32(data_ + size_ - sizeof(uint32_t)); } @@ -27,11 +27,12 @@ Block::Block(const BlockContents& contents) if (size_ < sizeof(uint32_t)) { size_ = 0; // Error marker } else { - restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); - if (restart_offset_ > size_ - sizeof(uint32_t)) { - // The size is too small for NumRestarts() and therefore - // restart_offset_ wrapped around. + size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t); + if (NumRestarts() > max_restarts_allowed) { + // The size is too small for NumRestarts() size_ = 0; + } else { + restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); } } } @@ -253,7 +254,7 @@ class Block::Iter : public Iterator { }; Iterator* Block::NewIterator(const Comparator* cmp) { - if (size_ < 2*sizeof(uint32_t)) { + if (size_ < sizeof(uint32_t)) { return NewErrorIterator(Status::Corruption("bad block contents")); } const uint32_t num_restarts = NumRestarts(); diff --git a/src/cpp/leveldb/table/table.cc b/src/cpp/leveldb/table/table.cc index dbd6d3a1b..71c1756e5 100644 --- a/src/cpp/leveldb/table/table.cc +++ b/src/cpp/leveldb/table/table.cc @@ -228,7 +228,6 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k, !filter->KeyMayMatch(handle.offset(), k)) { // Not found } else { - Slice handle = iiter->value(); Iterator* block_iter = BlockReader(this, options, iiter->value()); block_iter->Seek(k); if (block_iter->Valid()) { diff --git a/src/cpp/leveldb/table/table_test.cc b/src/cpp/leveldb/table/table_test.cc index 57cea2533..c723bf84c 100644 --- a/src/cpp/leveldb/table/table_test.cc +++ b/src/cpp/leveldb/table/table_test.cc @@ -644,6 +644,36 @@ class Harness { Constructor* constructor_; }; +// Test empty table/block. +TEST(Harness, Empty) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Test(&rnd); + } +} + +// Special test for a block with no restart entries. The C++ leveldb +// code never generates such blocks, but the Java version of leveldb +// seems to. +TEST(Harness, ZeroRestartPointsInBlock) { + char data[sizeof(uint32_t)]; + memset(data, 0, sizeof(data)); + BlockContents contents; + contents.data = Slice(data, sizeof(data)); + contents.cachable = false; + contents.heap_allocated = false; + Block block(contents); + Iterator* iter = block.NewIterator(BytewiseComparator()); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(!iter->Valid()); + iter->Seek("foo"); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + // Test the empty key TEST(Harness, SimpleEmptyKey) { for (int i = 0; i < kNumTestArgs; i++) { diff --git a/src/cpp/leveldb/util/cache.cc b/src/cpp/leveldb/util/cache.cc index 24f1f63f4..8b197bc02 100644 --- a/src/cpp/leveldb/util/cache.cc +++ b/src/cpp/leveldb/util/cache.cc @@ -116,7 +116,6 @@ class HandleTable { LRUHandle* h = list_[i]; while (h != NULL) { LRUHandle* next = h->next_hash; - Slice key = h->key(); uint32_t hash = h->hash; LRUHandle** ptr = &new_list[hash & (new_length - 1)]; h->next_hash = *ptr; @@ -160,7 +159,6 @@ class LRUCache { // mutex_ protects the following state. port::Mutex mutex_; size_t usage_; - uint64_t last_id_; // Dummy head of LRU list. // lru.prev is newest entry, lru.next is oldest entry. @@ -170,8 +168,7 @@ class LRUCache { }; LRUCache::LRUCache() - : usage_(0), - last_id_(0) { + : usage_(0) { // Make empty circular linked list lru_.next = &lru_; lru_.prev = &lru_; diff --git a/src/cpp/leveldb/util/env_posix.cc b/src/cpp/leveldb/util/env_posix.cc index db81f56d1..b328183ca 100644 --- a/src/cpp/leveldb/util/env_posix.cc +++ b/src/cpp/leveldb/util/env_posix.cc @@ -386,7 +386,7 @@ class PosixEnv : public Env { PosixEnv(); virtual ~PosixEnv() { fprintf(stderr, "Destroying Env::Default()\n"); - exit(1); + abort(); } virtual Status NewSequentialFile(const std::string& fname, @@ -589,7 +589,7 @@ class PosixEnv : public Env { void PthreadCall(const char* label, int result) { if (result != 0) { fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); - exit(1); + abort(); } } diff --git a/src/cpp/leveldb/util/hash.cc b/src/cpp/leveldb/util/hash.cc index ba1818082..07cf02206 100644 --- a/src/cpp/leveldb/util/hash.cc +++ b/src/cpp/leveldb/util/hash.cc @@ -6,6 +6,13 @@ #include "util/coding.h" #include "util/hash.h" +// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through +// between switch labels. The real definition should be provided externally. +// This one is a fallback version for unsupported compilers. +#ifndef FALLTHROUGH_INTENDED +#define FALLTHROUGH_INTENDED do { } while (0) +#endif + namespace leveldb { uint32_t Hash(const char* data, size_t n, uint32_t seed) { @@ -28,10 +35,10 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) { switch (limit - data) { case 3: h += data[2] << 16; - // fall through + FALLTHROUGH_INTENDED; case 2: h += data[1] << 8; - // fall through + FALLTHROUGH_INTENDED; case 1: h += data[0]; h *= m; From 0c5d4280d4570702d61ffd0a2e8eb2842434d432 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 15 May 2013 19:41:50 -0700 Subject: [PATCH 08/15] SConstruct changes. --- SConstruct | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index 6028302ff..02e57f840 100644 --- a/SConstruct +++ b/SConstruct @@ -147,10 +147,12 @@ WEBSOCKETPP_SRCS = [ ] RIPPLE_SRCS = glob.glob('src/cpp/ripple/*.cpp') -PROTO_SRCS = env.Protoc([], 'src/cpp/ripple/ripple.proto', PROTOCOUTDIR='build/proto', PROTOCPYTHONOUTDIR=None) -env.Append(CXXFLAGS = ['-Ibuild/proto']) +PROTO_SRCS = [ 'src/cpp/protobuf_core.cpp' ] +# PROTO_SRCS = env.Protoc([], 'src/cpp/ripple/ripple.proto', PROTOCOUTDIR='build/proto', PROTOCPYTHONOUTDIR=None) +# env.Append(CXXFLAGS = ['-Ibuild/proto']) +env.Append(CXXFLAGS = ['-Ibuild/proto', '-Isrc/cpp/protobuf/src', '-Isrc/cpp/protobuf/vsprojects' ]) -env.Clean(PROTO_SRCS, 'site_scons/site_tools/protoc.pyc') +# env.Clean(PROTO_SRCS, 'site_scons/site_tools/protoc.pyc') # Remove unused source files. UNUSED_SRCS = [] From 28c3af76cbcf6493b4134729d623bcfe8f682226 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 15 May 2013 20:17:19 -0700 Subject: [PATCH 09/15] Support unity build for leveldb and scons. --- SConstruct | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/SConstruct b/SConstruct index 02e57f840..7ba8277f9 100644 --- a/SConstruct +++ b/SConstruct @@ -47,7 +47,7 @@ else: # # Put objects files in their own directory. # -for dir in ['ripple', 'database', 'json', 'leveldb/db', 'leveldb/port', 'leveldb/include', 'leveldb/table', 'leveldb/util', 'websocketpp']: +for dir in ['.', 'ripple', 'database', 'json', 'leveldb/db', 'leveldb/port', 'leveldb/include', 'leveldb/table', 'leveldb/util', 'websocketpp']: VariantDir('build/obj/'+dir, 'src/cpp/'+dir, duplicate=0) # Use openssl @@ -118,19 +118,9 @@ if OSX: env.Append(CXXFLAGS = ['-I/usr/local/opt/openssl/include']) if LevelDB: - env.Append(CXXFLAGS = [ '-Isrc/cpp/leveldb', '-Isrc/cpp/leveldb/port', '-Isrc/cpp/leveldb/include', '-DUSE_LEVELDB', '-DLEVELDB_PLATFORM_POSIX']) + env.Append(CXXFLAGS = [ '-Isrc/cpp/leveldb', '-Isrc/cpp/leveldb/port', '-Isrc/cpp/leveldb/include']) - LEVELDB_PREFIX = 'src/cpp/leveldb' - PORTABLE_FILES = commands.getoutput('find ' - + LEVELDB_PREFIX + '/db ' - + LEVELDB_PREFIX + '/util ' - + LEVELDB_PREFIX + '/table ' - + ' -name *test*.cc -prune' - + ' -o -name *_bench.cc -prune' - + ' -o -name leveldb_main.cc -prune' - + ' -o -name "*.cc" -print | sort | tr "\n" " "').rstrip() - LEVELDB_SRCS = re.split(' ', PORTABLE_FILES) - LEVELDB_SRCS.append(LEVELDB_PREFIX + '/port/port_posix.cc') + LEVELDB_SRCS = [ 'src/cpp/leveldb_core.cpp' ] DB_SRCS = glob.glob('src/cpp/database/*.c') + glob.glob('src/cpp/database/*.cpp') JSON_SRCS = glob.glob('src/cpp/json/*.cpp') @@ -147,12 +137,11 @@ WEBSOCKETPP_SRCS = [ ] RIPPLE_SRCS = glob.glob('src/cpp/ripple/*.cpp') -PROTO_SRCS = [ 'src/cpp/protobuf_core.cpp' ] -# PROTO_SRCS = env.Protoc([], 'src/cpp/ripple/ripple.proto', PROTOCOUTDIR='build/proto', PROTOCPYTHONOUTDIR=None) -# env.Append(CXXFLAGS = ['-Ibuild/proto']) -env.Append(CXXFLAGS = ['-Ibuild/proto', '-Isrc/cpp/protobuf/src', '-Isrc/cpp/protobuf/vsprojects' ]) - -# env.Clean(PROTO_SRCS, 'site_scons/site_tools/protoc.pyc') +PROTO_SRCS = env.Protoc([], 'src/cpp/ripple/ripple.proto', PROTOCOUTDIR='build/proto', PROTOCPYTHONOUTDIR=None) +env.Append(CXXFLAGS = ['-Ibuild/proto']) +env.Clean(PROTO_SRCS, 'site_scons/site_tools/protoc.pyc') +# PROTO_SRCS = [ 'src/cpp/protobuf_core.cpp' ] +# env.Append(CXXFLAGS = ['-Ibuild/proto', '-Isrc/cpp/protobuf/src', '-Isrc/cpp/protobuf/vsprojects' ]) # Remove unused source files. UNUSED_SRCS = [] From 9c55b3b92a06da9920131fa3feedfc606c172770 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 20:26:12 -0700 Subject: [PATCH 10/15] Allow servers to explcitly vouch for correct signatures on transactions, proposals, and validations. This will allow cluster members to more safely distribute crypto work. --- src/cpp/ripple/ripple.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cpp/ripple/ripple.proto b/src/cpp/ripple/ripple.proto index 8e5def253..848802bdd 100644 --- a/src/cpp/ripple/ripple.proto +++ b/src/cpp/ripple/ripple.proto @@ -94,6 +94,7 @@ message TMTransaction { required bytes rawTransaction = 1; required TransactionStatus status = 2; optional uint64 receiveTimestamp = 3; + optional bool checkedSignature = 4; // no vouches for signature being correct } @@ -132,6 +133,7 @@ message TMProposeSet { required uint32 closeTime = 4; required bytes signature = 5; // signature of above fields optional bytes previousledger = 6; + optional bool checkedSignature = 7; // node vouches signature is correct repeated bytes addedTransactions = 10; // not required if number is large repeated bytes removedTransactions = 11; // not required if number is large } @@ -151,6 +153,7 @@ message TMHaveTransactionSet { // Used to sign a final closed ledger after reprocessing message TMValidation { required bytes validation = 1; // in SerializedValidation signed form + optional bool checkedSignature = 2; // node vouches signature is correct } From 379875ae475df8227042f25cbc6862d3d4942f99 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 16 May 2013 00:57:56 -0700 Subject: [PATCH 11/15] Hash is not valid if transaction is added to ledger. --- src/cpp/ripple/Ledger.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index 8bb7709e2..33344bcfd 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -256,6 +256,7 @@ bool Ledger::addTransaction(const uint256& txID, const Serializer& txn) cLog(lsWARNING) << "Attempt to add transaction to ledger that already had it"; return false; } + mValidHash = false; return true; } @@ -270,6 +271,7 @@ bool Ledger::addTransaction(const uint256& txID, const Serializer& txn, const Se cLog(lsFATAL) << "Attempt to add transaction+MD to ledger that already had it"; return false; } + mValidHash = false; return true; } From 6556719547a1f66523bb7a5547a45623079429b0 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 16 May 2013 00:58:20 -0700 Subject: [PATCH 12/15] LedgerMaster keeps cached snapshot of the current ledger. --- src/cpp/ripple/LedgerMaster.cpp | 8 ++++++++ src/cpp/ripple/LedgerMaster.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index dd93c393e..56a28a802 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -18,6 +18,14 @@ uint32 LedgerMaster::getCurrentLedgerIndex() return mCurrentLedger->getLedgerSeq(); } +Ledger::ref LedgerMaster::getCurrentSnapshot() +{ + if (!mCurrentSnapshot || (mCurrentSnapshot->getHash() != mCurrentLedger->getHash())) + mCurrentSnapshot = boost::make_shared(boost::ref(*mCurrentLedger), false); + assert(mCurrentSnapshot->isImmutable()); + return mCurrentSnapshot; +} + void LedgerMaster::addHeldTransaction(Transaction::ref transaction) { // returns true if transaction was added boost::recursive_mutex::scoped_lock ml(mLock); diff --git a/src/cpp/ripple/LedgerMaster.h b/src/cpp/ripple/LedgerMaster.h index 2d872bbdc..dddffaa42 100644 --- a/src/cpp/ripple/LedgerMaster.h +++ b/src/cpp/ripple/LedgerMaster.h @@ -26,6 +26,7 @@ protected: TransactionEngine mEngine; Ledger::pointer mCurrentLedger; // The ledger we are currently processiong + Ledger::pointer mCurrentSnapshot; // Snapshot of the current ledger Ledger::pointer mFinalizedLedger; // The ledger that most recently closed Ledger::pointer mValidLedger; // The highest-sequence ledger we have fully accepted Ledger::pointer mPubLedger; // The last ledger we have published @@ -74,6 +75,9 @@ public: // The current ledger is the ledger we believe new transactions should go in Ledger::ref getCurrentLedger() { return mCurrentLedger; } + // An immutable snapshot of the current ledger + Ledger::ref getCurrentSnapshot(); + // The finalized ledger is the last closed/accepted ledger Ledger::ref getClosedLedger() { return mFinalizedLedger; } From eecc9017b1f8de5cb5e693a5ca2647159f66bfcb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 16 May 2013 00:58:34 -0700 Subject: [PATCH 13/15] Wrapper to get current snapshot. --- src/cpp/ripple/NetworkOPs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cpp/ripple/NetworkOPs.h b/src/cpp/ripple/NetworkOPs.h index c18f8b55c..94c522d50 100644 --- a/src/cpp/ripple/NetworkOPs.h +++ b/src/cpp/ripple/NetworkOPs.h @@ -183,6 +183,7 @@ public: Ledger::ref getClosedLedger() { return mLedgerMaster->getClosedLedger(); } Ledger::ref getValidatedLedger() { return mLedgerMaster->getValidatedLedger(); } Ledger::ref getCurrentLedger() { return mLedgerMaster->getCurrentLedger(); } + Ledger::ref getCurrentSnapshot() { return mLedgerMaster->getCurrentSnapshot(); } Ledger::pointer getLedgerByHash(const uint256& hash) { return mLedgerMaster->getLedgerByHash(hash); } Ledger::pointer getLedgerBySeq(const uint32 seq); void missingNodeInLedger(const uint32 seq); From 28f54176f3a2e31d23e4a89db492866cbfcea330 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 16 May 2013 00:58:44 -0700 Subject: [PATCH 14/15] Use the current snapshot to get immutable ledger optimizations. --- src/cpp/ripple/RPCHandler.cpp | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index de444d4da..37ca82103 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -177,8 +177,7 @@ Json::Value RPCHandler::transactionSign(Json::Value jvRequest, bool bSubmit) return rpcError(rpcINVALID_PARAMS); } - Ledger::pointer lSnapshot = boost::make_shared( - boost::ref(*mNetOps->getCurrentLedger()), false); + Ledger::pointer lSnapshot = mNetOps->getCurrentSnapshot(); { ScopedUnlock su(theApp->getMasterLock()); bool bValid; @@ -1031,6 +1030,9 @@ Json::Value RPCHandler::doAccountLines(Json::Value jvRequest, int& cost, ScopedL if (!lpLedger) return jvResult; + if (lpLedger->isImmutable()) + MasterLockHolder.unlock(); + if (!jvRequest.isMember("account")) return rpcError(rpcINVALID_PARAMS); @@ -1065,9 +1067,6 @@ Json::Value RPCHandler::doAccountLines(Json::Value jvRequest, int& cost, ScopedL AccountState::pointer as = mNetOps->getAccountState(lpLedger, raAccount); - if (lpLedger->isImmutable()) - MasterLockHolder.unlock(); - if (as) { jvResult["account"] = raAccount.humanAccountID(); @@ -1124,6 +1123,9 @@ Json::Value RPCHandler::doAccountOffers(Json::Value jvRequest, int& cost, Scoped if (!lpLedger) return jvResult; + if (lpLedger->isImmutable()) + MasterLockHolder.unlock(); + if (!jvRequest.isMember("account")) return rpcError(rpcINVALID_PARAMS); @@ -1860,6 +1862,9 @@ Json::Value RPCHandler::doLedger(Json::Value jvRequest, int& cost, ScopedLock& M if (!lpLedger) return jvResult; + if (lpLedger->isImmutable()) + MasterLockHolder.unlock(); + bool bFull = jvRequest.isMember("full") && jvRequest["full"].asBool(); bool bTransactions = jvRequest.isMember("transactions") && jvRequest["transactions"].asBool(); bool bAccounts = jvRequest.isMember("accounts") && jvRequest["accounts"].asBool(); @@ -1869,15 +1874,6 @@ Json::Value RPCHandler::doLedger(Json::Value jvRequest, int& cost, ScopedLock& M | (bTransactions ? LEDGER_JSON_DUMP_TXRP : 0) | (bAccounts ? LEDGER_JSON_DUMP_STATE : 0); - if ((bFull || bAccounts) && !lpLedger->isImmutable()) - { // For full or accounts, it's cheaper to snapshot - lpLedger = boost::make_shared(*lpLedger, false); - assert(lpLedger->isImmutable()); - } - - if (lpLedger->isImmutable()) - MasterLockHolder.unlock(); - Json::Value ret(Json::objectValue); lpLedger->addJson(ret, iOptions); @@ -2620,7 +2616,7 @@ Json::Value RPCHandler::lookupLedger(Json::Value jvRequest, Ledger::pointer& lpL switch (iLedgerIndex) { case LEDGER_CURRENT: - lpLedger = mNetOps->getCurrentLedger(); + lpLedger = mNetOps->getCurrentSnapshot(); iLedgerIndex = lpLedger->getLedgerSeq(); break; From 44bc8c4e9fd3a3415375b099eded64df9f6568ff Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 16 May 2013 01:01:04 -0700 Subject: [PATCH 15/15] Code needs to know if LevelDB has been compiled in. --- SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 7ba8277f9..1985cb8e1 100644 --- a/SConstruct +++ b/SConstruct @@ -118,7 +118,7 @@ if OSX: env.Append(CXXFLAGS = ['-I/usr/local/opt/openssl/include']) if LevelDB: - env.Append(CXXFLAGS = [ '-Isrc/cpp/leveldb', '-Isrc/cpp/leveldb/port', '-Isrc/cpp/leveldb/include']) + env.Append(CXXFLAGS = [ '-Isrc/cpp/leveldb', '-Isrc/cpp/leveldb/port', '-Isrc/cpp/leveldb/include', '-DUSE_LEVELDB']) LEVELDB_SRCS = [ 'src/cpp/leveldb_core.cpp' ]