ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-20 03:35:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix recursion issue with fetchLedgerPage

Browse Source
This commit is contained in:
CJ Cobb
2021-10-05 18:30:11 -04:00
parent e67ea1afb6
commit 2cb74ae706
7 changed files with 37 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/backend/BackendInterface.cpp
View File

@@ -3,7 +3,7 @@
#include <backend/BackendInterface.h>
namespace Backend {
bool
BackendInterface::finishWrites(uint32_t ledgerSequence) const
BackendInterface::finishWrites(uint32_t ledgerSequence)
{
indexer_.finish(ledgerSequence, *this);
auto commitRes = doFinishWrites();
@@ -14,6 +14,7 @@ BackendInterface::finishWrites(uint32_t ledgerSequence) const
if (indexer_.isKeyFlagLedger(ledgerSequence))
indexer_.writeKeyFlagLedgerAsync(ledgerSequence, *this);
isFirst_ = false;
updateRange(ledgerSequence);
}
else
{
@@ -191,6 +192,7 @@ BackendInterface::fetchLedgerPage(
assert(limit != 0);
bool incomplete = !isLedgerIndexed(ledgerSequence);
BOOST_LOG_TRIVIAL(debug) << __func__ << " incomplete = " << incomplete;
std::cout << ledgerSequence << " - " << incomplete << std::endl;
// really low limits almost always miss
uint32_t adjustedLimit = std::max(limitHint, std::max(limit, (uint32_t)4));
LedgerPage page;
@@ -220,6 +222,7 @@ BackendInterface::fetchLedgerPage(
} while (page.objects.size() < limit && page.cursor);
if (incomplete)
{
std::cout << "recursing" << std::endl;
auto rng = fetchLedgerRange();
if (!rng)
return page;
@@ -247,6 +250,7 @@ BackendInterface::fetchLedgerPage(
std::move_iterator(lowerPage.objects.end()),
std::back_inserter(keys),
[](auto&& elt) { return std::move(elt.key); });
size_t upperPageSize = page.objects.size();
auto objs = fetchLedgerObjects(keys, ledgerSequence);
for (size_t i = 0; i < keys.size(); ++i)
{
@@ -258,7 +262,10 @@ BackendInterface::fetchLedgerPage(
std::sort(page.objects.begin(), page.objects.end(), [](auto a, auto b) {
return a.key < b.key;
});
page.warning = "Data may be incomplete";
if (page.objects.size() > limit)
page.objects.resize(limit);
if (page.objects.size() && page.objects.size() >= limit)
page.cursor = page.objects.back().key;
}
return page;
}

2
src/backend/BackendInterface.h
View File

@@ -233,7 +233,7 @@ protected:
// Tell the database we have finished writing all data for a particular
// ledger
bool
finishWrites(uint32_t ledgerSequence) const;
finishWrites(uint32_t ledgerSequence);
virtual bool
doOnlineDelete(uint32_t numLedgersToKeep) const = 0;

2
src/backend/CassandraBackend.cpp
View File

@@ -586,7 +586,7 @@ CassandraBackend::doFetchLedgerPage(
}
}
if (!cursor && (!keys.size() || !keys[0].isZero()))
if (!cursorIn && (!keys.size() || !keys[0].isZero()))
{
page.warning = "Data may be incomplete";
}

2
src/backend/PostgresBackend.cpp
View File

@@ -544,7 +544,7 @@ PostgresBackend::fetchAccountTransactions(
char const*& command = dbParams.first;
std::vector<std::optional<std::string>>& values = dbParams.second;
command =
"SELECT account_tx($1::bytea, $2::bigint, $3::bool"
"SELECT account_tx($1::bytea, $2::bigint, $3::bool, "
"$4::bigint, $5::bigint)";
values.resize(5);
values[0] = "\\x" + strHex(account);

5
src/rpc/RPCHelpers.cpp
View File

@@ -384,7 +384,7 @@ ledgerInfoFromRequest(Context const& ctx)
}
std::vector<unsigned char>
ledgerInfoToBlob(ripple::LedgerInfo const& info)
ledgerInfoToBlob(ripple::LedgerInfo const& info, bool includeHash)
{
ripple::Serializer s;
s.add32(info.seq);
@@ -396,7 +396,8 @@ ledgerInfoToBlob(ripple::LedgerInfo const& info)
s.add32(info.closeTime.time_since_epoch().count());
s.add8(info.closeTimeResolution.count());
s.add8(info.closeFlags);
// s.addBitString(info.hash);
if (includeHash)
s.addBitString(info.hash);
return s.peekData();
}

2
src/rpc/RPCHelpers.h
View File

@@ -84,7 +84,7 @@ std::vector<ripple::AccountID>
getAccountsFromTransaction(boost::json::object const& transaction);
std::vector<unsigned char>
ledgerInfoToBlob(ripple::LedgerInfo const& info);
ledgerInfoToBlob(ripple::LedgerInfo const& info, bool includeHash = false);
bool
isGlobalFrozen(

28
unittests/main.cpp
View File

@@ -40,13 +40,11 @@ TEST(BackendTest, Basic)
{"password", "postgres"},
{"indexer_key_shift", 2},
{"threads", 8}}}}}};
std::vector<boost::json::object> configs = {
cassandraConfig, postgresConfig};
std::vector<boost::json::object> configs = {cassandraConfig};
for (auto& config : configs)
{
std::cout << keyspace << std::endl;
auto backend = Backend::make_Backend(config);
backend->open(false);
std::string rawHeader =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
@@ -73,7 +71,7 @@ TEST(BackendTest, Basic)
return uint.fromVoid((void const*)bin.data());
};
auto ledgerInfoToBinaryString = [](auto const& info) {
auto blob = RPC::ledgerInfoToBlob(info);
auto blob = RPC::ledgerInfoToBlob(info, true);
std::string strBlob;
for (auto c : blob)
{
@@ -102,8 +100,10 @@ TEST(BackendTest, Basic)
}
{
std::cout << "fetching ledger by sequence" << std::endl;
auto retLgr = backend->fetchLedgerBySequence(lgrInfo.seq);
EXPECT_TRUE(retLgr.has_value());
std::cout << "fetched ledger by sequence" << std::endl;
ASSERT_TRUE(retLgr.has_value());
EXPECT_EQ(retLgr->seq, lgrInfo.seq);
EXPECT_EQ(
RPC::ledgerInfoToBlob(lgrInfo), RPC::ledgerInfoToBlob(*retLgr));
@@ -136,7 +136,9 @@ TEST(BackendTest, Basic)
EXPECT_EQ(seq, lgrInfoNext.seq);
}
{
std::cout << "fetching ledger by sequence" << std::endl;
auto retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq);
std::cout << "fetched ledger by sequence" << std::endl;
EXPECT_TRUE(retLgr.has_value());
EXPECT_EQ(retLgr->seq, lgrInfoNext.seq);
EXPECT_EQ(
@@ -570,13 +572,23 @@ TEST(BackendTest, Basic)
{
uint32_t limit = 10;
page = backend->fetchLedgerPage(page.cursor, seq, limit);
if (page.cursor)
EXPECT_EQ(page.objects.size(), limit);
std::cout << "fetched a page " << page.objects.size()
<< std::endl;
// if (page.cursor)
// EXPECT_EQ(page.objects.size(), limit);
retObjs.insert(
retObjs.end(), page.objects.begin(), page.objects.end());
++numLoops;
ASSERT_FALSE(page.warning.has_value());
} while (page.cursor);
std::cout << numLoops << std::endl;
std::cout << "base" << std::endl;
for (auto obj : objs)
if (obj.second.size() != 0)
std::cout << ripple::strHex(obj.first) << std::endl;
std::cout << "clio" << std::endl;
for (auto retObj : retObjs)
std::cout << ripple::strHex(retObj.key) << std::endl;
for (auto obj : objs)
{
bool found = false;
@@ -591,6 +603,8 @@ TEST(BackendTest, Basic)
ripple::strHex(retObj.blob));
}
}
if (found != (obj.second.size() != 0))
std::cout << ripple::strHex(obj.first) << std::endl;
ASSERT_EQ(found, obj.second.size() != 0);
}
};