ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-22 04:35:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

online delete with postgres. seems to work

Browse Source
This commit is contained in:
CJ Cobb
2021-03-23 15:55:09 -04:00
parent c6a42fa9f2
commit 564d46cb5f
8 changed files with 147 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
reporting/PostgresBackend.cpp
View File

@@ -583,5 +583,103 @@ PostgresBackend::finishWrites() const
numRowsInObjectsBuffer_ = 0;
return true;
}
bool
PostgresBackend::doOnlineDelete(uint32_t minLedgerToKeep) const
{
uint32_t limit = 2048;
PgQuery pgQuery(pgPool_);
{
std::stringstream sql;
sql << "DELETE FROM ledgers WHERE ledger_seq < "
<< std::to_string(minLedgerToKeep);
auto res = pgQuery(sql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from ledgers table");
}
std::string cursor;
do
{
std::stringstream sql;
sql << "SELECT DISTINCT ON (key) key,ledger_seq,object FROM objects"
<< " WHERE ledger_seq <= " << std::to_string(minLedgerToKeep);
if (cursor.size())
sql << " AND key < \'\\x" << cursor << "\'";
sql << " ORDER BY key DESC, ledger_seq DESC"
<< " LIMIT " << std::to_string(limit);
BOOST_LOG_TRIVIAL(trace) << __func__ << sql.str();
auto res = pgQuery(sql.str().data());
BOOST_LOG_TRIVIAL(debug) << __func__ << "Fetched a page";
if (size_t numRows = checkResult(res, 3))
{
std::stringstream deleteSql;
std::stringstream deleteOffersSql;
deleteSql << "DELETE FROM objects WHERE (";
deleteOffersSql << "DELETE FROM books WHERE (";
bool firstOffer = true;
for (size_t i = 0; i < numRows; ++i)
{
std::string_view keyView{res.c_str(i, 0) + 2};
int64_t sequence = res.asBigInt(i, 1);
std::string_view objView{res.c_str(i, 2) + 2};
if (i != 0)
deleteSql << " OR ";
deleteSql << "(key = "
<< "\'\\x" << keyView << "\'";
if (objView.size() == 0)
deleteSql << " AND ledger_seq <= "
<< std::to_string(sequence);
else
deleteSql << " AND ledger_seq < "
<< std::to_string(sequence);
deleteSql << ")";
bool deleteOffer = false;
if (objView.size())
{
deleteOffer = isOfferHex(objView);
}
else
{
// This is rather unelegant. For a deleted object, we don't
// know its type just from the key (or do we?). So, we just
// assume it is an offer and try to delete it. The
// alternative is to read the actual object out of the db
// from before it was deleted. This could result in a lot of
// individual reads though, so we chose to just delete
deleteOffer = true;
}
if (deleteOffer)
{
if (!firstOffer)
deleteOffersSql << " OR ";
deleteOffersSql << "( offer_key = "
<< "\'\\x" << keyView << "\')";
firstOffer = false;
}
}
if (numRows == limit)
cursor = res.c_str(numRows - 1, 0) + 2;
else
cursor = {};
deleteSql << ")";
deleteOffersSql << ")";
BOOST_LOG_TRIVIAL(trace) << __func__ << deleteSql.str();
res = pgQuery(deleteSql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from objects table");
if (!firstOffer)
{
BOOST_LOG_TRIVIAL(trace) << __func__ << deleteOffersSql.str();
res = pgQuery(deleteOffersSql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from books table");
}
BOOST_LOG_TRIVIAL(debug)
<< __func__ << "Deleted a page. Cursor = " << cursor;
}
} while (cursor.size());
return true;
}
} // namespace Backend