#include #include namespace Backend { PostgresBackend::PostgresBackend(boost::json::object const& config) : pgPool_(make_PgPool(config)) { } void PostgresBackend::writeLedger( ripple::LedgerInfo const& ledgerInfo, std::string&& ledgerHeader, bool isFirst) const { PgQuery pgQuery(pgPool_); BOOST_LOG_TRIVIAL(debug) << __func__; auto cmd = boost::format( R"(INSERT INTO ledgers VALUES (%u,'\x%s', '\x%s',%u,%u,%u,%u,%u,'\x%s','\x%s'))"); auto ledgerInsert = boost::str( cmd % ledgerInfo.seq % ripple::strHex(ledgerInfo.hash) % ripple::strHex(ledgerInfo.parentHash) % ledgerInfo.drops.drops() % ledgerInfo.closeTime.time_since_epoch().count() % ledgerInfo.parentCloseTime.time_since_epoch().count() % ledgerInfo.closeTimeResolution.count() % ledgerInfo.closeFlags % ripple::strHex(ledgerInfo.accountHash) % ripple::strHex(ledgerInfo.txHash)); BOOST_LOG_TRIVIAL(info) << __func__ << " : " << " : " << "query string = " << ledgerInsert; auto res = pgQuery(ledgerInsert.data()); abortWrite_ = !res; } void PostgresBackend::writeAccountTransactions( std::vector&& data) const { if (abortWrite_) return; PgQuery pg(pgPool_); for (auto const& record : data) { for (auto const& a : record.accounts) { std::string acct = ripple::strHex(a); accountTxBuffer_ << "\\\\x" << acct << '\t' << std::to_string(record.ledgerSequence) << '\t' << std::to_string(record.transactionIndex) << '\t' << "\\\\x" << ripple::strHex(record.txHash) << '\n'; } } } void PostgresBackend::writeLedgerObject( std::string&& key, uint32_t seq, std::string&& blob, bool isCreated, bool isDeleted, std::optional&& book) const { if (abortWrite_) return; static int numRows = 0; numRows++; objectsBuffer_ << "\\\\x" << ripple::strHex(key) << '\t' << std::to_string(seq) << '\t' << "\\\\x" << ripple::strHex(blob) << '\n'; // If the buffer gets too large, the insert fails. Not sure why. So we // insert after 1 million records if (numRows % 1000000 == 0) { PgQuery pgQuery(pgPool_); pgQuery.bulkInsert("objects", objectsBuffer_.str()); objectsBuffer_ = {}; } if (book) { booksBuffer_ << "\\\\x" << ripple::strHex(*book) << '\t' << std::to_string(seq) << '\t' << isDeleted << '\t' << "\\\\x" << ripple::strHex(key) << '\n'; } } void PostgresBackend::writeTransaction( std::string&& hash, uint32_t seq, std::string&& transaction, std::string&& metadata) const { if (abortWrite_) return; transactionsBuffer_ << "\\\\x" << ripple::strHex(hash) << '\t' << std::to_string(seq) << '\t' << "\\\\x" << ripple::strHex(transaction) << '\t' << "\\\\x" << ripple::strHex(metadata) << '\n'; } uint32_t checkResult(PgResult const& res, uint32_t numFieldsExpected) { if (!res) { assert(false); throw std::runtime_error("null postgres response"); } else if (res.status() != PGRES_TUPLES_OK) { std::stringstream msg; msg << " : Postgres response should have been " "PGRES_TUPLES_OK but instead was " << res.status() << " - msg = " << res.msg(); assert(false); throw std::runtime_error(msg.str()); } BOOST_LOG_TRIVIAL(trace) << __func__ << " Postgres result msg : " << res.msg(); if (res.isNull() || res.ntuples() == 0) { return 0; } else if (res.ntuples() > 0) { if (res.nfields() != numFieldsExpected) { std::stringstream msg; msg << "Wrong number of fields in Postgres " "response. Expected " << numFieldsExpected << ", but got " << res.nfields(); throw std::runtime_error(msg.str()); assert(false); } } return res.ntuples(); } ripple::LedgerInfo parseLedgerInfo(PgResult const& res) { std::int64_t ledgerSeq = res.asBigInt(0, 0); ripple::uint256 hash = res.asUInt256(0, 1); ripple::uint256 prevHash = res.asUInt256(0, 2); std::int64_t totalCoins = res.asBigInt(0, 3); std::int64_t closeTime = res.asBigInt(0, 4); std::int64_t parentCloseTime = res.asBigInt(0, 5); std::int64_t closeTimeRes = res.asBigInt(0, 6); std::int64_t closeFlags = res.asBigInt(0, 7); ripple::uint256 accountHash = res.asUInt256(0, 8); ripple::uint256 txHash = res.asUInt256(0, 9); using time_point = ripple::NetClock::time_point; using duration = ripple::NetClock::duration; ripple::LedgerInfo info; info.seq = ledgerSeq; info.hash = hash; info.parentHash = prevHash; info.drops = totalCoins; info.closeTime = time_point{duration{closeTime}}; info.parentCloseTime = time_point{duration{parentCloseTime}}; info.closeFlags = closeFlags; info.closeTimeResolution = duration{closeTimeRes}; info.accountHash = accountHash; info.txHash = txHash; info.validated = true; return info; } std::optional PostgresBackend::fetchLatestLedgerSequence() const { PgQuery pgQuery(pgPool_); auto res = pgQuery( "SELECT ledger_seq FROM ledgers ORDER BY ledger_seq DESC LIMIT 1"); if (checkResult(res, 1)) return res.asBigInt(0, 0); return {}; } std::optional PostgresBackend::fetchLedgerBySequence(uint32_t sequence) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT * FROM ledgers WHERE ledger_seq = " << std::to_string(sequence); auto res = pgQuery(sql.str().data()); if (checkResult(res, 10)) return parseLedgerInfo(res); return {}; } std::optional PostgresBackend::fetchLedgerRange() const { auto range = PgQuery(pgPool_)("SELECT complete_ledgers()"); if (!range) return {}; std::string res{range.c_str()}; try { size_t minVal = 0; size_t maxVal = 0; if (res == "empty" || res == "error" || res.empty()) return {}; else if (size_t delim = res.find('-'); delim != std::string::npos) { minVal = std::stol(res.substr(0, delim)); maxVal = std::stol(res.substr(delim + 1)); } else { minVal = maxVal = std::stol(res); } return LedgerRange{minVal, maxVal}; } catch (std::exception&) { BOOST_LOG_TRIVIAL(error) << __func__ << " : " << "Error parsing result of getCompleteLedgers()"; } return {}; } std::optional PostgresBackend::fetchLedgerObject( ripple::uint256 const& key, uint32_t sequence) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT object FROM objects WHERE key = " << "\'\\x" << ripple::strHex(key) << "\'" << " AND ledger_seq <= " << std::to_string(sequence) << " ORDER BY ledger_seq DESC LIMIT 1"; auto res = pgQuery(sql.str().data()); if (checkResult(res, 1)) { return res.asUnHexedBlob(0, 0); } return {}; } // returns a transaction, metadata pair std::optional PostgresBackend::fetchTransaction(ripple::uint256 const& hash) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT transaction,metadata,ledger_seq FROM transactions " "WHERE hash = " << "\'\\x" << ripple::strHex(hash) << "\'"; auto res = pgQuery(sql.str().data()); if (checkResult(res, 3)) { return {{res.asUnHexedBlob(0, 0), res.asUnHexedBlob(0, 1)}}; } return {}; } std::vector PostgresBackend::fetchAllTransactionsInLedger(uint32_t ledgerSequence) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT transaction, metadata, ledger_seq FROM transactions WHERE " << "ledger_seq = " << std::to_string(ledgerSequence); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 3)) { std::vector txns; for (size_t i = 0; i < numRows; ++i) { txns.push_back({res.asUnHexedBlob(i, 0), res.asUnHexedBlob(i, 1)}); } return txns; } return {}; } LedgerPage PostgresBackend::fetchLedgerPage( std::optional const& cursor, std::uint32_t ledgerSequence, std::uint32_t limit) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT key,object FROM" << " (SELECT DISTINCT ON (key) * FROM objects" << " WHERE ledger_seq <= " << std::to_string(ledgerSequence); if (cursor) sql << " AND key > \'x\\" << ripple::strHex(*cursor) << "\'"; sql << " ORDER BY key, ledger_seq DESC) sub" << " WHERE object != \'\\x\'" << " LIMIT " << std::to_string(limit); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 2)) { std::vector objects; for (size_t i = 0; i < numRows; ++i) { objects.push_back({res.asUInt256(i, 0), res.asUnHexedBlob(i, 1)}); } if (numRows == limit) return {objects, objects[objects.size() - 1].key}; else return {objects, {}}; } return {}; } std::pair, std::optional> PostgresBackend::fetchBookOffers( ripple::uint256 const& book, uint32_t ledgerSequence, std::uint32_t limit, std::optional const& cursor) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT offer_key FROM" << " (SELECT DISTINCT ON (offer_key) * FROM books WHERE book = " << "\'\\x" << ripple::strHex(book) << "\' AND ledger_seq <= " << std::to_string(ledgerSequence); if (cursor) sql << " AND offer_key > \'" << ripple::strHex(*cursor) << "\'"; sql << " ORDER BY offer_key DESC, ledger_seq DESC)" << " sub WHERE NOT deleted" << " LIMIT " << std::to_string(limit); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 1)) { std::vector keys; for (size_t i = 0; i < numRows; ++i) { keys.push_back(res.asUInt256(i, 0)); } std::vector blobs = fetchLedgerObjects(keys, ledgerSequence); std::vector results; std::transform( blobs.begin(), blobs.end(), keys.begin(), std::back_inserter(results), [](auto& blob, auto& key) { return LedgerObject{std::move(key), std::move(blob)}; }); return {results, results[results.size() - 1].key}; } return {{}, {}}; } std::vector PostgresBackend::fetchTransactions( std::vector const& hashes) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT transaction,metadata,ledger_seq FROM transactions " "WHERE "; bool first = true; for (auto const& hash : hashes) { if (!first) sql << " OR "; sql << "HASH = \'\\x" << ripple::strHex(hash) << "\'"; first = false; } auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 3)) { std::vector results; for (size_t i = 0; i < numRows; ++i) { results.push_back( {res.asUnHexedBlob(i, 0), res.asUnHexedBlob(i, 1)}); } return results; } return {}; } std::vector PostgresBackend::fetchLedgerObjects( std::vector const& keys, uint32_t sequence) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT DISTINCT ON(key) object FROM objects WHERE"; bool first = true; for (auto const& key : keys) { if (!first) { sql << " OR "; } else { sql << " ( "; first = false; } sql << " key = " << "\'\\x" << ripple::strHex(key) << "\'"; } sql << " ) " << " AND ledger_seq <= " << std::to_string(sequence) << " ORDER BY key, ledger_seq DESC"; BOOST_LOG_TRIVIAL(info) << sql.str(); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 1)) { std::vector results; for (size_t i = 0; i < numRows; ++i) { results.push_back(res.asUnHexedBlob(i, 0)); } return results; } return {}; } std::pair< std::vector, std::optional> PostgresBackend::fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t limit, std::optional const& cursor) const { PgQuery pgQuery(pgPool_); std::stringstream sql; sql << "SELECT hash, ledger_seq, transaction_index FROM " "account_transactions WHERE account = " << "\'\\x" << ripple::strHex(account) << "\'"; if (cursor) sql << " AND ledger_seq < " << cursor->ledgerSequence << " AND transaction_index < " << cursor->transactionIndex; sql << " LIMIT " << std::to_string(limit); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 3)) { std::vector hashes; for (size_t i = 0; i < numRows; ++i) { hashes.push_back(res.asUInt256(i, 0)); } if (numRows == limit) { AccountTransactionsCursor retCursor{ res.asBigInt(numRows - 1, 1), res.asBigInt(numRows - 1, 2)}; return {fetchTransactions(hashes), {retCursor}}; } else { return {fetchTransactions(hashes), {}}; } } return {}; } void PostgresBackend::open() { initSchema(pgPool_); } void PostgresBackend::close() { } void PostgresBackend::startWrites() const { PgQuery pg(pgPool_); auto res = pg("BEGIN"); if (!res || res.status() != PGRES_COMMAND_OK) { std::stringstream msg; msg << "Postgres error creating transaction: " << res.msg(); throw std::runtime_error(msg.str()); } } bool PostgresBackend::finishWrites() const { if (abortWrite_) return false; PgQuery pg(pgPool_); pg.bulkInsert("transactions", transactionsBuffer_.str()); pg.bulkInsert("books", booksBuffer_.str()); pg.bulkInsert("account_transactions", accountTxBuffer_.str()); std::string objectsStr = objectsBuffer_.str(); if (objectsStr.size()) pg.bulkInsert("objects", objectsStr); auto res = pg("COMMIT"); if (!res || res.status() != PGRES_COMMAND_OK) { std::stringstream msg; msg << "Postgres error committing transaction: " << res.msg(); throw std::runtime_error(msg.str()); } transactionsBuffer_.str(""); transactionsBuffer_.clear(); objectsBuffer_.str(""); objectsBuffer_.clear(); booksBuffer_.str(""); booksBuffer_.clear(); accountTxBuffer_.str(""); accountTxBuffer_.clear(); return true; } } // namespace Backend