Improve AccountID string conversion caching:

Caching the base58check encoded version of an `AccountID` has
performance advantages, because because of the computationally
heavy cost associated with the conversion, which requires the
application of SHA-256 twice.

This commit makes the cache significantly more efficient in terms
of memory used: it eliminates the map, using a vector with a size
that is determined by the configured size of the node, and a hash
function to directly map any given `AccountID` to a specific slot
in the cache; the eviction policy is simple: in case of collision
the existing entry is removed and replaced with the new data.

Previously, use of the cache was optional and required additional
effort by the programmer. Now the cache is automatic and does not
require any additional work or information.

The new cache also utilizes a 64-way spinlock, to help reduce any
contention that the pressure on the cache would impose.
This commit is contained in:
Nik Bougalis
2022-05-25 11:22:47 -07:00
parent 5a15229eeb
commit e2eed966b0
21 changed files with 157 additions and 196 deletions

View File

@@ -31,11 +31,9 @@ AcceptedLedger::AcceptedLedger(
transactions_.reserve(256); transactions_.reserve(256);
auto insertAll = [&](auto const& txns) { auto insertAll = [&](auto const& txns) {
auto const& idcache = app.accountIDCache();
for (auto const& item : txns) for (auto const& item : txns)
transactions_.emplace_back(std::make_unique<AcceptedLedgerTx>( transactions_.emplace_back(std::make_unique<AcceptedLedgerTx>(
ledger, item.first, item.second, idcache)); ledger, item.first, item.second));
}; };
if (app.config().reporting()) if (app.config().reporting())

View File

@@ -28,8 +28,7 @@ namespace ripple {
AcceptedLedgerTx::AcceptedLedgerTx( AcceptedLedgerTx::AcceptedLedgerTx(
std::shared_ptr<ReadView const> const& ledger, std::shared_ptr<ReadView const> const& ledger,
std::shared_ptr<STTx const> const& txn, std::shared_ptr<STTx const> const& txn,
std::shared_ptr<STObject const> const& met, std::shared_ptr<STObject const> const& met)
AccountIDCache const& accountCache)
: mTxn(txn) : mTxn(txn)
, mMeta(txn->getTransactionID(), ledger->seq(), *met) , mMeta(txn->getTransactionID(), ledger->seq(), *met)
, mAffected(mMeta.getAffectedAccounts()) , mAffected(mMeta.getAffectedAccounts())
@@ -52,7 +51,7 @@ AcceptedLedgerTx::AcceptedLedgerTx(
{ {
Json::Value& affected = (mJson[jss::affected] = Json::arrayValue); Json::Value& affected = (mJson[jss::affected] = Json::arrayValue);
for (auto const& account : mAffected) for (auto const& account : mAffected)
affected.append(accountCache.toBase58(account)); affected.append(toBase58(account));
} }
if (mTxn->getTxnType() == ttOFFER_CREATE) if (mTxn->getTxnType() == ttOFFER_CREATE)

View File

@@ -46,8 +46,7 @@ public:
AcceptedLedgerTx( AcceptedLedgerTx(
std::shared_ptr<ReadView const> const& ledger, std::shared_ptr<ReadView const> const& ledger,
std::shared_ptr<STTx const> const&, std::shared_ptr<STTx const> const&,
std::shared_ptr<STObject const> const&, std::shared_ptr<STObject const> const&);
AccountIDCache const&);
std::shared_ptr<STTx const> const& std::shared_ptr<STTx const> const&
getTxn() const getTxn() const

View File

@@ -181,7 +181,6 @@ public:
NodeStoreScheduler m_nodeStoreScheduler; NodeStoreScheduler m_nodeStoreScheduler;
std::unique_ptr<SHAMapStore> m_shaMapStore; std::unique_ptr<SHAMapStore> m_shaMapStore;
PendingSaves pendingSaves_; PendingSaves pendingSaves_;
AccountIDCache accountIDCache_;
std::optional<OpenLedger> openLedger_; std::optional<OpenLedger> openLedger_;
NodeCache m_tempNodeCache; NodeCache m_tempNodeCache;
@@ -336,8 +335,6 @@ public:
m_nodeStoreScheduler, m_nodeStoreScheduler,
logs_->journal("SHAMapStore"))) logs_->journal("SHAMapStore")))
, accountIDCache_(128000)
, m_tempNodeCache( , m_tempNodeCache(
"NodeCache", "NodeCache",
16384, 16384,
@@ -494,6 +491,8 @@ public:
config_->reporting() ? std::make_unique<ReportingETL>(*this) config_->reporting() ? std::make_unique<ReportingETL>(*this)
: nullptr) : nullptr)
{ {
initAccountIdCache(config_->getValueFor(SizedItem::accountIdCacheSize));
add(m_resourceManager.get()); add(m_resourceManager.get());
// //
@@ -856,12 +855,6 @@ public:
return pendingSaves_; return pendingSaves_;
} }
AccountIDCache const&
accountIDCache() const override
{
return accountIDCache_;
}
OpenLedger& OpenLedger&
openLedger() override openLedger() override
{ {

View File

@@ -90,7 +90,6 @@ class PathRequests;
class PendingSaves; class PendingSaves;
class PublicKey; class PublicKey;
class SecretKey; class SecretKey;
class AccountIDCache;
class STLedgerEntry; class STLedgerEntry;
class TimeKeeper; class TimeKeeper;
class TransactionMaster; class TransactionMaster;
@@ -251,8 +250,6 @@ public:
getSHAMapStore() = 0; getSHAMapStore() = 0;
virtual PendingSaves& virtual PendingSaves&
pendingSaves() = 0; pendingSaves() = 0;
virtual AccountIDCache const&
accountIDCache() const = 0;
virtual OpenLedger& virtual OpenLedger&
openLedger() = 0; openLedger() = 0;
virtual OpenLedger const& virtual OpenLedger const&

View File

@@ -552,9 +552,16 @@ PathRequest::findPaths(
continueCallback); continueCallback);
mContext[issue] = ps; mContext[issue] = ps;
auto& sourceAccount = !isXRP(issue.account) auto const& sourceAccount = [&] {
? issue.account if (!isXRP(issue.account))
: isXRP(issue.currency) ? xrpAccount() : *raSrcAccount; return issue.account;
if (isXRP(issue.currency))
return xrpAccount();
return *raSrcAccount;
}();
STAmount saMaxAmount = saSendMax.value_or( STAmount saMaxAmount = saSendMax.value_or(
STAmount({issue.currency, sourceAccount}, 1u, 0, true)); STAmount({issue.currency, sourceAccount}, 1u, 0, true));
@@ -675,10 +682,8 @@ PathRequest::doUpdate(
destCurrencies.append(to_string(c)); destCurrencies.append(to_string(c));
} }
newStatus[jss::source_account] = newStatus[jss::source_account] = toBase58(*raSrcAccount);
app_.accountIDCache().toBase58(*raSrcAccount); newStatus[jss::destination_account] = toBase58(*raDstAccount);
newStatus[jss::destination_account] =
app_.accountIDCache().toBase58(*raDstAccount);
newStatus[jss::destination_amount] = saDstAmount.getJson(JsonOptions::none); newStatus[jss::destination_amount] = saDstAmount.getJson(JsonOptions::none);
newStatus[jss::full_reply] = !fast; newStatus[jss::full_reply] = !fast;

View File

@@ -389,7 +389,6 @@ getNewestAccountTxsB(
* account which match given criteria starting from given marker * account which match given criteria starting from given marker
* and calls callback for each found transaction. * and calls callback for each found transaction.
* @param session Session with database. * @param session Session with database.
* @param idCache Account ID cache.
* @param onUnsavedLedger Callback function to call on each found unsaved * @param onUnsavedLedger Callback function to call on each found unsaved
* ledger within given range. * ledger within given range.
* @param onTransaction Callback function to call on each found transaction. * @param onTransaction Callback function to call on each found transaction.
@@ -408,7 +407,6 @@ getNewestAccountTxsB(
std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int> std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int>
oldestAccountTxPage( oldestAccountTxPage(
soci::session& session, soci::session& session,
AccountIDCache const& idCache,
std::function<void(std::uint32_t)> const& onUnsavedLedger, std::function<void(std::uint32_t)> const& onUnsavedLedger,
std::function< std::function<
void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const&
@@ -422,7 +420,6 @@ oldestAccountTxPage(
* account which match given criteria starting from given marker * account which match given criteria starting from given marker
* and calls callback for each found transaction. * and calls callback for each found transaction.
* @param session Session with database. * @param session Session with database.
* @param idCache Account ID cache.
* @param onUnsavedLedger Callback function to call on each found unsaved * @param onUnsavedLedger Callback function to call on each found unsaved
* ledger within given range. * ledger within given range.
* @param onTransaction Callback function to call on each found transaction. * @param onTransaction Callback function to call on each found transaction.
@@ -441,7 +438,6 @@ oldestAccountTxPage(
std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int> std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int>
newestAccountTxPage( newestAccountTxPage(
soci::session& session, soci::session& session,
AccountIDCache const& idCache,
std::function<void(std::uint32_t)> const& onUnsavedLedger, std::function<void(std::uint32_t)> const& onUnsavedLedger,
std::function< std::function<
void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const&

View File

@@ -307,7 +307,7 @@ saveValidatedLedger(
sql += txnId; sql += txnId;
sql += "','"; sql += "','";
sql += app.accountIDCache().toBase58(account); sql += toBase58(account);
sql += "',"; sql += "',";
sql += ledgerSeq; sql += ledgerSeq;
sql += ","; sql += ",";
@@ -760,8 +760,7 @@ transactionsSQL(
sql = boost::str( sql = boost::str(
boost::format("SELECT %s FROM AccountTransactions " boost::format("SELECT %s FROM AccountTransactions "
"WHERE Account = '%s' %s %s LIMIT %u, %u;") % "WHERE Account = '%s' %s %s LIMIT %u, %u;") %
selection % app.accountIDCache().toBase58(options.account) % selection % toBase58(options.account) % maxClause % minClause %
maxClause % minClause %
beast::lexicalCastThrow<std::string>(options.offset) % beast::lexicalCastThrow<std::string>(options.offset) %
beast::lexicalCastThrow<std::string>(numberOfResults)); beast::lexicalCastThrow<std::string>(numberOfResults));
else else
@@ -774,9 +773,9 @@ transactionsSQL(
"ORDER BY AccountTransactions.LedgerSeq %s, " "ORDER BY AccountTransactions.LedgerSeq %s, "
"AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s " "AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s "
"LIMIT %u, %u;") % "LIMIT %u, %u;") %
selection % app.accountIDCache().toBase58(options.account) % selection % toBase58(options.account) % maxClause % minClause %
maxClause % minClause % (descending ? "DESC" : "ASC") %
(descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") %
(descending ? "DESC" : "ASC") %
beast::lexicalCastThrow<std::string>(options.offset) % beast::lexicalCastThrow<std::string>(options.offset) %
beast::lexicalCastThrow<std::string>(numberOfResults)); beast::lexicalCastThrow<std::string>(numberOfResults));
JLOG(j.trace()) << "txSQL query: " << sql; JLOG(j.trace()) << "txSQL query: " << sql;
@@ -1049,7 +1048,6 @@ getNewestAccountTxsB(
* account that matches the given criteria starting from the provided * account that matches the given criteria starting from the provided
* marker and invokes the callback parameter for each found transaction. * marker and invokes the callback parameter for each found transaction.
* @param session Session with the database. * @param session Session with the database.
* @param idCache Account ID cache.
* @param onUnsavedLedger Callback function to call on each found unsaved * @param onUnsavedLedger Callback function to call on each found unsaved
* ledger within the given range. * ledger within the given range.
* @param onTransaction Callback function to call on each found transaction. * @param onTransaction Callback function to call on each found transaction.
@@ -1069,7 +1067,6 @@ getNewestAccountTxsB(
static std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int> static std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int>
accountTxPage( accountTxPage(
soci::session& session, soci::session& session,
AccountIDCache const& idCache,
std::function<void(std::uint32_t)> const& onUnsavedLedger, std::function<void(std::uint32_t)> const& onUnsavedLedger,
std::function< std::function<
void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const&
@@ -1135,8 +1132,8 @@ accountTxPage(
ORDER BY AccountTransactions.LedgerSeq %s, ORDER BY AccountTransactions.LedgerSeq %s,
AccountTransactions.TxnSeq %s AccountTransactions.TxnSeq %s
LIMIT %u;)")) % LIMIT %u;)")) %
idCache.toBase58(options.account) % options.minLedger % toBase58(options.account) % options.minLedger % options.maxLedger %
options.maxLedger % order % order % queryLimit); order % order % queryLimit);
} }
else else
{ {
@@ -1146,7 +1143,7 @@ accountTxPage(
const std::uint32_t maxLedger = const std::uint32_t maxLedger =
forward ? options.maxLedger : findLedger - 1; forward ? options.maxLedger : findLedger - 1;
auto b58acct = idCache.toBase58(options.account); auto b58acct = toBase58(options.account);
sql = boost::str( sql = boost::str(
boost::format(( boost::format((
R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq, R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,
@@ -1250,7 +1247,6 @@ accountTxPage(
std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int> std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int>
oldestAccountTxPage( oldestAccountTxPage(
soci::session& session, soci::session& session,
AccountIDCache const& idCache,
std::function<void(std::uint32_t)> const& onUnsavedLedger, std::function<void(std::uint32_t)> const& onUnsavedLedger,
std::function< std::function<
void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const&
@@ -1261,7 +1257,6 @@ oldestAccountTxPage(
{ {
return accountTxPage( return accountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
options, options,
@@ -1273,7 +1268,6 @@ oldestAccountTxPage(
std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int> std::pair<std::optional<RelationalDatabase::AccountTxMarker>, int>
newestAccountTxPage( newestAccountTxPage(
soci::session& session, soci::session& session,
AccountIDCache const& idCache,
std::function<void(std::uint32_t)> const& onUnsavedLedger, std::function<void(std::uint32_t)> const& onUnsavedLedger,
std::function< std::function<
void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const&
@@ -1284,7 +1278,6 @@ newestAccountTxPage(
{ {
return accountTxPage( return accountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
options, options,

View File

@@ -1322,7 +1322,6 @@ SQLiteDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options)
return {}; return {};
static std::uint32_t const page_length(200); static std::uint32_t const page_length(200);
auto& idCache = app_.accountIDCache();
auto onUnsavedLedger = auto onUnsavedLedger =
std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1);
AccountTxs ret; AccountTxs ret;
@@ -1338,15 +1337,10 @@ SQLiteDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options)
if (existsTransaction()) if (existsTransaction())
{ {
auto db = checkoutTransaction(); auto db = checkoutTransaction();
auto newmarker = detail::oldestAccountTxPage( auto newmarker =
*db, detail::oldestAccountTxPage(
idCache, *db, onUnsavedLedger, onTransaction, options, 0, page_length)
onUnsavedLedger, .first;
onTransaction,
options,
0,
page_length)
.first;
return {ret, newmarker}; return {ret, newmarker};
} }
@@ -1363,7 +1357,6 @@ SQLiteDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options)
return false; return false;
auto [marker, total] = detail::oldestAccountTxPage( auto [marker, total] = detail::oldestAccountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
opt, opt,
@@ -1391,7 +1384,6 @@ SQLiteDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options)
return {}; return {};
static std::uint32_t const page_length(200); static std::uint32_t const page_length(200);
auto& idCache = app_.accountIDCache();
auto onUnsavedLedger = auto onUnsavedLedger =
std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1);
AccountTxs ret; AccountTxs ret;
@@ -1407,15 +1399,10 @@ SQLiteDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options)
if (existsTransaction()) if (existsTransaction())
{ {
auto db = checkoutTransaction(); auto db = checkoutTransaction();
auto newmarker = detail::newestAccountTxPage( auto newmarker =
*db, detail::newestAccountTxPage(
idCache, *db, onUnsavedLedger, onTransaction, options, 0, page_length)
onUnsavedLedger, .first;
onTransaction,
options,
0,
page_length)
.first;
return {ret, newmarker}; return {ret, newmarker};
} }
@@ -1432,7 +1419,6 @@ SQLiteDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options)
return false; return false;
auto [marker, total] = detail::newestAccountTxPage( auto [marker, total] = detail::newestAccountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
opt, opt,
@@ -1460,7 +1446,6 @@ SQLiteDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options)
return {}; return {};
static std::uint32_t const page_length(500); static std::uint32_t const page_length(500);
auto& idCache = app_.accountIDCache();
auto onUnsavedLedger = auto onUnsavedLedger =
std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1);
MetaTxsList ret; MetaTxsList ret;
@@ -1475,15 +1460,10 @@ SQLiteDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options)
if (existsTransaction()) if (existsTransaction())
{ {
auto db = checkoutTransaction(); auto db = checkoutTransaction();
auto newmarker = detail::oldestAccountTxPage( auto newmarker =
*db, detail::oldestAccountTxPage(
idCache, *db, onUnsavedLedger, onTransaction, options, 0, page_length)
onUnsavedLedger, .first;
onTransaction,
options,
0,
page_length)
.first;
return {ret, newmarker}; return {ret, newmarker};
} }
@@ -1500,7 +1480,6 @@ SQLiteDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options)
return false; return false;
auto [marker, total] = detail::oldestAccountTxPage( auto [marker, total] = detail::oldestAccountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
opt, opt,
@@ -1528,7 +1507,6 @@ SQLiteDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options)
return {}; return {};
static std::uint32_t const page_length(500); static std::uint32_t const page_length(500);
auto& idCache = app_.accountIDCache();
auto onUnsavedLedger = auto onUnsavedLedger =
std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1);
MetaTxsList ret; MetaTxsList ret;
@@ -1543,15 +1521,10 @@ SQLiteDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options)
if (existsTransaction()) if (existsTransaction())
{ {
auto db = checkoutTransaction(); auto db = checkoutTransaction();
auto newmarker = detail::newestAccountTxPage( auto newmarker =
*db, detail::newestAccountTxPage(
idCache, *db, onUnsavedLedger, onTransaction, options, 0, page_length)
onUnsavedLedger, .first;
onTransaction,
options,
0,
page_length)
.first;
return {ret, newmarker}; return {ret, newmarker};
} }
@@ -1568,7 +1541,6 @@ SQLiteDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options)
return false; return false;
auto [marker, total] = detail::newestAccountTxPage( auto [marker, total] = detail::newestAccountTxPage(
session, session,
idCache,
onUnsavedLedger, onUnsavedLedger,
onTransaction, onTransaction,
opt, opt,

View File

@@ -57,7 +57,8 @@ enum class SizedItem : std::size_t {
lgrDBCache, lgrDBCache,
openFinalLimit, openFinalLimit,
burstSize, burstSize,
ramSizeGB ramSizeGB,
accountIdCacheSize,
}; };
// This entire derived class is deprecated. // This entire derived class is deprecated.

View File

@@ -108,26 +108,27 @@ namespace ripple {
// clang-format off // clang-format off
// The configurable node sizes are "tiny", "small", "medium", "large", "huge" // The configurable node sizes are "tiny", "small", "medium", "large", "huge"
inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 12> inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 13>
sizedItems sizedItems
{{ {{
// FIXME: We should document each of these items, explaining exactly // FIXME: We should document each of these items, explaining exactly
// what they control and whether there exists an explicit // what they control and whether there exists an explicit
// config option that can be used to override the default. // config option that can be used to override the default.
// tiny small medium large huge // tiny small medium large huge
{SizedItem::sweepInterval, {{ 10, 30, 60, 90, 120 }}}, {SizedItem::sweepInterval, {{ 10, 30, 60, 90, 120 }}},
{SizedItem::treeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}}, {SizedItem::treeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}},
{SizedItem::treeCacheAge, {{ 30, 60, 90, 120, 900 }}}, {SizedItem::treeCacheAge, {{ 30, 60, 90, 120, 900 }}},
{SizedItem::ledgerSize, {{ 32, 32, 64, 256, 384 }}}, {SizedItem::ledgerSize, {{ 32, 32, 64, 256, 384 }}},
{SizedItem::ledgerAge, {{ 30, 60, 180, 300, 600 }}}, {SizedItem::ledgerAge, {{ 30, 60, 180, 300, 600 }}},
{SizedItem::ledgerFetch, {{ 2, 3, 4, 5, 8 }}}, {SizedItem::ledgerFetch, {{ 2, 3, 4, 5, 8 }}},
{SizedItem::hashNodeDBCache, {{ 4, 12, 24, 64, 128 }}}, {SizedItem::hashNodeDBCache, {{ 4, 12, 24, 64, 128 }}},
{SizedItem::txnDBCache, {{ 4, 12, 24, 64, 128 }}}, {SizedItem::txnDBCache, {{ 4, 12, 24, 64, 128 }}},
{SizedItem::lgrDBCache, {{ 4, 8, 16, 32, 128 }}}, {SizedItem::lgrDBCache, {{ 4, 8, 16, 32, 128 }}},
{SizedItem::openFinalLimit, {{ 8, 16, 32, 64, 128 }}}, {SizedItem::openFinalLimit, {{ 8, 16, 32, 64, 128 }}},
{SizedItem::burstSize, {{ 4, 8, 16, 32, 48 }}}, {SizedItem::burstSize, {{ 4, 8, 16, 32, 48 }}},
{SizedItem::ramSizeGB, {{ 8, 12, 16, 24, 32 }}}, {SizedItem::ramSizeGB, {{ 8, 12, 16, 24, 32 }}},
{SizedItem::accountIdCacheSize, {{ 20047, 50053, 77081, 150061, 300007 }}}
}}; }};
// Ensure that the order of entries in the table corresponds to the // Ensure that the order of entries in the table corresponds to the

View File

@@ -106,41 +106,20 @@ operator<<(std::ostream& os, AccountID const& x)
return os; return os;
} }
//------------------------------------------------------------------------------ /** Initialize the global cache used to map AccountID to base58 conversions.
/** Caches the base58 representations of AccountIDs The cache is optional and need not be initialized. But because conversion
is expensive (it requires a SHA-256 operation) in most cases the overhead
of the cache is worth the benefit.
This operation occurs with sufficient frequency to @param count The number of entries the cache should accomodate. Zero will
justify having a cache. In the future, rippled should disable the cache, releasing any memory associated with it.
require clients to receive "binary" results, where
AccountIDs are hex-encoded.
*/
class AccountIDCache
{
private:
std::mutex mutable mutex_;
std::size_t capacity_;
hash_map<AccountID, std::string> mutable m0_;
hash_map<AccountID, std::string> mutable m1_;
public: @note The function will only initialize the cache the first time it is
AccountIDCache(AccountIDCache const&) = delete; invoked. Subsequent invocations do nothing.
AccountIDCache& */
operator=(AccountIDCache const&) = delete; void
initAccountIdCache(std::size_t count);
explicit AccountIDCache(std::size_t capacity);
/** Return ripple::toBase58 for the AccountID
Thread Safety:
Safe to call from any thread concurrently
@note This function intentionally returns a
copy for correctness.
*/
std::string
toBase58(AccountID const&) const;
};
} // namespace ripple } // namespace ripple

View File

@@ -17,17 +17,95 @@
*/ */
//============================================================================== //==============================================================================
#include <ripple/basics/hardened_hash.h>
#include <ripple/basics/spinlock.h>
#include <ripple/protocol/AccountID.h> #include <ripple/protocol/AccountID.h>
#include <ripple/protocol/PublicKey.h> #include <ripple/protocol/PublicKey.h>
#include <ripple/protocol/digest.h> #include <ripple/protocol/digest.h>
#include <ripple/protocol/tokens.h> #include <ripple/protocol/tokens.h>
#include <array>
#include <cstring> #include <cstring>
#include <mutex>
namespace ripple { namespace ripple {
namespace detail {
/** Caches the base58 representations of AccountIDs */
class AccountIdCache
{
private:
struct CachedAccountID
{
AccountID id;
char encoding[40] = {0};
};
// The actual cache
std::vector<CachedAccountID> cache_;
// We use a hash function designed to resist algorithmic complexity attacks
hardened_hash<> hasher_;
// 64 spinlocks, packed into a single 64-bit value
std::atomic<std::uint64_t> locks_ = 0;
public:
AccountIdCache(std::size_t count) : cache_(count)
{
// This is non-binding, but we try to avoid wasting memory that
// is caused by overallocation.
cache_.shrink_to_fit();
}
std::string
toBase58(AccountID const& id)
{
auto const index = hasher_(id) % cache_.size();
packed_spinlock sl(locks_, index % 64);
{
std::lock_guard lock(sl);
// The check against the first character of the encoding ensures
// that we don't mishandle the case of the all-zero account:
if (cache_[index].encoding[0] != 0 && cache_[index].id == id)
return cache_[index].encoding;
}
auto ret =
encodeBase58Token(TokenType::AccountID, id.data(), id.size());
assert(ret.size() <= 38);
{
std::lock_guard lock(sl);
cache_[index].id = id;
std::strcpy(cache_[index].encoding, ret.c_str());
}
return ret;
}
};
} // namespace detail
static std::unique_ptr<detail::AccountIdCache> accountIdCache;
void
initAccountIdCache(std::size_t count)
{
if (!accountIdCache && count != 0)
accountIdCache = std::make_unique<detail::AccountIdCache>(count);
}
std::string std::string
toBase58(AccountID const& v) toBase58(AccountID const& v)
{ {
if (accountIdCache)
return accountIdCache->toBase58(v);
return encodeBase58Token(TokenType::AccountID, v.data(), v.size()); return encodeBase58Token(TokenType::AccountID, v.data(), v.size());
} }
@@ -112,52 +190,4 @@ to_issuer(AccountID& issuer, std::string const& s)
return true; return true;
} }
//------------------------------------------------------------------------------
/* VFALCO NOTE
An alternate implementation could use a pair of insert-only
hash maps that each use a single large memory allocation
to store a fixed size hash table and all of the AccountID/string
pairs laid out in memory (wouldn't use std::string here just a
length prefixed or zero terminated array). Possibly using
boost::intrusive as the basis for the unordered container.
This would cut down to one allocate/free cycle per swap of
the map.
*/
AccountIDCache::AccountIDCache(std::size_t capacity) : capacity_(capacity)
{
m1_.reserve(capacity_);
}
std::string
AccountIDCache::toBase58(AccountID const& id) const
{
std::lock_guard lock(mutex_);
auto iter = m1_.find(id);
if (iter != m1_.end())
return iter->second;
iter = m0_.find(id);
std::string result;
if (iter != m0_.end())
{
result = iter->second;
// Can use insert-only hash maps if
// we didn't erase from here.
m0_.erase(iter);
}
else
{
result = ripple::toBase58(id);
}
if (m1_.size() >= capacity_)
{
m0_ = std::move(m1_);
m1_.clear();
m1_.reserve(capacity_);
}
m1_.emplace(id, result);
return result;
}
} // namespace ripple } // namespace ripple

View File

@@ -202,7 +202,7 @@ doAccountChannels(RPC::JsonContext& context)
to_string(*marker) + "," + std::to_string(nextHint); to_string(*marker) + "," + std::to_string(nextHint);
} }
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
for (auto const& item : visitData.items) for (auto const& item : visitData.items)
addChannel(jsonChannels, *item); addChannel(jsonChannels, *item);

View File

@@ -215,7 +215,7 @@ doAccountInfo(RPC::JsonContext& context)
} }
else else
{ {
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
RPC::inject_error(rpcACT_NOT_FOUND, result); RPC::inject_error(rpcACT_NOT_FOUND, result);
} }

View File

@@ -252,7 +252,7 @@ doAccountLines(RPC::JsonContext& context)
to_string(*marker) + "," + std::to_string(nextHint); to_string(*marker) + "," + std::to_string(nextHint);
} }
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
for (auto const& item : visitData.items) for (auto const& item : visitData.items)
addLine(jsonLines, item); addLine(jsonLines, item);

View File

@@ -160,7 +160,7 @@ doAccountNFTs(RPC::JsonContext& context)
cp = nullptr; cp = nullptr;
} }
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
context.loadType = Resource::feeMediumBurdenRPC; context.loadType = Resource::feeMediumBurdenRPC;
return result; return result;
} }
@@ -275,7 +275,7 @@ doAccountObjects(RPC::JsonContext& context)
result[jss::account_objects] = Json::arrayValue; result[jss::account_objects] = Json::arrayValue;
} }
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
context.loadType = Resource::feeMediumBurdenRPC; context.loadType = Resource::feeMediumBurdenRPC;
return result; return result;
} }

View File

@@ -77,7 +77,7 @@ doAccountOffers(RPC::JsonContext& context)
} }
// Get info on account. // Get info on account.
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
if (!ledger->exists(keylet::account(accountID))) if (!ledger->exists(keylet::account(accountID)))
return rpcError(rpcACT_NOT_FOUND); return rpcError(rpcACT_NOT_FOUND);

View File

@@ -80,7 +80,7 @@ doGatewayBalances(RPC::JsonContext& context)
context.loadType = Resource::feeHighBurdenRPC; context.loadType = Resource::feeHighBurdenRPC;
result[jss::account] = context.app.accountIDCache().toBase58(accountID); result[jss::account] = toBase58(accountID);
// Parse the specified hotwallet(s), if any // Parse the specified hotwallet(s), if any
std::set<AccountID> hotWallets; std::set<AccountID> hotWallets;

View File

@@ -41,12 +41,10 @@ appendNftOfferJson(
obj[jss::nft_offer_index] = to_string(offer->key()); obj[jss::nft_offer_index] = to_string(offer->key());
obj[jss::flags] = (*offer)[sfFlags]; obj[jss::flags] = (*offer)[sfFlags];
obj[jss::owner] = obj[jss::owner] = toBase58(offer->getAccountID(sfOwner));
app.accountIDCache().toBase58(offer->getAccountID(sfOwner));
if (offer->isFieldPresent(sfDestination)) if (offer->isFieldPresent(sfDestination))
obj[jss::destination] = obj[jss::destination] = toBase58(offer->getAccountID(sfDestination));
app.accountIDCache().toBase58(offer->getAccountID(sfDestination));
if (offer->isFieldPresent(sfExpiration)) if (offer->isFieldPresent(sfExpiration))
obj[jss::expiration] = offer->getFieldU32(sfExpiration); obj[jss::expiration] = offer->getFieldU32(sfExpiration);

View File

@@ -40,7 +40,7 @@ fillTransaction(
ReadView const& ledger) ReadView const& ledger)
{ {
txArray["Sequence"] = Json::UInt(sequence++); txArray["Sequence"] = Json::UInt(sequence++);
txArray["Account"] = context.app.accountIDCache().toBase58(accountID); txArray["Account"] = toBase58(accountID);
auto& fees = ledger.fees(); auto& fees = ledger.fees();
// Convert the reference transaction cost in fee units to drops // Convert the reference transaction cost in fee units to drops
// scaled to represent the current fee load. // scaled to represent the current fee load.