//------------------------------------------------------------------------------ /* This file is part of clio: https://github.com/XRPLF/clio Copyright (c) 2023, the clio developers. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== #include "data/BackendInterface.hpp" #include "data/CassandraBackend.hpp" #include "data/DBHelpers.hpp" #include "data/LedgerCache.hpp" #include "data/LedgerHeaderCache.hpp" #include "data/Types.hpp" #include "data/cassandra/Handle.hpp" #include "data/cassandra/SettingsProvider.hpp" #include "data/cassandra/Types.hpp" #include "etl/NFTHelpers.hpp" #include "rpc/RPCHelpers.hpp" #include "util/AsioContextTestFixture.hpp" #include "util/LedgerUtils.hpp" #include "util/MockLedgerHeaderCache.hpp" #include "util/MockPrometheus.hpp" #include "util/Random.hpp" #include "util/Spawn.hpp" #include "util/StringUtils.hpp" #include "util/config/ConfigValue.hpp" #include "util/config/ObjectView.hpp" #include "util/config/Types.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace util; using namespace util::config; using namespace std; using namespace rpc; using namespace prometheus; using namespace data::cassandra; class BackendCassandraTestBase : public SyncAsioContextTest, public WithPrometheus { protected: static constexpr auto kCASSANDRA = "cassandra"; ClioConfigDefinition cfg_{ {"database.type", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)}, {"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, {"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()}, {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)}, {"database.cassandra.provider", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)}, {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}, {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)}, {"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)}, {"database.cassandra.threads", ConfigValue{ConfigType::Integer}.defaultValue(static_cast(std::thread::hardware_concurrency()))}, {"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)}, {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()}, {"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)}, {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(10).optional()}, {"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(10).optional()}, {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)} }; static constexpr auto kRAWHEADER = "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E" "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A" "6DB6FE30CC5909B285080FCD6773CC883F9FE0EE4D439340AC592AADB973ED3CF5" "3E2232B33EF57CECAC2816E3122816E31A0A00F8377CD95DFA484CFAE282656A58" "CE5AA29652EFFD80AC59CD91416E4E13DBBE"; ObjectView obj_ = cfg_.getObject("database.cassandra"); SettingsProvider settingsProvider_{obj_}; // recreated for each test data::LedgerCache cache_; std::default_random_engine randomEngine_{0}; public: ~BackendCassandraTestBase() override { // drop the keyspace for next test Handle const handle{TestGlobals::instance().backendHost}; EXPECT_TRUE(handle.connect()); handle.execute("DROP KEYSPACE " + TestGlobals::instance().backendKeyspace); } }; class BackendCassandraTest : public BackendCassandraTestBase { protected: std::unique_ptr backend_{std::make_unique(settingsProvider_, cache_, false)}; }; TEST_F(BackendCassandraTest, Basic) { std::atomic_bool done = false; auto work = std::make_optional(boost::asio::make_work_guard(ctx_)); util::spawn(ctx_, [this, &done, &work](boost::asio::yield_context yield) { std::string const rawHeader = "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E" "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A" "6DB6FE30CC5909B285080FCD6773CC883F9FE0EE4D439340AC592AADB973ED3CF5" "3E2232B33EF57CECAC2816E3122816E31A0A00F8377CD95DFA484CFAE282656A58" "CE5AA29652EFFD80AC59CD91416E4E13DBBE"; std::string rawHeaderBlob = hexStringToBinaryString(rawHeader); ripple::LedgerHeader const lgrInfo = util::deserializeHeader(ripple::makeSlice(rawHeaderBlob)); backend_->writeLedger(lgrInfo, std::move(rawHeaderBlob)); backend_->writeSuccessor(uint256ToString(data::kFIRST_KEY), lgrInfo.seq, uint256ToString(data::kLAST_KEY)); ASSERT_TRUE(backend_->finishWrites(lgrInfo.seq)); { auto rng = backend_->fetchLedgerRange(); ASSERT_TRUE(rng.has_value()); EXPECT_EQ(rng->minSequence, rng->maxSequence); EXPECT_EQ(rng->maxSequence, lgrInfo.seq); } { auto seq = backend_->fetchLatestLedgerSequence(yield); ASSERT_TRUE(seq.has_value()); EXPECT_EQ(*seq, lgrInfo.seq); } { auto retLgr = backend_->fetchLedgerBySequence(lgrInfo.seq, yield); ASSERT_TRUE(retLgr.has_value()); EXPECT_EQ(retLgr->seq, lgrInfo.seq); EXPECT_EQ(ledgerHeaderToBlob(lgrInfo), ledgerHeaderToBlob(*retLgr)); } EXPECT_FALSE(backend_->fetchLedgerBySequence(lgrInfo.seq + 1, yield).has_value()); auto lgrInfoOld = lgrInfo; auto lgrInfoNext = lgrInfo; lgrInfoNext.seq = lgrInfo.seq + 1; lgrInfoNext.parentHash = lgrInfo.hash; lgrInfoNext.hash++; lgrInfoNext.accountHash = ~lgrInfo.accountHash; { std::string infoBlob = ledgerHeaderToBinaryString(lgrInfoNext); backend_->writeLedger(lgrInfoNext, std::move(infoBlob)); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng.has_value()); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); } { auto seq = backend_->fetchLatestLedgerSequence(yield); EXPECT_EQ(seq, lgrInfoNext.seq); } { auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr.has_value()); EXPECT_EQ(retLgr->seq, lgrInfoNext.seq); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); EXPECT_NE(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoOld)); retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq - 1, yield); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoOld)); EXPECT_NE(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq - 2, yield); EXPECT_FALSE(backend_->fetchLedgerBySequence(lgrInfoNext.seq - 2, yield).has_value()); auto txns = backend_->fetchAllTransactionsInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(txns.size(), 0); auto hashes = backend_->fetchAllTransactionHashesInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(hashes.size(), 0); } // the below dummy data is not expected to be consistent. The // metadata string does represent valid metadata. Don't assume // though that the transaction or its hash correspond to the // metadata, or anything like that. These tests are purely // binary tests to make sure the same data that goes in, comes // back out std::string const metaHex = "201C0000001AF8E411006F560A3E08122A05AC91DEFA87052B0554E4A29B46" "3A27642EBB060B6052196592EEE72200000000240480FDB52503CE1A863300" "000000000000003400000000000000005529983CBAED30F547471452921C3C" "6B9F9685F292F6291000EED0A44413AF18C250101AC09600F4B502C8F7F830" "F80B616DCB6F3970CB79AB70975A05ED5B66860B9564400000001FE217CB65" "D54B640B31521B05000000000000000000000000434E5900000000000360E3" "E0751BD9A566CD03FA6CAFC78118B82BA081142252F328CF91263417762570" "D67220CCB33B1370E1E1E3110064561AC09600F4B502C8F7F830F80B616DCB" "6F3970CB79AB70975A05ED33DF783681E8365A05ED33DF783681581AC09600" "F4B502C8F7F830F80B616DCB6F3970CB79AB70975A05ED33DF783681031100" "0000000000000000000000434E59000000000004110360E3E0751BD9A566CD" "03FA6CAFC78118B82BA0E1E1E4110064561AC09600F4B502C8F7F830F80B61" "6DCB6F3970CB79AB70975A05ED5B66860B95E72200000000365A05ED5B6686" "0B95581AC09600F4B502C8F7F830F80B616DCB6F3970CB79AB70975A05ED5B" "66860B95011100000000000000000000000000000000000000000211000000" "00000000000000000000000000000000000311000000000000000000000000" "434E59000000000004110360E3E0751BD9A566CD03FA6CAFC78118B82BA0E1" "E1E311006F5647B05E66DE9F3DF2689E8F4CE6126D3136B6C5E79587F9D24B" "D71A952B0852BAE8240480FDB950101AC09600F4B502C8F7F830F80B616DCB" "6F3970CB79AB70975A05ED33DF78368164400000033C83A95F65D59D9A6291" "9C2D18000000000000000000000000434E5900000000000360E3E0751BD9A5" "66CD03FA6CAFC78118B82BA081142252F328CF91263417762570D67220CCB3" "3B1370E1E1E511006456AEA3074F10FE15DAC592F8A0405C61FB7D4C98F588" "C2D55C84718FAFBBD2604AE722000000003100000000000000003200000000" "0000000058AEA3074F10FE15DAC592F8A0405C61FB7D4C98F588C2D55C8471" "8FAFBBD2604A82142252F328CF91263417762570D67220CCB33B1370E1E1E5" "1100612503CE1A8755CE935137F8C6C8DEF26B5CD93BE18105CA83F65E1E90" "CEC546F562D25957DC0856E0311EB450B6177F969B94DBDDA83E99B7A0576A" "CD9079573876F16C0C004F06E6240480FDB9624000000005FF0E2BE1E72200" "000000240480FDBA2D00000005624000000005FF0E1F81142252F328CF9126" "3417762570D67220CCB33B1370E1E1F1031000"; std::string const txnHex = "1200072200000000240480FDB920190480FDB5201B03CE1A8964400000033C" "83A95F65D59D9A62919C2D18000000000000000000000000434E5900000000" "000360E3E0751BD9A566CD03FA6CAFC78118B82BA068400000000000000C73" "21022D40673B44C82DEE1DDB8B9BB53DCCE4F97B27404DB850F068DD91D685" "E337EA7446304402202EA6B702B48B39F2197112382838F92D4C02948E9911" "FE6B2DEBCF9183A426BC022005DAC06CD4517E86C2548A80996019F3AC60A0" "9EED153BF60C992930D68F09F981142252F328CF91263417762570D67220CC" "B33B1370"; std::string const hashHex = "0A81FB3D6324C2DCF73131505C6E4DC67981D7FC39F5E9574CEC4B1F22D28BF7"; // this account is not related to the above transaction and // metadata std::string const accountHex = "1100612200000000240480FDBC2503CE1A872D0000000555516931B2AD018EFFBE" "17C5C9DCCF872F36837C2C6136ACF80F2A24079CF81FD0624000000005FF0E0781" "142252F328CF91263417762570D67220CCB33B1370"; std::string const accountIndexHex = "E0311EB450B6177F969B94DBDDA83E99B7A0576ACD9079573876F16C0C004F06"; // An NFTokenMint tx std::string const nftTxnHex = "1200192200000008240011CC9B201B001F71D6202A0000000168400000" "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" "DE3538F248662FC73C"; std::string const nftTxnMeta = "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" "C200000028751868747470733A2F2F677265677765697362726F642E63" "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" "9808B6B90000001D751868747470733A2F2F677265677765697362726F" "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" "00001F751868747470733A2F2F677265677765697362726F642E636F6D" "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" "5DAE00000014751868747470733A2F2F677265677765697362726F642E" "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" "3CCE1462A500000009751868747470733A2F2F67726567776569736272" "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" "00000029751868747470733A2F2F677265677765697362726F642E636F" "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" "EE87B80000001E751868747470733A2F2F677265677765697362726F64" "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" "B100000015751868747470733A2F2F677265677765697362726F642E63" "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" "2FC73CE1E1F1031000"; std::string const nftTxnHashHex = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757" "E9284E"; ripple::uint256 nftID; EXPECT_TRUE(nftID.parseHex( "000800006203F49C21D5D6E022CB16DE3538F248662" "FC73CEF7FF5C60000002C" )); std::string metaBlob = hexStringToBinaryString(metaHex); std::string txnBlob = hexStringToBinaryString(txnHex); std::string const hashBlob = hexStringToBinaryString(hashHex); std::string accountBlob = hexStringToBinaryString(accountHex); std::string const accountIndexBlob = hexStringToBinaryString(accountIndexHex); std::vector affectedAccounts; std::string nftTxnBlob = hexStringToBinaryString(nftTxnHex); std::string const nftTxnMetaBlob = hexStringToBinaryString(nftTxnMeta); { lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.txHash = ~lgrInfo.txHash; lgrInfoNext.accountHash = lgrInfoNext.accountHash ^ lgrInfoNext.txHash; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; ripple::uint256 hash256; EXPECT_TRUE(hash256.parseHex(hashHex)); ripple::TxMeta const txMeta{hash256, lgrInfoNext.seq, metaBlob}; auto accountsSet = txMeta.getAffectedAccounts(); for (auto& a : accountsSet) { affectedAccounts.push_back(a); } std::vector accountTxData; accountTxData.emplace_back(txMeta, hash256); ripple::uint256 nftHash256; EXPECT_TRUE(nftHash256.parseHex(nftTxnHashHex)); ripple::TxMeta const nftTxMeta{nftHash256, lgrInfoNext.seq, nftTxnMetaBlob}; ripple::SerialIter it{nftTxnBlob.data(), nftTxnBlob.size()}; ripple::STTx const sttx{it}; auto const [parsedNFTTxsRef, parsedNFT] = etl::getNFTDataFromTx(nftTxMeta, sttx); // need to copy the nft txns so we can std::move later std::vector parsedNFTTxs; parsedNFTTxs.insert(parsedNFTTxs.end(), parsedNFTTxsRef.begin(), parsedNFTTxsRef.end()); EXPECT_EQ(parsedNFTTxs.size(), 1); EXPECT_TRUE(parsedNFT.has_value()); EXPECT_EQ(parsedNFT->tokenID, nftID); std::vector nftData; nftData.push_back(*parsedNFT); backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); backend_->writeTransaction( std::string{hashBlob}, lgrInfoNext.seq, lgrInfoNext.closeTime.time_since_epoch().count(), std::string{txnBlob}, std::string{metaBlob} ); backend_->writeAccountTransactions(std::move(accountTxData)); backend_->writeNFTs(nftData); backend_->writeNFTTransactions(parsedNFTTxs); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{accountBlob}); backend_->writeSuccessor(uint256ToString(data::kFIRST_KEY), lgrInfoNext.seq, std::string{accountIndexBlob}); backend_->writeSuccessor(std::string{accountIndexBlob}, lgrInfoNext.seq, uint256ToString(data::kLAST_KEY)); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); auto allTransactions = backend_->fetchAllTransactionsInLedger(lgrInfoNext.seq, yield); ASSERT_EQ(allTransactions.size(), 1); EXPECT_STREQ( reinterpret_cast(allTransactions[0].transaction.data()), static_cast(txnBlob.data()) ); EXPECT_STREQ( reinterpret_cast(allTransactions[0].metadata.data()), static_cast(metaBlob.data()) ); auto hashes = backend_->fetchAllTransactionHashesInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(hashes.size(), 1); EXPECT_EQ(ripple::strHex(hashes[0]), hashHex); for (auto& a : affectedAccounts) { auto [accountTransactions, cursor] = backend_->fetchAccountTransactions(a, 100, true, {}, yield); EXPECT_EQ(accountTransactions.size(), 1); EXPECT_EQ(accountTransactions[0], accountTransactions[0]); EXPECT_FALSE(cursor); } auto nft = backend_->fetchNFT(nftID, lgrInfoNext.seq, yield); EXPECT_TRUE(nft.has_value()); auto [nftTxns, cursor] = backend_->fetchNFTTransactions(nftID, 100, true, {}, yield); EXPECT_EQ(nftTxns.size(), 1); EXPECT_EQ(nftTxns[0], nftTxns[0]); EXPECT_FALSE(cursor); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } std::string accountBlobOld = accountBlob; { lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; lgrInfoNext.txHash = lgrInfoNext.txHash ^ lgrInfoNext.accountHash; lgrInfoNext.accountHash = ~(lgrInfoNext.accountHash ^ lgrInfoNext.txHash); backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); std::shuffle(accountBlob.begin(), accountBlob.end(), randomEngine_); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{accountBlob}); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); auto txns = backend_->fetchAllTransactionsInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(txns.size(), 0); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq - 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlobOld.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } { lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; lgrInfoNext.txHash = lgrInfoNext.txHash ^ lgrInfoNext.accountHash; lgrInfoNext.accountHash = ~(lgrInfoNext.accountHash ^ lgrInfoNext.txHash); backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{}); backend_->writeSuccessor( uint256ToString(data::kFIRST_KEY), lgrInfoNext.seq, uint256ToString(data::kLAST_KEY) ); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); auto txns = backend_->fetchAllTransactionsInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(txns.size(), 0); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_FALSE(obj); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_FALSE(obj); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq - 2, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlobOld.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } auto generateObjects = [](size_t numObjects, uint32_t ledgerSequence) { std::vector> res{numObjects}; ripple::uint256 key; key = ledgerSequence * 100000ul; for (auto& blob : res) { ++key; std::string const keyStr{reinterpret_cast(key.data()), ripple::uint256::size()}; blob.first = keyStr; blob.second = std::to_string(ledgerSequence) + keyStr; } return res; }; auto updateObjects = [](uint32_t ledgerSequence, auto objs) { for (auto& [key, obj] : objs) { obj = std::to_string(ledgerSequence) + obj; } return objs; }; auto generateTxns = [](size_t numTxns, uint32_t ledgerSequence) { std::vector> res{numTxns}; ripple::uint256 base; base = ledgerSequence * 100000ul; for (auto& blob : res) { ++base; std::string const hashStr{reinterpret_cast(base.data()), ripple::uint256::size()}; std::string const txnStr = "tx" + std::to_string(ledgerSequence) + hashStr; std::string const metaStr = "meta" + std::to_string(ledgerSequence) + hashStr; blob = std::make_tuple(hashStr, txnStr, metaStr); } return res; }; auto generateAccounts = [](uint32_t ledgerSequence, uint32_t numAccounts) { std::vector accounts; ripple::AccountID base; base = ledgerSequence * 998765ul; for (size_t i = 0; i < numAccounts; ++i) { ++base; accounts.push_back(base); } return accounts; }; auto generateAccountTx = [&](uint32_t ledgerSequence, auto txns) { std::vector ret; util::MTRandomGenerator randomGenerator; auto accounts = generateAccounts(ledgerSequence, 10); uint32_t idx = 0; for (auto& [hash, txn, meta] : txns) { AccountTransactionsData data; data.ledgerSequence = ledgerSequence; data.transactionIndex = idx; data.txHash = hash; for (size_t i = 0; i < 3; ++i) { data.accounts.insert(accounts[randomGenerator.uniform(0ul, accounts.size() - 1)]); } ++idx; ret.push_back(data); } return ret; }; auto generateNextLedger = [this](auto lgrInfo) { ++lgrInfo.seq; lgrInfo.parentHash = lgrInfo.hash; std::shuffle(lgrInfo.txHash.begin(), lgrInfo.txHash.end(), randomEngine_); std::shuffle(lgrInfo.accountHash.begin(), lgrInfo.accountHash.end(), randomEngine_); std::shuffle(lgrInfo.hash.begin(), lgrInfo.hash.end(), randomEngine_); return lgrInfo; }; auto writeLedger = [&](auto lgrInfo, auto txns, auto objs, auto accountTx, auto state) { backend_->startWrites(); backend_->writeLedger(lgrInfo, ledgerHeaderToBinaryString(lgrInfo)); for (auto [hash, txn, meta] : txns) { backend_->writeTransaction( std::move(hash), lgrInfo.seq, lgrInfo.closeTime.time_since_epoch().count(), std::move(txn), std::move(meta) ); } for (auto const& [key, obj] : objs) { backend_->writeLedgerObject(std::string{key}, lgrInfo.seq, std::string{obj}); } if (state.count(lgrInfo.seq - 1) == 0 || std::find_if(state[lgrInfo.seq - 1].begin(), state[lgrInfo.seq - 1].end(), [&](auto obj) { return obj.first == objs[0].first; }) == state[lgrInfo.seq - 1].end()) { for (size_t i = 0; i < objs.size(); ++i) { if (i + 1 < objs.size()) { backend_->writeSuccessor( std::string{objs[i].first}, lgrInfo.seq, std::string{objs[i + 1].first} ); } else { backend_->writeSuccessor( std::string{objs[i].first}, lgrInfo.seq, uint256ToString(data::kLAST_KEY) ); } } if (state.contains(lgrInfo.seq - 1)) { backend_->writeSuccessor( std::string{state[lgrInfo.seq - 1].back().first}, lgrInfo.seq, std::string{objs[0].first} ); } else { backend_->writeSuccessor( uint256ToString(data::kFIRST_KEY), lgrInfo.seq, std::string{objs[0].first} ); } } backend_->writeAccountTransactions(std::move(accountTx)); ASSERT_TRUE(backend_->finishWrites(lgrInfo.seq)); }; auto checkLedger = [&](auto lgrInfo, auto txns, auto objs, auto accountTx) { auto rng = backend_->fetchLedgerRange(); auto seq = lgrInfo.seq; EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_GE(rng->maxSequence, seq); auto retLgr = backend_->fetchLedgerBySequence(seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfo)); auto retTxns = backend_->fetchAllTransactionsInLedger(seq, yield); for (auto [hash, txn, meta] : txns) { bool found = false; for (auto [retTxn, retMeta, retSeq, retDate] : retTxns) { if (std::strncmp( reinterpret_cast(retTxn.data()), static_cast(txn.data()), txn.size() ) == 0 && std::strncmp( reinterpret_cast(retMeta.data()), static_cast(meta.data()), meta.size() ) == 0) found = true; } ASSERT_TRUE(found); } for (auto [account, data] : accountTx) { std::vector retData; std::optional cursor; do { uint32_t const limit = 10; auto [accountTransactions, retCursor] = backend_->fetchAccountTransactions(account, limit, false, cursor, yield); if (retCursor) EXPECT_EQ(accountTransactions.size(), limit); retData.insert(retData.end(), accountTransactions.begin(), accountTransactions.end()); cursor = retCursor; } while (cursor); EXPECT_EQ(retData.size(), data.size()); for (size_t i = 0; i < retData.size(); ++i) { auto [txn, meta, _, _2] = retData[i]; auto [_3, expTxn, expMeta] = data[i]; EXPECT_STREQ(reinterpret_cast(txn.data()), static_cast(expTxn.data())); EXPECT_STREQ(reinterpret_cast(meta.data()), static_cast(expMeta.data())); } } std::vector keys; for (auto [key, obj] : objs) { auto retObj = backend_->fetchLedgerObject(binaryStringToUint256(key), seq, yield); if (obj.size()) { ASSERT_TRUE(retObj.has_value()); EXPECT_STREQ(static_cast(obj.data()), reinterpret_cast(retObj->data())); } else { ASSERT_FALSE(retObj.has_value()); } keys.push_back(binaryStringToUint256(key)); } { auto retObjs = backend_->fetchLedgerObjects(keys, seq, yield); ASSERT_EQ(retObjs.size(), objs.size()); for (size_t i = 0; i < keys.size(); ++i) { auto [key, obj] = objs[i]; auto retObj = retObjs[i]; if (obj.size()) { ASSERT_TRUE(retObj.size()); EXPECT_STREQ( static_cast(obj.data()), reinterpret_cast(retObj.data()) ); } else { ASSERT_FALSE(retObj.size()); } } } data::LedgerPage page; std::vector retObjs; do { uint32_t const limit = 10; page = backend_->fetchLedgerPage(page.cursor, seq, limit, false, yield); retObjs.insert(retObjs.end(), page.objects.begin(), page.objects.end()); } while (page.cursor); for (auto const& obj : objs) { bool found = false; for (auto const& retObj : retObjs) { if (ripple::strHex(obj.first) == ripple::strHex(retObj.key)) { found = true; ASSERT_EQ(ripple::strHex(obj.second), ripple::strHex(retObj.blob)); } } if (found != (obj.second.size() != 0)) ASSERT_EQ(found, obj.second.size() != 0); } }; std::map>> state; std::map>> allTxns; std::unordered_map> allTxnsMap; std::map>> allAccountTx; std::map lgrInfos; for (size_t i = 0; i < 10; ++i) { lgrInfoNext = generateNextLedger(lgrInfoNext); auto objs = generateObjects(25, lgrInfoNext.seq); auto txns = generateTxns(10, lgrInfoNext.seq); auto accountTx = generateAccountTx(lgrInfoNext.seq, txns); for (auto rec : accountTx) { for (auto account : rec.accounts) { allAccountTx[lgrInfoNext.seq][account].emplace_back( reinterpret_cast(rec.txHash.data()), ripple::uint256::size() ); } } EXPECT_EQ(objs.size(), 25); EXPECT_NE(objs[0], objs[1]); EXPECT_EQ(txns.size(), 10); EXPECT_NE(txns[0], txns[1]); std::ranges::sort(objs); state[lgrInfoNext.seq] = objs; writeLedger(lgrInfoNext, txns, objs, accountTx, state); allTxns[lgrInfoNext.seq] = txns; lgrInfos[lgrInfoNext.seq] = lgrInfoNext; for (auto& [hash, txn, meta] : txns) { allTxnsMap[hash] = std::make_pair(txn, meta); } } std::vector> objs; for (size_t i = 0; i < 10; ++i) { lgrInfoNext = generateNextLedger(lgrInfoNext); if (objs.empty()) { objs = generateObjects(25, lgrInfoNext.seq); } else { objs = updateObjects(lgrInfoNext.seq, objs); } auto txns = generateTxns(10, lgrInfoNext.seq); auto accountTx = generateAccountTx(lgrInfoNext.seq, txns); for (auto rec : accountTx) { for (auto account : rec.accounts) { allAccountTx[lgrInfoNext.seq][account].emplace_back( reinterpret_cast(rec.txHash.data()), ripple::uint256::size() ); } } EXPECT_EQ(objs.size(), 25); EXPECT_NE(objs[0], objs[1]); EXPECT_EQ(txns.size(), 10); EXPECT_NE(txns[0], txns[1]); std::ranges::sort(objs); state[lgrInfoNext.seq] = objs; writeLedger(lgrInfoNext, txns, objs, accountTx, state); allTxns[lgrInfoNext.seq] = txns; lgrInfos[lgrInfoNext.seq] = lgrInfoNext; for (auto& [hash, txn, meta] : txns) { allTxnsMap[hash] = std::make_pair(txn, meta); } } auto flatten = [&](uint32_t max) { std::vector> flat; std::map objs; for (auto const& [seq, diff] : state) { for (auto const& [k, v] : diff) { if (seq > max) { if (!objs.contains(k)) objs[k] = ""; } else { objs[k] = v; } } } flat.reserve(objs.size()); for (auto const& [key, value] : objs) { flat.emplace_back(key, value); } return flat; }; auto flattenAccountTx = [&](uint32_t max) { std::unordered_map>> accountTx; for (auto const& [seq, map] : allAccountTx) { if (seq > max) break; for (auto& [account, hashes] : map) { for (auto& hash : hashes) { auto& [txn, meta] = allTxnsMap[hash]; accountTx[account].emplace_back(hash, txn, meta); } } } for (auto& [account, data] : accountTx) std::ranges::reverse(data); return accountTx; }; for (auto const& [seq, diff] : state) { auto flat = flatten(seq); checkLedger(lgrInfos[seq], allTxns[seq], flat, flattenAccountTx(seq)); } done = true; work.reset(); }); ctx_.run(); ASSERT_EQ(done, true); } TEST_F(BackendCassandraTest, CacheIntegration) { std::atomic_bool done = false; auto work = std::make_optional(boost::asio::make_work_guard(ctx_)); util::spawn(ctx_, [this, &done, &work](boost::asio::yield_context yield) { backend_->cache().setFull(); // this account is not related to the above transaction and // metadata std::string const accountHex = "1100612200000000240480FDBC2503CE1A872D0000000555516931B2AD018EFFBE" "17C5C9DCCF872F36837C2C6136ACF80F2A24079CF81FD0624000000005FF0E0781" "142252F328CF91263417762570D67220CCB33B1370"; std::string const accountIndexHex = "E0311EB450B6177F969B94DBDDA83E99B7A0576ACD9079573876F16C0C004F06"; std::string rawHeaderBlob = hexStringToBinaryString(kRAWHEADER); std::string accountBlob = hexStringToBinaryString(accountHex); std::string const accountIndexBlob = hexStringToBinaryString(accountIndexHex); ripple::LedgerHeader const lgrInfo = util::deserializeHeader(ripple::makeSlice(rawHeaderBlob)); backend_->startWrites(); backend_->writeLedger(lgrInfo, std::move(rawHeaderBlob)); backend_->writeSuccessor(uint256ToString(data::kFIRST_KEY), lgrInfo.seq, uint256ToString(data::kLAST_KEY)); ASSERT_TRUE(backend_->finishWrites(lgrInfo.seq)); { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng.has_value()); EXPECT_EQ(rng->minSequence, rng->maxSequence); EXPECT_EQ(rng->maxSequence, lgrInfo.seq); } { auto seq = backend_->fetchLatestLedgerSequence(yield); EXPECT_TRUE(seq.has_value()); EXPECT_EQ(*seq, lgrInfo.seq); } { auto retLgr = backend_->fetchLedgerBySequence(lgrInfo.seq, yield); ASSERT_TRUE(retLgr.has_value()); EXPECT_EQ(retLgr->seq, lgrInfo.seq); EXPECT_EQ(ledgerHeaderToBlob(lgrInfo), ledgerHeaderToBlob(*retLgr)); } EXPECT_FALSE(backend_->fetchLedgerBySequence(lgrInfo.seq + 1, yield).has_value()); auto lgrInfoOld = lgrInfo; auto lgrInfoNext = lgrInfo; lgrInfoNext.seq = lgrInfo.seq + 1; lgrInfoNext.parentHash = lgrInfo.hash; lgrInfoNext.hash++; lgrInfoNext.accountHash = ~lgrInfo.accountHash; { std::string infoBlob = ledgerHeaderToBinaryString(lgrInfoNext); backend_->startWrites(); backend_->writeLedger(lgrInfoNext, std::move(infoBlob)); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng.has_value()); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); } { auto seq = backend_->fetchLatestLedgerSequence(yield); EXPECT_EQ(seq, lgrInfoNext.seq); } { auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr.has_value()); EXPECT_EQ(retLgr->seq, lgrInfoNext.seq); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); EXPECT_NE(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoOld)); retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq - 1, yield); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoOld)); EXPECT_NE(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq - 2, yield); EXPECT_FALSE(backend_->fetchLedgerBySequence(lgrInfoNext.seq - 2, yield).has_value()); auto txns = backend_->fetchAllTransactionsInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(txns.size(), 0); auto hashes = backend_->fetchAllTransactionHashesInLedger(lgrInfoNext.seq, yield); EXPECT_EQ(hashes.size(), 0); } { backend_->startWrites(); lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.txHash = ~lgrInfo.txHash; lgrInfoNext.accountHash = lgrInfoNext.accountHash ^ lgrInfoNext.txHash; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{accountBlob}); auto key = ripple::uint256::fromVoidChecked(accountIndexBlob); backend_->cache().update( {{.key = *key, .blob = {accountBlob.begin(), accountBlob.end()}}}, lgrInfoNext.seq ); backend_->writeSuccessor(uint256ToString(data::kFIRST_KEY), lgrInfoNext.seq, std::string{accountIndexBlob}); backend_->writeSuccessor(std::string{accountIndexBlob}, lgrInfoNext.seq, uint256ToString(data::kLAST_KEY)); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfoNext)); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } std::string accountBlobOld = accountBlob; { backend_->startWrites(); lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; lgrInfoNext.txHash = lgrInfoNext.txHash ^ lgrInfoNext.accountHash; lgrInfoNext.accountHash = ~(lgrInfoNext.accountHash ^ lgrInfoNext.txHash); backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); std::shuffle(accountBlob.begin(), accountBlob.end(), randomEngine_); auto key = ripple::uint256::fromVoidChecked(accountIndexBlob); backend_->cache().update( {{.key = *key, .blob = {accountBlob.begin(), accountBlob.end()}}}, lgrInfoNext.seq ); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{accountBlob}); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlob.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq - 1, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlobOld.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } { backend_->startWrites(); lgrInfoNext.seq = lgrInfoNext.seq + 1; lgrInfoNext.parentHash = lgrInfoNext.hash; lgrInfoNext.hash++; lgrInfoNext.txHash = lgrInfoNext.txHash ^ lgrInfoNext.accountHash; lgrInfoNext.accountHash = ~(lgrInfoNext.accountHash ^ lgrInfoNext.txHash); backend_->writeLedger(lgrInfoNext, ledgerHeaderToBinaryString(lgrInfoNext)); auto key = ripple::uint256::fromVoidChecked(accountIndexBlob); backend_->cache().update({{.key = *key, .blob = {}}}, lgrInfoNext.seq); backend_->writeLedgerObject(std::string{accountIndexBlob}, lgrInfoNext.seq, std::string{}); backend_->writeSuccessor( uint256ToString(data::kFIRST_KEY), lgrInfoNext.seq, uint256ToString(data::kLAST_KEY) ); ASSERT_TRUE(backend_->finishWrites(lgrInfoNext.seq)); } { auto rng = backend_->fetchLedgerRange(); EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq); auto retLgr = backend_->fetchLedgerBySequence(lgrInfoNext.seq, yield); EXPECT_TRUE(retLgr); ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq, yield); EXPECT_FALSE(obj); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq + 1, yield); EXPECT_FALSE(obj); obj = backend_->fetchLedgerObject(key256, lgrInfoNext.seq - 2, yield); EXPECT_TRUE(obj); EXPECT_STREQ(reinterpret_cast(obj->data()), static_cast(accountBlobOld.data())); obj = backend_->fetchLedgerObject(key256, lgrInfoOld.seq - 1, yield); EXPECT_FALSE(obj); } auto generateObjects = [](size_t numObjects, uint64_t ledgerSequence) { std::vector> res{numObjects}; ripple::uint256 key; key = ledgerSequence * 100000; for (auto& blob : res) { ++key; std::string const keyStr{reinterpret_cast(key.data()), ripple::uint256::size()}; blob.first = keyStr; blob.second = std::to_string(ledgerSequence) + keyStr; } return res; }; auto updateObjects = [](uint32_t ledgerSequence, auto objs) { for (auto& [key, obj] : objs) { obj = std::to_string(ledgerSequence) + obj; } return objs; }; auto generateNextLedger = [this](auto lgrInfo) { ++lgrInfo.seq; lgrInfo.parentHash = lgrInfo.hash; std::shuffle(lgrInfo.txHash.begin(), lgrInfo.txHash.end(), randomEngine_); std::shuffle(lgrInfo.accountHash.begin(), lgrInfo.accountHash.end(), randomEngine_); std::shuffle(lgrInfo.hash.begin(), lgrInfo.hash.end(), randomEngine_); return lgrInfo; }; auto writeLedger = [&](auto lgrInfo, auto objs, auto state) { backend_->startWrites(); backend_->writeLedger(lgrInfo, std::move(ledgerHeaderToBinaryString(lgrInfo))); std::vector cacheUpdates; for (auto [key, obj] : objs) { backend_->writeLedgerObject(std::string{key}, lgrInfo.seq, std::string{obj}); auto key256 = ripple::uint256::fromVoidChecked(key); cacheUpdates.push_back({*key256, {obj.begin(), obj.end()}}); } backend_->cache().update(cacheUpdates, lgrInfo.seq); if (state.count(lgrInfo.seq - 1) == 0 || std::find_if(state[lgrInfo.seq - 1].begin(), state[lgrInfo.seq - 1].end(), [&](auto obj) { return obj.first == objs[0].first; }) == state[lgrInfo.seq - 1].end()) { for (size_t i = 0; i < objs.size(); ++i) { if (i + 1 < objs.size()) { backend_->writeSuccessor( std::string{objs[i].first}, lgrInfo.seq, std::string{objs[i + 1].first} ); } else { backend_->writeSuccessor( std::string{objs[i].first}, lgrInfo.seq, uint256ToString(data::kLAST_KEY) ); } } if (state.contains(lgrInfo.seq - 1)) { backend_->writeSuccessor( std::string{state[lgrInfo.seq - 1].back().first}, lgrInfo.seq, std::string{objs[0].first} ); } else { backend_->writeSuccessor( uint256ToString(data::kFIRST_KEY), lgrInfo.seq, std::string{objs[0].first} ); } } ASSERT_TRUE(backend_->finishWrites(lgrInfo.seq)); }; auto checkLedger = [&](auto lgrInfo, auto objs) { auto rng = backend_->fetchLedgerRange(); auto seq = lgrInfo.seq; EXPECT_TRUE(rng); EXPECT_EQ(rng->minSequence, lgrInfoOld.seq); EXPECT_GE(rng->maxSequence, seq); auto retLgr = backend_->fetchLedgerBySequence(seq, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfo)); retLgr = backend_->fetchLedgerByHash(lgrInfo.hash, yield); EXPECT_TRUE(retLgr); EXPECT_EQ(ledgerHeaderToBlob(*retLgr), ledgerHeaderToBlob(lgrInfo)) << "retLgr seq:" << retLgr->seq << "; lgrInfo seq:" << lgrInfo.seq << "; retLgr hash:" << retLgr->hash << "; lgrInfo hash:" << lgrInfo.hash << "; retLgr parentHash:" << retLgr->parentHash << "; lgr Info parentHash:" << lgrInfo.parentHash; std::vector keys; for (auto [key, obj] : objs) { auto retObj = backend_->fetchLedgerObject(binaryStringToUint256(key), seq, yield); if (obj.size()) { ASSERT_TRUE(retObj.has_value()); EXPECT_STREQ(static_cast(obj.data()), reinterpret_cast(retObj->data())); } else { ASSERT_FALSE(retObj.has_value()); } keys.push_back(binaryStringToUint256(key)); } { auto retObjs = backend_->fetchLedgerObjects(keys, seq, yield); ASSERT_EQ(retObjs.size(), objs.size()); for (size_t i = 0; i < keys.size(); ++i) { auto [key, obj] = objs[i]; auto retObj = retObjs[i]; if (obj.size()) { ASSERT_TRUE(retObj.size()); EXPECT_STREQ( static_cast(obj.data()), reinterpret_cast(retObj.data()) ); } else { ASSERT_FALSE(retObj.size()); } } } data::LedgerPage page; std::vector retObjs; do { uint32_t const limit = 10; page = backend_->fetchLedgerPage(page.cursor, seq, limit, false, yield); retObjs.insert(retObjs.end(), page.objects.begin(), page.objects.end()); } while (page.cursor); for (auto const& obj : objs) { bool found = false; for (auto const& retObj : retObjs) { if (ripple::strHex(obj.first) == ripple::strHex(retObj.key)) { found = true; ASSERT_EQ(ripple::strHex(obj.second), ripple::strHex(retObj.blob)); } } if (found != (obj.second.size() != 0)) ASSERT_EQ(found, obj.second.size() != 0); } }; std::map>> state; std::map lgrInfos; for (size_t i = 0; i < 10; ++i) { lgrInfoNext = generateNextLedger(lgrInfoNext); auto objs = generateObjects(25, lgrInfoNext.seq); EXPECT_EQ(objs.size(), 25); EXPECT_NE(objs[0], objs[1]); std::ranges::sort(objs); state[lgrInfoNext.seq] = objs; writeLedger(lgrInfoNext, objs, state); lgrInfos[lgrInfoNext.seq] = lgrInfoNext; } std::vector> objs; for (size_t i = 0; i < 10; ++i) { lgrInfoNext = generateNextLedger(lgrInfoNext); if (objs.empty()) { objs = generateObjects(25, lgrInfoNext.seq); } else { objs = updateObjects(lgrInfoNext.seq, objs); } EXPECT_EQ(objs.size(), 25); EXPECT_NE(objs[0], objs[1]); std::ranges::sort(objs); state[lgrInfoNext.seq] = objs; writeLedger(lgrInfoNext, objs, state); lgrInfos[lgrInfoNext.seq] = lgrInfoNext; } auto flatten = [&](uint32_t max) { std::vector> flat; std::map objs; for (auto const& [seq, diff] : state) { for (auto const& [k, v] : diff) { if (seq > max) { if (!objs.contains(k)) objs[k] = ""; } else { objs[k] = v; } } } flat.reserve(objs.size()); for (auto const& [key, value] : objs) { flat.emplace_back(key, value); } return flat; }; for (auto const& [seq, diff] : state) { auto flat = flatten(seq); checkLedger(lgrInfos[seq], flat); } done = true; work.reset(); }); ctx_.run(); ASSERT_EQ(done, true); } class CacheBackendCassandraTest : public BackendCassandraTestBase { protected: using TestBackendType = data::cassandra::BasicCassandraBackend< SettingsProvider, data::cassandra::impl::DefaultExecutionStrategy<>, MockLedgerHeaderCache>; std::unique_ptr backend_{std::make_unique(settingsProvider_, cache_, false)}; public: MockLedgerHeaderCache& getMockCache() { return dynamic_cast(*backend_).ledgerCache_; } }; TEST_F(CacheBackendCassandraTest, CacheFetchLedgerBySeq) { runSpawn([&](boost::asio::yield_context yield) { auto rawHeaderBlob = hexStringToBinaryString(kRAWHEADER); ripple::LedgerHeader const lgrInfo = util::deserializeHeader(ripple::makeSlice(rawHeaderBlob)); backend_->writeLedger(lgrInfo, std::move(rawHeaderBlob)); auto const testLedgerSeq = lgrInfo.seq; ASSERT_TRUE(backend_->finishWrites(lgrInfo.seq)); EXPECT_CALL(getMockCache(), put(data::FetchLedgerCache::CacheEntry{lgrInfo, testLedgerSeq})); { testing::InSequence const s; // first time, getSeq doesn't match ledger sequence EXPECT_CALL(getMockCache(), get()).WillOnce(testing::Return(std::nullopt)); // second time, it would be cached EXPECT_CALL(getMockCache(), get()) .WillOnce(testing::Return(data::FetchLedgerCache::CacheEntry{.ledger = lgrInfo, .seq = testLedgerSeq})); } { // backend should cache the result of fetchLedgerBySequence auto const ledger = backend_->fetchLedgerBySequence(testLedgerSeq, yield); ASSERT_TRUE(ledger.has_value()); EXPECT_EQ(ledger->seq, lgrInfo.seq); } { // Second call: should return from cache auto const ledger = backend_->fetchLedgerBySequence(testLedgerSeq, yield); ASSERT_TRUE(ledger.has_value()); EXPECT_EQ(ledger->seq, lgrInfo.seq); } }); } struct BackendCassandraNodeMessageTest : BackendCassandraTest { boost::uuids::random_generator generateUuid{}; }; TEST_F(BackendCassandraNodeMessageTest, UpdateFetch) { static boost::uuids::uuid const kUUID = generateUuid(); static std::string const kMESSAGE = "some message"; EXPECT_NO_THROW({ backend_->writeNodeMessage(kUUID, kMESSAGE); }); runSpawn([&](boost::asio::yield_context yield) { auto const readResult = backend_->fetchClioNodesData(yield); ASSERT_TRUE(readResult) << readResult.error(); ASSERT_EQ(readResult->size(), 1); auto const& [uuid, message] = (*readResult)[0]; EXPECT_EQ(uuid, kUUID); EXPECT_EQ(message, kMESSAGE); }); } TEST_F(BackendCassandraNodeMessageTest, UpdateFetchMultipleMessages) { std::unordered_map kDATA = { {generateUuid(), std::string{"some message"}}, {generateUuid(), std::string{"other message"}}, {generateUuid(), std::string{"message 3"}} }; EXPECT_NO_THROW({ for (auto const& [uuid, message] : kDATA) { backend_->writeNodeMessage(uuid, message); } }); runSpawn([&](boost::asio::yield_context yield) { auto const readResult = backend_->fetchClioNodesData(yield); ASSERT_TRUE(readResult) << readResult.error(); ASSERT_EQ(readResult->size(), kDATA.size()); for (size_t i = 0; i < readResult->size(); ++i) { auto const& [uuid, message] = (*readResult)[i]; auto const it = kDATA.find(uuid); ASSERT_NE(it, kDATA.end()) << uuid << " not found"; EXPECT_EQ(it->second, message); } }); } TEST_F(BackendCassandraNodeMessageTest, MessageDisappearsAfterTTL) { EXPECT_NO_THROW({ backend_->writeNodeMessage(generateUuid(), "some message"); }); std::this_thread::sleep_for(std::chrono::milliseconds{2005}); runSpawn([&](boost::asio::yield_context yield) { auto const readResult = backend_->fetchClioNodesData(yield); ASSERT_TRUE(readResult) << readResult.error(); EXPECT_TRUE(readResult->empty()); }); } TEST_F(BackendCassandraNodeMessageTest, UpdatingMessageKeepsItAlive) { #if defined(__APPLE__) GTEST_SKIP() << "Skipping test on Apple platform due to slow DB"; #else static boost::uuids::uuid const kUUID = generateUuid(); static std::string const kUPDATED_MESSAGE = "updated message"; EXPECT_NO_THROW({ backend_->writeNodeMessage(kUUID, "some message"); }); std::this_thread::sleep_for(std::chrono::milliseconds{1000}); EXPECT_NO_THROW({ backend_->writeNodeMessage(kUUID, kUPDATED_MESSAGE); }); std::this_thread::sleep_for(std::chrono::milliseconds{1005}); runSpawn([&](boost::asio::yield_context yield) { auto const readResult = backend_->fetchClioNodesData(yield); ASSERT_TRUE(readResult) << readResult.error(); ASSERT_EQ(readResult->size(), 1); auto const& [uuid, message] = (*readResult)[0]; EXPECT_EQ(uuid, kUUID); EXPECT_EQ(message, kUPDATED_MESSAGE); }); #endif }