From 6bf8c5bc4e26ebe2808e2917cae7e96ec2d25a11 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Tue, 26 Jul 2022 15:01:14 -0400 Subject: [PATCH] Add NFT-specific data stores and add nft_info API (#98) --- CMakeLists.txt | 3 + README.md | 9 + src/backend/BackendInterface.h | 25 ++- src/backend/CassandraBackend.cpp | 288 +++++++++++++++++++++++- src/backend/CassandraBackend.h | 59 ++++- src/backend/DBHelpers.h | 55 ++++- src/backend/PostgresBackend.cpp | 39 +++- src/backend/PostgresBackend.h | 24 +- src/backend/Types.h | 25 ++- src/etl/ETLHelpers.h | 2 +- src/etl/NFTHelpers.cpp | 370 +++++++++++++++++++++++++++++++ src/etl/ReportingETL.cpp | 68 ++++-- src/etl/ReportingETL.h | 25 ++- src/rpc/Handlers.h | 3 + src/rpc/RPC.cpp | 1 + src/rpc/handlers/AccountTx.cpp | 2 +- src/rpc/handlers/NFTInfo.cpp | 146 ++++++++++++ unittests/main.cpp | 194 +++++++++++++++- 18 files changed, 1288 insertions(+), 50 deletions(-) create mode 100644 src/etl/NFTHelpers.cpp create mode 100644 src/rpc/handlers/NFTInfo.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e25d9afc..0a50733b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ target_sources(clio PRIVATE src/backend/SimpleCache.cpp ## ETL src/etl/ETLSource.cpp + src/etl/NFTHelpers.cpp src/etl/ReportingETL.cpp ## Subscriptions src/subscriptions/SubscriptionManager.cpp @@ -68,6 +69,8 @@ target_sources(clio PRIVATE src/rpc/handlers/AccountObjects.cpp src/rpc/handlers/GatewayBalances.cpp src/rpc/handlers/NoRippleCheck.cpp + # NFT + src/rpc/handlers/NFTInfo.cpp # Ledger src/rpc/handlers/Ledger.cpp src/rpc/handlers/LedgerData.cpp diff --git a/README.md b/README.md index 24b447b0..def0a42a 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,15 @@ which can cause high latencies. A possible alternative to this is to just deploy a database in each region, and the Clio nodes in each region use their region's database. This is effectively two systems. +## Developing against `rippled` in standalone mode + +If you wish you develop against a `rippled` instance running in standalone +mode there are a few quirks of both clio and rippled you need to keep in mind. +You must: + +1. Advance the `rippled` ledger to at least ledger 256 +2. Wait 10 minutes before first starting clio against this standalone node. + ## Logging Clio provides several logging options, all are configurable via the config file and are detailed below. diff --git a/src/backend/BackendInterface.h b/src/backend/BackendInterface.h index a675491c..4f7c0aee 100644 --- a/src/backend/BackendInterface.h +++ b/src/backend/BackendInterface.h @@ -162,12 +162,12 @@ public: std::vector const& hashes, boost::asio::yield_context& yield) const = 0; - virtual AccountTransactions + virtual TransactionsAndCursor fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t const limit, bool forward, - std::optional const& cursor, + std::optional const& cursor, boost::asio::yield_context& yield) const = 0; virtual std::vector @@ -180,6 +180,21 @@ public: std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0; + // *** NFT methods + virtual std::optional + fetchNFT( + ripple::uint256 const& tokenID, + std::uint32_t const ledgerSequence, + boost::asio::yield_context& yield) const = 0; + + virtual TransactionsAndCursor + fetchNFTTransactions( + ripple::uint256 const& tokenID, + std::uint32_t const limit, + bool const forward, + std::optional const& cursorIn, + boost::asio::yield_context& yield) const = 0; + // *** state data methods std::optional fetchLedgerObject( @@ -285,9 +300,15 @@ public: std::string&& transaction, std::string&& metadata) = 0; + virtual void + writeNFTs(std::vector&& data) = 0; + virtual void writeAccountTransactions(std::vector&& data) = 0; + virtual void + writeNFTTransactions(std::vector&& data) = 0; + virtual void writeSuccessor( std::string&& key, diff --git a/src/backend/CassandraBackend.cpp b/src/backend/CassandraBackend.cpp index 69835d12..f6d7c4c2 100644 --- a/src/backend/CassandraBackend.cpp +++ b/src/backend/CassandraBackend.cpp @@ -1,7 +1,9 @@ +#include #include #include #include #include + namespace Backend { // Type alias for async completion handlers @@ -256,6 +258,7 @@ CassandraBackend::writeLedger( "ledger_hash"); ledgerSequence_ = ledgerInfo.seq; } + void CassandraBackend::writeAccountTransactions( std::vector&& data) @@ -266,11 +269,11 @@ CassandraBackend::writeAccountTransactions( { makeAndExecuteAsyncWrite( this, - std::move(std::make_tuple( + std::make_tuple( std::move(account), record.ledgerSequence, record.transactionIndex, - record.txHash)), + record.txHash), [this](auto& params) { CassandraStatement statement(insertAccountTx_); auto& [account, lgrSeq, txnIdx, hash] = params.data; @@ -283,6 +286,31 @@ CassandraBackend::writeAccountTransactions( } } } + +void +CassandraBackend::writeNFTTransactions(std::vector&& data) +{ + for (NFTTransactionsData const& record : data) + { + makeAndExecuteAsyncWrite( + this, + std::make_tuple( + record.tokenID, + record.ledgerSequence, + record.transactionIndex, + record.txHash), + [this](auto const& params) { + CassandraStatement statement(insertNFTTx_); + auto const& [tokenID, lgrSeq, txnIdx, txHash] = params.data; + statement.bindNextBytes(tokenID); + statement.bindNextIntTuple(lgrSeq, txnIdx); + statement.bindNextBytes(txHash); + return statement; + }, + "nf_token_transactions"); + } +} + void CassandraBackend::writeTransaction( std::string&& hash, @@ -325,6 +353,43 @@ CassandraBackend::writeTransaction( "transaction"); } +void +CassandraBackend::writeNFTs(std::vector&& data) +{ + for (NFTsData const& record : data) + { + makeAndExecuteAsyncWrite( + this, + std::make_tuple( + record.tokenID, + record.ledgerSequence, + record.owner, + record.isBurned), + [this](auto const& params) { + CassandraStatement statement{insertNFT_}; + auto const& [tokenID, lgrSeq, owner, isBurned] = params.data; + statement.bindNextBytes(tokenID); + statement.bindNextInt(lgrSeq); + statement.bindNextBytes(owner); + statement.bindNextBoolean(isBurned); + return statement; + }, + "nf_tokens"); + + makeAndExecuteAsyncWrite( + this, + std::make_tuple(record.tokenID), + [this](auto const& params) { + CassandraStatement statement{insertIssuerNFT_}; + auto const& [tokenID] = params.data; + statement.bindNextBytes(ripple::nft::getIssuer(tokenID)); + statement.bindNextBytes(tokenID); + return statement; + }, + "issuer_nf_tokens"); + } +} + std::optional CassandraBackend::hardFetchLedgerRange(boost::asio::yield_context& yield) const { @@ -502,12 +567,113 @@ CassandraBackend::fetchAllTransactionHashesInLedger( return hashes; } -AccountTransactions +std::optional +CassandraBackend::fetchNFT( + ripple::uint256 const& tokenID, + std::uint32_t const ledgerSequence, + boost::asio::yield_context& yield) const +{ + CassandraStatement statement{selectNFT_}; + statement.bindNextBytes(tokenID); + statement.bindNextInt(ledgerSequence); + CassandraResult response = executeAsyncRead(statement, yield); + if (!response) + return {}; + + NFT result; + result.tokenID = tokenID; + result.ledgerSequence = response.getUInt32(); + result.owner = response.getBytes(); + result.isBurned = response.getBool(); + return result; +} + +TransactionsAndCursor +CassandraBackend::fetchNFTTransactions( + ripple::uint256 const& tokenID, + std::uint32_t const limit, + bool const forward, + std::optional const& cursorIn, + boost::asio::yield_context& yield) const +{ + auto cursor = cursorIn; + auto rng = fetchLedgerRange(); + if (!rng) + return {{}, {}}; + + CassandraStatement statement = forward + ? CassandraStatement(selectNFTTxForward_) + : CassandraStatement(selectNFTTx_); + + statement.bindNextBytes(tokenID); + + if (cursor) + { + statement.bindNextIntTuple( + cursor->ledgerSequence, cursor->transactionIndex); + BOOST_LOG_TRIVIAL(debug) << " token_id = " << ripple::strHex(tokenID) + << " tuple = " << cursor->ledgerSequence + << " : " << cursor->transactionIndex; + } + else + { + int const seq = forward ? rng->minSequence : rng->maxSequence; + int const placeHolder = + forward ? 0 : std::numeric_limits::max(); + + statement.bindNextIntTuple(placeHolder, placeHolder); + BOOST_LOG_TRIVIAL(debug) + << " token_id = " << ripple::strHex(tokenID) << " idx = " << seq + << " tuple = " << placeHolder; + } + + statement.bindNextUInt(limit); + + CassandraResult result = executeAsyncRead(statement, yield); + + if (!result.hasResult()) + { + BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows returned"; + return {}; + } + + std::vector hashes = {}; + auto numRows = result.numRows(); + BOOST_LOG_TRIVIAL(info) << "num_rows = " << numRows; + do + { + hashes.push_back(result.getUInt256()); + if (--numRows == 0) + { + BOOST_LOG_TRIVIAL(debug) << __func__ << " setting cursor"; + auto const [lgrSeq, txnIdx] = result.getInt64Tuple(); + cursor = { + static_cast(lgrSeq), + static_cast(txnIdx)}; + + if (forward) + ++cursor->transactionIndex; + } + } while (result.nextRow()); + + auto txns = fetchTransactions(hashes, yield); + BOOST_LOG_TRIVIAL(debug) << __func__ << " txns = " << txns.size(); + + if (txns.size() == limit) + { + BOOST_LOG_TRIVIAL(debug) << __func__ << " returning cursor"; + return {txns, cursor}; + } + + return {txns, {}}; +} + +TransactionsAndCursor CassandraBackend::fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t const limit, bool const forward, - std::optional const& cursorIn, + std::optional const& cursorIn, boost::asio::yield_context& yield) const { auto rng = fetchLedgerRange(); @@ -535,8 +701,8 @@ CassandraBackend::fetchAccountTransactions( } else { - int seq = forward ? rng->minSequence : rng->maxSequence; - int placeHolder = + int const seq = forward ? rng->minSequence : rng->maxSequence; + int const placeHolder = forward ? 0 : std::numeric_limits::max(); statement.bindNextIntTuple(placeHolder, placeHolder); @@ -584,6 +750,7 @@ CassandraBackend::fetchAccountTransactions( return {txns, {}}; } + std::optional CassandraBackend::doFetchSuccessorKey( ripple::uint256 key, @@ -1179,6 +1346,64 @@ CassandraBackend::open(bool readOnly) << " LIMIT 1"; if (!executeSimpleStatement(query.str())) continue; + + query.str(""); + query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "nf_tokens" + << " (" + << " token_id blob," + << " sequence bigint," + << " owner blob," + << " is_burned boolean," + << " PRIMARY KEY (token_id, sequence)" + << " )" + << " WITH CLUSTERING ORDER BY (sequence DESC)" + << " AND default_time_to_live = " << ttl; + if (!executeSimpleStatement(query.str())) + continue; + + query.str(""); + query << "SELECT * FROM " << tablePrefix << "nf_tokens" + << " LIMIT 1"; + if (!executeSimpleStatement(query.str())) + continue; + + query.str(""); + query << "CREATE TABLE IF NOT EXISTS " << tablePrefix + << "issuer_nf_tokens" + << " (" + << " issuer blob," + << " token_id blob," + << " PRIMARY KEY (issuer, token_id)" + << " )"; + if (!executeSimpleStatement(query.str())) + continue; + + query.str(""); + query << "SELECT * FROM " << tablePrefix << "issuer_nf_tokens" + << " LIMIT 1"; + if (!executeSimpleStatement(query.str())) + continue; + + query.str(""); + query << "CREATE TABLE IF NOT EXISTS " << tablePrefix + << "nf_token_transactions" + << " (" + << " token_id blob," + << " seq_idx tuple," + << " hash blob," + << " PRIMARY KEY (token_id, seq_idx)" + << " )" + << " WITH CLUSTERING ORDER BY (seq_idx DESC)" + << " AND default_time_to_live = " << ttl; + if (!executeSimpleStatement(query.str())) + continue; + + query.str(""); + query << "SELECT * FROM " << tablePrefix << "nf_token_transactions" + << " LIMIT 1"; + if (!executeSimpleStatement(query.str())) + continue; + setupSessionAndTable = true; } @@ -1296,6 +1521,57 @@ CassandraBackend::open(bool readOnly) if (!selectAccountTxForward_.prepareStatement(query, session_.get())) continue; + query.str(""); + query << "INSERT INTO " << tablePrefix << "nf_tokens" + << " (token_id,sequence,owner,is_burned)" + << " VALUES (?,?,?,?)"; + if (!insertNFT_.prepareStatement(query, session_.get())) + continue; + + query.str(""); + query << "SELECT sequence,owner,is_burned" + << " FROM " << tablePrefix << "nf_tokens WHERE" + << " token_id = ? AND" + << " sequence <= ?" + << " ORDER BY sequence DESC" + << " LIMIT 1"; + if (!selectNFT_.prepareStatement(query, session_.get())) + continue; + + query.str(""); + query << "INSERT INTO " << tablePrefix << "issuer_nf_tokens" + << " (issuer,token_id)" + << " VALUES (?,?)"; + if (!insertIssuerNFT_.prepareStatement(query, session_.get())) + continue; + + query.str(""); + query << "INSERT INTO " << tablePrefix << "nf_token_transactions" + << " (token_id,seq_idx,hash)" + << " VALUES (?,?,?)"; + if (!insertNFTTx_.prepareStatement(query, session_.get())) + continue; + + query.str(""); + query << "SELECT hash,seq_idx" + << " FROM " << tablePrefix << "nf_token_transactions WHERE" + << " token_id = ? AND" + << " seq_idx < ?" + << " ORDER BY seq_idx DESC" + << " LIMIT ?"; + if (!selectNFTTx_.prepareStatement(query, session_.get())) + continue; + + query.str(""); + query << "SELECT hash,seq_idx" + << " FROM " << tablePrefix << "nf_token_transactions WHERE" + << " token_id = ? AND" + << " seq_idx >= ?" + << " ORDER BY seq_idx ASC" + << " LIMIT ?"; + if (!selectNFTTxForward_.prepareStatement(query, session_.get())) + continue; + query.str(""); query << " INSERT INTO " << tablePrefix << "ledgers " << " (sequence, header) VALUES(?,?)"; diff --git a/src/backend/CassandraBackend.h b/src/backend/CassandraBackend.h index 1c60cd84..4cdddace 100644 --- a/src/backend/CassandraBackend.h +++ b/src/backend/CassandraBackend.h @@ -115,7 +115,7 @@ public: throw std::runtime_error( "CassandraStatement::bindNextBoolean - statement_ is null"); CassError rc = cass_statement_bind_bool( - statement_, 1, static_cast(val)); + statement_, curBindingIndex_, static_cast(val)); if (rc != CASS_OK) { std::stringstream ss; @@ -481,6 +481,33 @@ public: return {first, second}; } + // TODO: should be replaced with a templated implementation as is very + // similar to other getters + bool + getBool() + { + if (!row_) + { + std::stringstream msg; + msg << __func__ << " - no result"; + BOOST_LOG_TRIVIAL(error) << msg.str(); + throw std::runtime_error(msg.str()); + } + cass_bool_t val; + CassError rc = + cass_value_get_bool(cass_row_get_column(row_, curGetIndex_), &val); + if (rc != CASS_OK) + { + std::stringstream msg; + msg << __func__ << " - error getting value: " << rc << ", " + << cass_error_desc(rc); + BOOST_LOG_TRIVIAL(error) << msg.str(); + throw std::runtime_error(msg.str()); + } + ++curGetIndex_; + return val; + } + ~CassandraResult() { if (result_ != nullptr) @@ -599,6 +626,12 @@ private: CassandraPreparedStatement insertAccountTx_; CassandraPreparedStatement selectAccountTx_; CassandraPreparedStatement selectAccountTxForward_; + CassandraPreparedStatement insertNFT_; + CassandraPreparedStatement selectNFT_; + CassandraPreparedStatement insertIssuerNFT_; + CassandraPreparedStatement insertNFTTx_; + CassandraPreparedStatement selectNFTTx_; + CassandraPreparedStatement selectNFTTxForward_; CassandraPreparedStatement insertLedgerHeader_; CassandraPreparedStatement insertLedgerHash_; CassandraPreparedStatement updateLedgerRange_; @@ -683,12 +716,12 @@ public: open_ = false; } - AccountTransactions + TransactionsAndCursor fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t const limit, bool forward, - std::optional const& cursor, + std::optional const& cursor, boost::asio::yield_context& yield) const override; bool @@ -852,6 +885,20 @@ public: std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override; + std::optional + fetchNFT( + ripple::uint256 const& tokenID, + std::uint32_t const ledgerSequence, + boost::asio::yield_context& yield) const override; + + TransactionsAndCursor + fetchNFTTransactions( + ripple::uint256 const& tokenID, + std::uint32_t const limit, + bool const forward, + std::optional const& cursorIn, + boost::asio::yield_context& yield) const override; + // Synchronously fetch the object with key key, as of ledger with sequence // sequence std::optional @@ -941,6 +988,9 @@ public: writeAccountTransactions( std::vector&& data) override; + void + writeNFTTransactions(std::vector&& data) override; + void writeTransaction( std::string&& hash, @@ -949,6 +999,9 @@ public: std::string&& transaction, std::string&& metadata) override; + void + writeNFTs(std::vector&& data) override; + void startWrites() const override { diff --git a/src/backend/DBHelpers.h b/src/backend/DBHelpers.h index 04696b3e..fdebea61 100644 --- a/src/backend/DBHelpers.h +++ b/src/backend/DBHelpers.h @@ -9,8 +9,8 @@ #include #include -/// Struct used to keep track of what to write to transactions and -/// account_transactions tables in Postgres +/// Struct used to keep track of what to write to +/// account_transactions/account_tx tables struct AccountTransactionsData { boost::container::flat_set accounts; @@ -32,6 +32,57 @@ struct AccountTransactionsData AccountTransactionsData() = default; }; +/// Represents a link from a tx to an NFT that was targeted/modified/created +/// by it. Gets written to nf_token_transactions table and the like. +struct NFTTransactionsData +{ + ripple::uint256 tokenID; + std::uint32_t ledgerSequence; + std::uint32_t transactionIndex; + ripple::uint256 txHash; + + NFTTransactionsData( + ripple::uint256 const& tokenID, + ripple::TxMeta const& meta, + ripple::uint256 const& txHash) + : tokenID(tokenID) + , ledgerSequence(meta.getLgrSeq()) + , transactionIndex(meta.getIndex()) + , txHash(txHash) + { + } +}; + +/// Represents an NFT state at a particular ledger. Gets written to nf_tokens +/// table and the like. +struct NFTsData +{ + ripple::uint256 tokenID; + std::uint32_t ledgerSequence; + + // The transaction index is only stored because we want to store only the + // final state of an NFT per ledger. Since we pull this from transactions + // we keep track of which tx index created this so we can de-duplicate, as + // it is possible for one ledger to have multiple txs that change the + // state of the same NFT. + std::uint32_t transactionIndex; + ripple::AccountID owner; + bool isBurned; + + NFTsData( + ripple::uint256 const& tokenID, + ripple::AccountID const& owner, + ripple::TxMeta const& meta, + bool isBurned) + : tokenID(tokenID) + , ledgerSequence(meta.getLgrSeq()) + , transactionIndex(meta.getIndex()) + , owner(owner) + , isBurned(isBurned) + { + } +}; + template inline bool isOffer(T const& object) diff --git a/src/backend/PostgresBackend.cpp b/src/backend/PostgresBackend.cpp index 722b231a..be67e4c3 100644 --- a/src/backend/PostgresBackend.cpp +++ b/src/backend/PostgresBackend.cpp @@ -2,6 +2,7 @@ #include #include #include + namespace Backend { // Type alias for async completion handlers @@ -77,6 +78,12 @@ PostgresBackend::writeAccountTransactions( } } +void +PostgresBackend::writeNFTTransactions(std::vector&& data) +{ + throw std::runtime_error("Not implemented"); +} + void PostgresBackend::doWriteLedgerObject( std::string&& key, @@ -152,6 +159,12 @@ PostgresBackend::writeTransaction( << '\t' << "\\\\x" << ripple::strHex(metadata) << '\n'; } +void +PostgresBackend::writeNFTs(std::vector&& data) +{ + throw std::runtime_error("Not implemented"); +} + std::uint32_t checkResult(PgResult const& res, std::uint32_t const numFieldsExpected) { @@ -419,6 +432,15 @@ PostgresBackend::fetchAllTransactionHashesInLedger( return {}; } +std::optional +PostgresBackend::fetchNFT( + ripple::uint256 const& tokenID, + std::uint32_t const ledgerSequence, + boost::asio::yield_context& yield) const +{ + throw std::runtime_error("Not implemented"); +} + std::optional PostgresBackend::doFetchSuccessorKey( ripple::uint256 key, @@ -637,12 +659,25 @@ PostgresBackend::fetchLedgerDiff( return {}; } -AccountTransactions +// TODO this implementation and fetchAccountTransactions should be +// generalized +TransactionsAndCursor +PostgresBackend::fetchNFTTransactions( + ripple::uint256 const& tokenID, + std::uint32_t const limit, + bool forward, + std::optional const& cursor, + boost::asio::yield_context& yield) const +{ + throw std::runtime_error("Not implemented"); +} + +TransactionsAndCursor PostgresBackend::fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t const limit, bool forward, - std::optional const& cursor, + std::optional const& cursor, boost::asio::yield_context& yield) const { PgQuery pgQuery(pgPool_); diff --git a/src/backend/PostgresBackend.h b/src/backend/PostgresBackend.h index 3d08d441..64fbf5c5 100644 --- a/src/backend/PostgresBackend.h +++ b/src/backend/PostgresBackend.h @@ -62,6 +62,20 @@ public: std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override; + std::optional + fetchNFT( + ripple::uint256 const& tokenID, + std::uint32_t const ledgerSequence, + boost::asio::yield_context& yield) const override; + + TransactionsAndCursor + fetchNFTTransactions( + ripple::uint256 const& tokenID, + std::uint32_t const limit, + bool const forward, + std::optional const& cursorIn, + boost::asio::yield_context& yield) const override; + std::vector fetchLedgerDiff( std::uint32_t const ledgerSequence, @@ -87,12 +101,12 @@ public: std::uint32_t const sequence, boost::asio::yield_context& yield) const override; - AccountTransactions + TransactionsAndCursor fetchAccountTransactions( ripple::AccountID const& account, std::uint32_t const limit, bool forward, - std::optional const& cursor, + std::optional const& cursor, boost::asio::yield_context& yield) const override; void @@ -120,10 +134,16 @@ public: std::string&& transaction, std::string&& metadata) override; + void + writeNFTs(std::vector&& data) override; + void writeAccountTransactions( std::vector&& data) override; + void + writeNFTTransactions(std::vector&& data) override; + void open(bool readOnly) override; diff --git a/src/backend/Types.h b/src/backend/Types.h index 95b4bf96..b382e95e 100644 --- a/src/backend/Types.h +++ b/src/backend/Types.h @@ -1,6 +1,7 @@ #ifndef CLIO_TYPES_H_INCLUDED #define CLIO_TYPES_H_INCLUDED #include +#include #include #include #include @@ -46,16 +47,34 @@ struct TransactionAndMetadata } }; -struct AccountTransactionsCursor +struct TransactionsCursor { std::uint32_t ledgerSequence; std::uint32_t transactionIndex; }; -struct AccountTransactions +struct TransactionsAndCursor { std::vector txns; - std::optional cursor; + std::optional cursor; +}; + +struct NFT +{ + ripple::uint256 tokenID; + std::uint32_t ledgerSequence; + ripple::AccountID owner; + bool isBurned; + + // clearly two tokens are the same if they have the same ID, but this + // struct stores the state of a given token at a given ledger sequence, so + // we also need to compare with ledgerSequence + bool + operator==(NFT const& other) const + { + return tokenID == other.tokenID && + ledgerSequence == other.ledgerSequence; + } }; struct LedgerRange diff --git a/src/etl/ETLHelpers.h b/src/etl/ETLHelpers.h index ea9173cf..d5552c6d 100644 --- a/src/etl/ETLHelpers.h +++ b/src/etl/ETLHelpers.h @@ -174,4 +174,4 @@ getMarkers(size_t numMarkers) return markers; } -#endif // RIPPLE_APP_REPORTING_ETLHELPERS_H_INCLUDED \ No newline at end of file +#endif // RIPPLE_APP_REPORTING_ETLHELPERS_H_INCLUDED diff --git a/src/etl/NFTHelpers.cpp b/src/etl/NFTHelpers.cpp new file mode 100644 index 00000000..00eb58d3 --- /dev/null +++ b/src/etl/NFTHelpers.cpp @@ -0,0 +1,370 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +std::pair, std::optional> +getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx) +{ + // To find the minted token ID, we put all tokenIDs referenced in the + // metadata from prior to the tx application into one vector, then all + // tokenIDs referenced in the metadata from after the tx application into + // another, then find the one tokenID that was added by this tx + // application. + std::vector prevIDs; + std::vector finalIDs; + + // The owner is not necessarily the issuer, if using authorized minter + // flow. Determine owner from the ledger object ID of the NFTokenPages + // that were changed. + std::optional owner; + + for (ripple::STObject const& node : txMeta.getNodes()) + { + if (node.getFieldU16(ripple::sfLedgerEntryType) != + ripple::ltNFTOKEN_PAGE) + continue; + + if (!owner) + owner = ripple::AccountID::fromVoid( + node.getFieldH256(ripple::sfLedgerIndex).data()); + + if (node.getFName() == ripple::sfCreatedNode) + { + ripple::STArray const& toAddNFTs = + node.peekAtField(ripple::sfNewFields) + .downcast() + .getFieldArray(ripple::sfNFTokens); + std::transform( + toAddNFTs.begin(), + toAddNFTs.end(), + std::back_inserter(finalIDs), + [](ripple::STObject const& nft) { + return nft.getFieldH256(ripple::sfNFTokenID); + }); + } + // Else it's modified, as there should never be a deleted NFToken page + // as a result of a mint. + else + { + // When a mint results in splitting an existing page, + // it results in a created page and a modified node. Sometimes, + // the created node needs to be linked to a third page, resulting + // in modifying that third page's PreviousPageMin or NextPageMin + // field changing, but no NFTs within that page changing. In this + // case, there will be no previous NFTs and we need to skip. + // However, there will always be NFTs listed in the final fields, + // as rippled outputs all fields in final fields even if they were + // not changed. + ripple::STObject const& previousFields = + node.peekAtField(ripple::sfPreviousFields) + .downcast(); + if (!previousFields.isFieldPresent(ripple::sfNFTokens)) + continue; + + ripple::STArray const& toAddNFTs = + previousFields.getFieldArray(ripple::sfNFTokens); + std::transform( + toAddNFTs.begin(), + toAddNFTs.end(), + std::back_inserter(prevIDs), + [](ripple::STObject const& nft) { + return nft.getFieldH256(ripple::sfNFTokenID); + }); + + ripple::STArray const& toAddFinalNFTs = + node.peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldArray(ripple::sfNFTokens); + std::transform( + toAddFinalNFTs.begin(), + toAddFinalNFTs.end(), + std::back_inserter(finalIDs), + [](ripple::STObject const& nft) { + return nft.getFieldH256(ripple::sfNFTokenID); + }); + } + } + + std::sort(finalIDs.begin(), finalIDs.end()); + std::sort(prevIDs.begin(), prevIDs.end()); + std::vector tokenIDResult; + std::set_difference( + finalIDs.begin(), + finalIDs.end(), + prevIDs.begin(), + prevIDs.end(), + std::inserter(tokenIDResult, tokenIDResult.begin())); + if (tokenIDResult.size() == 1 && owner) + return { + {NFTTransactionsData( + tokenIDResult.front(), txMeta, sttx.getTransactionID())}, + NFTsData(tokenIDResult.front(), *owner, txMeta, false)}; + + std::stringstream msg; + msg << __func__ << " - unexpected NFTokenMint data in tx " + << sttx.getTransactionID(); + throw std::runtime_error(msg.str()); +} + +std::pair, std::optional> +getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx) +{ + ripple::uint256 const tokenID = sttx.getFieldH256(ripple::sfNFTokenID); + std::vector const txs = { + NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}; + + // Determine who owned the token when it was burned by finding an + // NFTokenPage that was deleted or modified that contains this + // tokenID. + for (ripple::STObject const& node : txMeta.getNodes()) + { + if (node.getFieldU16(ripple::sfLedgerEntryType) != + ripple::ltNFTOKEN_PAGE || + node.getFName() == ripple::sfCreatedNode) + continue; + + // NFT burn can result in an NFTokenPage being modified to no longer + // include the target, or an NFTokenPage being deleted. If this is + // modified, we want to look for the target in the fields prior to + // modification. If deleted, it's possible that the page was modified + // to remove the target NFT prior to the entire page being deleted. In + // this case, we need to look in the PreviousFields. Otherwise, the + // page was not modified prior to deleting and we need to look in the + // FinalFields. + std::optional prevNFTs; + + if (node.isFieldPresent(ripple::sfPreviousFields)) + { + ripple::STObject const& previousFields = + node.peekAtField(ripple::sfPreviousFields) + .downcast(); + if (previousFields.isFieldPresent(ripple::sfNFTokens)) + prevNFTs = previousFields.getFieldArray(ripple::sfNFTokens); + } + else if (!prevNFTs && node.getFName() == ripple::sfDeletedNode) + prevNFTs = node.peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldArray(ripple::sfNFTokens); + + if (!prevNFTs) + continue; + + auto const nft = std::find_if( + prevNFTs->begin(), + prevNFTs->end(), + [&tokenID](ripple::STObject const& candidate) { + return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID; + }); + if (nft != prevNFTs->end()) + return std::make_pair( + txs, + NFTsData( + tokenID, + ripple::AccountID::fromVoid( + node.getFieldH256(ripple::sfLedgerIndex).data()), + txMeta, + true)); + } + + std::stringstream msg; + msg << __func__ << " - could not determine owner at burntime for tx " + << sttx.getTransactionID(); + throw std::runtime_error(msg.str()); +} + +std::pair, std::optional> +getNFTokenAcceptOfferData( + ripple::TxMeta const& txMeta, + ripple::STTx const& sttx) +{ + // If we have the buy offer from this tx, we can determine the owner + // more easily by just looking at the owner of the accepted NFTokenOffer + // object. + if (sttx.isFieldPresent(ripple::sfNFTokenBuyOffer)) + { + auto const affectedBuyOffer = std::find_if( + txMeta.getNodes().begin(), + txMeta.getNodes().end(), + [&sttx](ripple::STObject const& node) { + return node.getFieldH256(ripple::sfLedgerIndex) == + sttx.getFieldH256(ripple::sfNFTokenBuyOffer); + }); + if (affectedBuyOffer == txMeta.getNodes().end()) + { + std::stringstream msg; + msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx " + << sttx.getTransactionID(); + throw std::runtime_error(msg.str()); + } + + ripple::uint256 const tokenID = + affectedBuyOffer->peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldH256(ripple::sfNFTokenID); + + ripple::AccountID const owner = + affectedBuyOffer->peekAtField(ripple::sfFinalFields) + .downcast() + .getAccountID(ripple::sfOwner); + return { + {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, + NFTsData(tokenID, owner, txMeta, false)}; + } + + // Otherwise we have to infer the new owner from the affected nodes. + auto const affectedSellOffer = std::find_if( + txMeta.getNodes().begin(), + txMeta.getNodes().end(), + [&sttx](ripple::STObject const& node) { + return node.getFieldH256(ripple::sfLedgerIndex) == + sttx.getFieldH256(ripple::sfNFTokenSellOffer); + }); + if (affectedSellOffer == txMeta.getNodes().end()) + { + std::stringstream msg; + msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx " + << sttx.getTransactionID(); + throw std::runtime_error(msg.str()); + } + + ripple::uint256 const tokenID = + affectedSellOffer->peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldH256(ripple::sfNFTokenID); + + ripple::AccountID const seller = + affectedSellOffer->peekAtField(ripple::sfFinalFields) + .downcast() + .getAccountID(ripple::sfOwner); + + for (ripple::STObject const& node : txMeta.getNodes()) + { + if (node.getFieldU16(ripple::sfLedgerEntryType) != + ripple::ltNFTOKEN_PAGE || + node.getFName() == ripple::sfDeletedNode) + continue; + + ripple::AccountID const nodeOwner = ripple::AccountID::fromVoid( + node.getFieldH256(ripple::sfLedgerIndex).data()); + if (nodeOwner == seller) + continue; + + ripple::STArray const& nfts = [&node] { + if (node.getFName() == ripple::sfCreatedNode) + return node.peekAtField(ripple::sfNewFields) + .downcast() + .getFieldArray(ripple::sfNFTokens); + return node.peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldArray(ripple::sfNFTokens); + }(); + + auto const nft = std::find_if( + nfts.begin(), + nfts.end(), + [&tokenID](ripple::STObject const& candidate) { + return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID; + }); + if (nft != nfts.end()) + return { + {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, + NFTsData(tokenID, nodeOwner, txMeta, false)}; + } + + std::stringstream msg; + msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx " + << sttx.getTransactionID(); + throw std::runtime_error(msg.str()); +} + +// This is the only transaction where there can be more than 1 element in +// the returned vector, because you can cancel multiple offers in one +// transaction using this feature. This transaction also never returns an +// NFTsData because it does not change the state of an NFT itself. +std::pair, std::optional> +getNFTokenCancelOfferData( + ripple::TxMeta const& txMeta, + ripple::STTx const& sttx) +{ + std::vector txs; + for (ripple::STObject const& node : txMeta.getNodes()) + { + if (node.getFieldU16(ripple::sfLedgerEntryType) != + ripple::ltNFTOKEN_OFFER) + continue; + + ripple::uint256 const tokenID = node.peekAtField(ripple::sfFinalFields) + .downcast() + .getFieldH256(ripple::sfNFTokenID); + txs.emplace_back(tokenID, txMeta, sttx.getTransactionID()); + } + + // Deduplicate any transactions based on tokenID/txIdx combo. Can't just + // use txIdx because in this case one tx can cancel offers for several + // NFTs. + std::sort( + txs.begin(), + txs.end(), + [](NFTTransactionsData const& a, NFTTransactionsData const& b) { + return a.tokenID < b.tokenID && + a.transactionIndex < b.transactionIndex; + }); + auto last = std::unique( + txs.begin(), + txs.end(), + [](NFTTransactionsData const& a, NFTTransactionsData const& b) { + return a.tokenID == b.tokenID && + a.transactionIndex == b.transactionIndex; + }); + txs.erase(last, txs.end()); + return {txs, {}}; +} + +// This transaction never returns an NFTokensData because it does not +// change the state of an NFT itself. +std::pair, std::optional> +getNFTokenCreateOfferData( + ripple::TxMeta const& txMeta, + ripple::STTx const& sttx) +{ + return { + {NFTTransactionsData( + sttx.getFieldH256(ripple::sfNFTokenID), + txMeta, + sttx.getTransactionID())}, + {}}; +} + +std::pair, std::optional> +getNFTData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx) +{ + if (txMeta.getResultTER() != ripple::tesSUCCESS) + return {{}, {}}; + + switch (sttx.getTxnType()) + { + case ripple::TxType::ttNFTOKEN_MINT: + return getNFTokenMintData(txMeta, sttx); + + case ripple::TxType::ttNFTOKEN_BURN: + return getNFTokenBurnData(txMeta, sttx); + + case ripple::TxType::ttNFTOKEN_ACCEPT_OFFER: + return getNFTokenAcceptOfferData(txMeta, sttx); + + case ripple::TxType::ttNFTOKEN_CANCEL_OFFER: + return getNFTokenCancelOfferData(txMeta, sttx); + + case ripple::TxType::ttNFTOKEN_CREATE_OFFER: + return getNFTokenCreateOfferData(txMeta, sttx); + + default: + return {{}, {}}; + } +} diff --git a/src/etl/ReportingETL.cpp b/src/etl/ReportingETL.cpp index dd0bd248..18116f82 100644 --- a/src/etl/ReportingETL.cpp +++ b/src/etl/ReportingETL.cpp @@ -28,12 +28,13 @@ toString(ripple::LedgerInfo const& info) } } // namespace detail -std::vector +FormattedTransactionsData ReportingETL::insertTransactions( ripple::LedgerInfo const& ledger, org::xrpl::rpc::v1::GetLedgerResponse& data) { - std::vector accountTxData; + FormattedTransactionsData result; + for (auto& txn : *(data.mutable_transactions_list()->mutable_transactions())) { @@ -42,21 +43,22 @@ ReportingETL::insertTransactions( ripple::SerialIter it{raw->data(), raw->size()}; ripple::STTx sttx{it}; - auto txSerializer = - std::make_shared(sttx.getSerializer()); - - ripple::TxMeta txMeta{ - sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; - - auto metaSerializer = std::make_shared( - txMeta.getAsObject().getSerializer()); - BOOST_LOG_TRIVIAL(trace) << __func__ << " : " << "Inserting transaction = " << sttx.getTransactionID(); + ripple::TxMeta txMeta{ + sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; + + auto const [nftTxs, maybeNFT] = getNFTData(txMeta, sttx); + result.nfTokenTxData.insert( + result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end()); + if (maybeNFT) + result.nfTokensData.push_back(*maybeNFT); + auto journal = ripple::debugLog(); - accountTxData.emplace_back(txMeta, sttx.getTransactionID(), journal); + result.accountTxData.emplace_back( + txMeta, sttx.getTransactionID(), journal); std::string keyStr{(const char*)sttx.getTransactionID().data(), 32}; backend_->writeTransaction( std::move(keyStr), @@ -65,7 +67,27 @@ ReportingETL::insertTransactions( std::move(*raw), std::move(*txn.mutable_metadata_blob())); } - return accountTxData; + + // Remove all but the last NFTsData for each id. unique removes all + // but the first of a group, so we want to reverse sort by transaction + // index + std::sort( + result.nfTokensData.begin(), + result.nfTokensData.end(), + [](NFTsData const& a, NFTsData const& b) { + return a.tokenID > b.tokenID && + a.transactionIndex > b.transactionIndex; + }); + // Now we can unique the NFTs by tokenID. + auto last = std::unique( + result.nfTokensData.begin(), + result.nfTokensData.end(), + [](NFTsData const& a, NFTsData const& b) { + return a.tokenID == b.tokenID; + }); + result.nfTokensData.erase(last, result.nfTokensData.end()); + + return result; } std::optional @@ -106,7 +128,7 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence) lgrInfo, std::move(*ledgerData->mutable_ledger_header())); BOOST_LOG_TRIVIAL(debug) << __func__ << " wrote ledger"; - std::vector accountTxData = + FormattedTransactionsData insertTxResult = insertTransactions(lgrInfo, *ledgerData); BOOST_LOG_TRIVIAL(debug) << __func__ << " inserted txns"; @@ -119,8 +141,12 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence) BOOST_LOG_TRIVIAL(debug) << __func__ << " loaded initial ledger"; if (!stopping_) - backend_->writeAccountTransactions(std::move(accountTxData)); - + { + backend_->writeAccountTransactions( + std::move(insertTxResult.accountTxData)); + backend_->writeNFTs(std::move(insertTxResult.nfTokensData)); + backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData)); + } backend_->finishWrites(startingSequence); auto end = std::chrono::system_clock::now(); @@ -511,15 +537,15 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData) << __func__ << " : " << "Inserted/modified/deleted all objects. Number of objects = " << rawData.ledger_objects().objects_size(); - std::vector accountTxData{ - insertTransactions(lgrInfo, rawData)}; + FormattedTransactionsData insertTxResult = + insertTransactions(lgrInfo, rawData); BOOST_LOG_TRIVIAL(debug) << __func__ << " : " << "Inserted all transactions. Number of transactions = " << rawData.transactions_list().transactions_size(); - - backend_->writeAccountTransactions(std::move(accountTxData)); - + backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData)); + backend_->writeNFTs(std::move(insertTxResult.nfTokensData)); + backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData)); BOOST_LOG_TRIVIAL(debug) << __func__ << " : " << "wrote account_tx"; auto start = std::chrono::system_clock::now(); diff --git a/src/etl/ReportingETL.h b/src/etl/ReportingETL.h index f21439a9..ebc40845 100644 --- a/src/etl/ReportingETL.h +++ b/src/etl/ReportingETL.h @@ -19,7 +19,22 @@ #include +/** + * Helper function for the ReportingETL, implemented in NFTHelpers.cpp, to + * pull to-write data out of a transaction that relates to NFTs. + */ +std::pair, std::optional> +getNFTData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx); + struct AccountTransactionsData; +struct NFTTransactionsData; +struct NFTsData; +struct FormattedTransactionsData +{ + std::vector accountTxData; + std::vector nfTokenTxData; + std::vector nfTokensData; +}; class SubscriptionManager; /** @@ -221,14 +236,16 @@ private: std::optional fetchLedgerDataAndDiff(uint32_t sequence); - /// Insert all of the extracted transactions into the ledger + /// Insert all of the extracted transactions into the ledger, returning + /// transactions related to accounts, transactions related to NFTs, and + /// NFTs themselves for later processsing. /// @param ledger ledger to insert transactions into /// @param data data extracted from an ETL source /// @return struct that contains the neccessary info to write to the - /// transctions and account_transactions tables in Postgres (mostly - /// transaction hashes, corresponding nodestore hashes and affected + /// account_transactions/account_tx and nft_token_transactions tables + /// (mostly transaction hashes, corresponding nodestore hashes and affected /// accounts) - std::vector + FormattedTransactionsData insertTransactions( ripple::LedgerInfo const& ledger, org::xrpl::rpc::v1::GetLedgerResponse& data); diff --git a/src/rpc/Handlers.h b/src/rpc/Handlers.h index 315be86e..6207243e 100644 --- a/src/rpc/Handlers.h +++ b/src/rpc/Handlers.h @@ -55,6 +55,9 @@ doNFTBuyOffers(Context const& context); Result doNFTSellOffers(Context const& context); +Result +doNFTInfo(Context const& context); + // ledger methods Result doLedger(Context const& context); diff --git a/src/rpc/RPC.cpp b/src/rpc/RPC.cpp index f0227f6b..99ce33b2 100644 --- a/src/rpc/RPC.cpp +++ b/src/rpc/RPC.cpp @@ -191,6 +191,7 @@ static HandlerTable handlerTable{ {"ledger", &doLedger, {}}, {"ledger_data", &doLedgerData, LimitRange{1, 100, 2048}}, {"nft_buy_offers", &doNFTBuyOffers, LimitRange{1, 50, 100}}, + {"nft_info", &doNFTInfo}, {"nft_sell_offers", &doNFTSellOffers, LimitRange{1, 50, 100}}, {"ledger_entry", &doLedgerEntry, {}}, {"ledger_range", &doLedgerRange, {}}, diff --git a/src/rpc/handlers/AccountTx.cpp b/src/rpc/handlers/AccountTx.cpp index 072df070..ff1ef624 100644 --- a/src/rpc/handlers/AccountTx.cpp +++ b/src/rpc/handlers/AccountTx.cpp @@ -19,7 +19,7 @@ doAccountTx(Context const& context) bool const binary = getBool(request, JS(binary), false); bool const forward = getBool(request, JS(forward), false); - std::optional cursor; + std::optional cursor; if (request.contains(JS(marker))) { diff --git a/src/rpc/handlers/NFTInfo.cpp b/src/rpc/handlers/NFTInfo.cpp new file mode 100644 index 00000000..f185c9e0 --- /dev/null +++ b/src/rpc/handlers/NFTInfo.cpp @@ -0,0 +1,146 @@ +#include +#include +#include + +#include +#include + +// { +// nft_id: +// ledger_hash: +// ledger_index: +// } + +namespace RPC { + +std::variant +getURI(Backend::NFT const& dbResponse, Context const& context) +{ + // Fetch URI from ledger + // The correct page will be > bookmark and <= last. We need to calculate + // the first possible page however, since bookmark is not guaranteed to + // exist. + auto const bookmark = ripple::keylet::nftpage( + ripple::keylet::nftpage_min(dbResponse.owner), dbResponse.tokenID); + auto const last = ripple::keylet::nftpage_max(dbResponse.owner); + + ripple::uint256 nextKey = last.key; + std::optional sle; + + // when this loop terminates, `sle` will contain the correct page for + // this NFT. + // + // 1) We start at the last NFTokenPage, which is guaranteed to exist, + // grab the object from the DB and deserialize it. + // + // 2) If that NFTokenPage has a PreviousPageMin value and the + // PreviousPageMin value is > bookmark, restart loop. Otherwise + // terminate and use the `sle` from this iteration. + do + { + auto const blob = context.backend->fetchLedgerObject( + ripple::Keylet(ripple::ltNFTOKEN_PAGE, nextKey).key, + dbResponse.ledgerSequence, + context.yield); + + if (!blob || blob->size() == 0) + return Status{ + Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"}; + + sle = ripple::STLedgerEntry( + ripple::SerialIter{blob->data(), blob->size()}, nextKey); + + if (sle->isFieldPresent(ripple::sfPreviousPageMin)) + nextKey = sle->getFieldH256(ripple::sfPreviousPageMin); + + } while (sle && sle->key() != nextKey && nextKey > bookmark.key); + + if (!sle) + return Status{ + Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"}; + + auto const nfts = sle->getFieldArray(ripple::sfNFTokens); + auto const nft = std::find_if( + nfts.begin(), + nfts.end(), + [&dbResponse](ripple::STObject const& candidate) { + return candidate.getFieldH256(ripple::sfNFTokenID) == + dbResponse.tokenID; + }); + + if (nft == nfts.end()) + return Status{ + Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"}; + + ripple::Blob const uriField = nft->getFieldVL(ripple::sfURI); + + // NOTE this cannot use a ternary or value_or because then the + // expression's type is unclear. We want to explicitly set the `uri` + // field to null when not present to avoid any confusion. + if (std::string const uri = std::string(uriField.begin(), uriField.end()); + uri.size() > 0) + return uri; + return std::monostate{}; +} + +Result +doNFTInfo(Context const& context) +{ + auto request = context.params; + boost::json::object response = {}; + + if (!request.contains("nft_id")) + return Status{Error::rpcINVALID_PARAMS, "Missing nft_id"}; + + auto const& jsonTokenID = request.at("nft_id"); + if (!jsonTokenID.is_string()) + return Status{Error::rpcINVALID_PARAMS, "nft_id is not a string"}; + + ripple::uint256 tokenID; + if (!tokenID.parseHex(jsonTokenID.as_string().c_str())) + return Status{Error::rpcINVALID_PARAMS, "Malformed nft_id"}; + + // We only need to fetch the ledger header because the ledger hash is + // supposed to be included in the response. The ledger sequence is specified + // in the request + auto v = ledgerInfoFromRequest(context); + if (auto status = std::get_if(&v)) + return *status; + ripple::LedgerInfo lgrInfo = std::get(v); + + std::optional dbResponse = + context.backend->fetchNFT(tokenID, lgrInfo.seq, context.yield); + if (!dbResponse) + return Status{Error::rpcOBJECT_NOT_FOUND, "NFT not found"}; + + response["nft_id"] = ripple::strHex(dbResponse->tokenID); + response["ledger_index"] = dbResponse->ledgerSequence; + response["owner"] = ripple::toBase58(dbResponse->owner); + response["is_burned"] = dbResponse->isBurned; + + response["flags"] = ripple::nft::getFlags(dbResponse->tokenID); + response["transfer_fee"] = ripple::nft::getTransferFee(dbResponse->tokenID); + response["issuer"] = + ripple::toBase58(ripple::nft::getIssuer(dbResponse->tokenID)); + response["nft_taxon"] = + ripple::nft::toUInt32(ripple::nft::getTaxon(dbResponse->tokenID)); + response["nft_sequence"] = ripple::nft::getSerial(dbResponse->tokenID); + + if (!dbResponse->isBurned) + { + auto const maybeURI = getURI(*dbResponse, context); + // An error occurred + if (Status const* status = std::get_if(&maybeURI)) + return *status; + // A URI was found + if (std::string const* uri = std::get_if(&maybeURI)) + response["uri"] = *uri; + // A URI was not found, explicitly set to null + else + response["uri"] = nullptr; + } + + return response; +} + +} // namespace RPC diff --git a/unittests/main.cpp b/unittests/main.cpp index 5761709e..70f0e755 100644 --- a/unittests/main.cpp +++ b/unittests/main.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -296,6 +297,122 @@ TEST(BackendTest, Basic) "E0311EB450B6177F969B94DBDDA83E99B7A0576ACD9079573876F16C0C" "004F06"; + // An NFTokenMint tx + std::string nftTxnHex = + "1200192200000008240011CC9B201B001F71D6202A0000000168400000" + "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" + "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" + "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" + "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" + "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" + "DE3538F248662FC73C"; + + std::string nftTxnMeta = + "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" + "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" + "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" + "C200000028751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "9808B6B90000001D751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" + "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" + "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" + "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" + "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" + "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" + "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" + "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" + "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" + "00001F751868747470733A2F2F677265677765697362726F642E636F6D" + "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" + "5DAE00000014751868747470733A2F2F677265677765697362726F642E" + "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" + "3CCE1462A500000009751868747470733A2F2F67726567776569736272" + "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" + "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" + "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" + "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" + "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" + "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" + "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" + "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" + "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" + "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" + "00000029751868747470733A2F2F677265677765697362726F642E636F" + "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" + "EE87B80000001E751868747470733A2F2F677265677765697362726F64" + "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" + "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" + "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" + "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" + "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" + "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" + "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" + "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" + "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" + "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" + "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" + "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" + "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" + "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" + "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" + "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" + "B100000015751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" + "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" + "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" + "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" + "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" + "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" + "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" + "2FC73CE1E1F1031000"; + std::string nftTxnHashHex = + "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757" + "E9284E"; + ripple::uint256 nftID; + EXPECT_TRUE( + nftID.parseHex("000800006203F49C21D5D6E022CB16DE3538F248662" + "FC73CEF7FF5C60000002C")); + std::string metaBlob = hexStringToBinaryString(metaHex); std::string txnBlob = hexStringToBinaryString(txnHex); std::string hashBlob = hexStringToBinaryString(hashHex); @@ -304,6 +421,10 @@ TEST(BackendTest, Basic) hexStringToBinaryString(accountIndexHex); std::vector affectedAccounts; + std::string nftTxnBlob = hexStringToBinaryString(nftTxnHex); + std::string nftTxnMetaBlob = + hexStringToBinaryString(nftTxnMeta); + { backend->startWrites(); lgrInfoNext.seq = lgrInfoNext.seq + 1; @@ -322,9 +443,29 @@ TEST(BackendTest, Basic) { affectedAccounts.push_back(a); } - std::vector accountTxData; accountTxData.emplace_back(txMeta, hash256, journal); + + ripple::uint256 nftHash256; + EXPECT_TRUE(nftHash256.parseHex(nftTxnHashHex)); + ripple::TxMeta nftTxMeta{ + nftHash256, lgrInfoNext.seq, nftTxnMetaBlob}; + ripple::SerialIter it{nftTxnBlob.data(), nftTxnBlob.size()}; + ripple::STTx sttx{it}; + auto const [parsedNFTTxsRef, parsedNFT] = + getNFTData(nftTxMeta, sttx); + // need to copy the nft txns so we can std::move later + std::vector parsedNFTTxs; + parsedNFTTxs.insert( + parsedNFTTxs.end(), + parsedNFTTxsRef.begin(), + parsedNFTTxsRef.end()); + EXPECT_EQ(parsedNFTTxs.size(), 1); + EXPECT_TRUE(parsedNFT.has_value()); + EXPECT_EQ(parsedNFT->tokenID, nftID); + std::vector nftData; + nftData.push_back(*parsedNFT); + backend->writeLedger( lgrInfoNext, std::move(ledgerInfoToBinaryString(lgrInfoNext))); @@ -335,6 +476,26 @@ TEST(BackendTest, Basic) std::move(std::string{txnBlob}), std::move(std::string{metaBlob})); backend->writeAccountTransactions(std::move(accountTxData)); + + // NFT writing not yet implemented for pg + if (config == cassandraConfig) + { + backend->writeNFTs(std::move(nftData)); + backend->writeNFTTransactions(std::move(parsedNFTTxs)); + } + else + { + EXPECT_THROW( + { backend->writeNFTs(std::move(nftData)); }, + std::runtime_error); + EXPECT_THROW( + { + backend->writeNFTTransactions( + std::move(parsedNFTTxs)); + }, + std::runtime_error); + } + backend->writeLedgerObject( std::move(std::string{accountIndexBlob}), lgrInfoNext.seq, @@ -384,6 +545,34 @@ TEST(BackendTest, Basic) EXPECT_FALSE(cursor); } + // NFT fetching not yet implemented for pg + if (config == cassandraConfig) + { + auto nft = + backend->fetchNFT(nftID, lgrInfoNext.seq, yield); + EXPECT_TRUE(nft.has_value()); + auto [nftTxns, cursor] = backend->fetchNFTTransactions( + nftID, 100, true, {}, yield); + EXPECT_EQ(nftTxns.size(), 1); + EXPECT_EQ(nftTxns[0], nftTxns[0]); + EXPECT_FALSE(cursor); + } + else + { + EXPECT_THROW( + { + backend->fetchNFT( + nftID, lgrInfoNext.seq, yield); + }, + std::runtime_error); + EXPECT_THROW( + { + backend->fetchNFTTransactions( + nftID, 100, true, {}, yield); + }, + std::runtime_error); + } + ripple::uint256 key256; EXPECT_TRUE(key256.parseHex(accountIndexHex)); auto obj = backend->fetchLedgerObject( @@ -729,8 +918,7 @@ TEST(BackendTest, Basic) for (auto [account, data] : accountTx) { std::vector retData; - std::optional - cursor; + std::optional cursor; do { uint32_t limit = 10;