diff --git a/examples/js_client/hp-client-lib.js b/examples/js_client/hp-client-lib.js index 36ff0227..527dbcb5 100644 --- a/examples/js_client/hp-client-lib.js +++ b/examples/js_client/hp-client-lib.js @@ -583,7 +583,7 @@ const resolver = ledgerQueryResolvers[m.reply_for]; if (resolver) { const results = m.results.map(r => { - return { + const result = { seqNo: r.seq_no, timestamp: r.timestamp, hash: r.hash, @@ -594,6 +594,11 @@ inputHash: r.input_hash, outputHash: r.output_hash } + if (r.raw_inputs) + result.rawInputs = r.raw_inputs; + if (r.raw_outputs) + result.rawOutputs = r.raw_outputs; + return result; }); if (resolver.type == "seq_no") resolver.resolver(results.length > 0 ? results[0] : null) // Return as a single object rather than an array. diff --git a/src/ledger/ledger.cpp b/src/ledger/ledger.cpp index 002a24c2..9fef9fef 100644 --- a/src/ledger/ledger.cpp +++ b/src/ledger/ledger.cpp @@ -507,7 +507,7 @@ namespace ledger { outputs.push_back(output.message); } - blob.outputs.emplace(user_output.userpubkey, outputs); + blob.outputs.emplace(user_output.userpubkey, std::move(outputs)); } flatbuffers::FlatBufferBuilder builder(1024); diff --git a/src/ledger/ledger.hpp b/src/ledger/ledger.hpp index 68f4c073..33e953ec 100644 --- a/src/ledger/ledger.hpp +++ b/src/ledger/ledger.hpp @@ -63,13 +63,6 @@ namespace ledger } }; - struct ledger_blob - { - util::h32 ledger_hash; - std::map> inputs; - std::map> outputs; - }; - extern ledger_context ctx; extern ledger::ledger_mount ledger_fs; // Global ledger file system instance. extern ledger::ledger_sync ledger_sync_worker; // Global ledger file system sync instance. diff --git a/src/ledger/ledger_common.hpp b/src/ledger/ledger_common.hpp index 2ec32126..390bf329 100644 --- a/src/ledger/ledger_common.hpp +++ b/src/ledger/ledger_common.hpp @@ -2,6 +2,7 @@ #define _HP_LEDGER_LEDGER_COMMON_ #include "../pchheader.hpp" +#include "../util/h32.hpp" namespace ledger { @@ -27,6 +28,17 @@ namespace ledger std::string output_hash; }; + /** + * Struct to hold ledger raw inputs and outputs. + * This is used with flatbuffers to persist to disk. + */ + struct ledger_blob + { + util::h32 ledger_hash; + std::map> inputs; + std::map> outputs; + }; + // Holds the global genesis ledger. extern ledger_record genesis; } diff --git a/src/ledger/ledger_query.cpp b/src/ledger/ledger_query.cpp index 42e97d59..64414cfc 100644 --- a/src/ledger/ledger_query.cpp +++ b/src/ledger/ledger_query.cpp @@ -2,6 +2,7 @@ #include "ledger_common.hpp" #include "ledger.hpp" #include "sqlite.hpp" +#include "../msg/fbuf/ledger_helpers.hpp" namespace ledger::query { @@ -32,12 +33,16 @@ namespace ledger::query if (q.index() == 0) // Filter by seq no. { ledger_record ledger; - const int seq_no_res = get_ledger_by_seq_no(ledger, std::get(q), fs_sess_name); + const seq_no_query &seq_q = std::get(q); + const int seq_no_res = get_ledger_by_seq_no(ledger, seq_q, fs_sess_name); if (seq_no_res != -1) { if (seq_no_res == 1) // Ledger found. records.push_back(query_result_record{std::move(ledger)}); - res = std::move(records); + + // Fill raw input/output data into results. + if (fill_blob_data(records, seq_q.raw_inputs, seq_q.raw_outputs, fs_sess_name) != -1) + res = std::move(records); } } @@ -45,6 +50,68 @@ namespace ledger::query return res; } + /** + * Fills in the raw input/output blob data to the specified ledger query result records. + * @param records List of query results to fill in. + * @param raw_inputs Whether raw inputs must be filled. + * @param raw_outputs Whether raw outputs must be filled. + * @param fs_sess_name The ledger hosting fs session name. + */ + int fill_blob_data(std::vector &records, const bool raw_inputs, const bool raw_outputs, const std::string &fs_sess_name) + { + // If blob data is not requested to be filled, the relevant field (inputs or outputs) in each result will contain NULL. + // If blob data is requested to be filled, then the relevant field will contain the map of blobs or empty map. + + if (!raw_inputs && !raw_outputs) + return 0; // Nothing to fill. + + for (query_result_record &r : records) + { + // Populate with empty map if inputs/outputs requested. + if (raw_inputs) + r.raw_inputs = blob_map(); + if (raw_outputs) + r.raw_outputs = blob_map(); + + if (r.ledger.seq_no == 0) + return 0; // Nothing to fill for GENESIS ledger. + + const uint64_t shard_seq_no = (r.ledger.seq_no - 1) / ledger::BLOB_SHARD_SIZE; + const std::string file_vpath = std::string(ledger::BLOB_DIR) + "/" + std::to_string(shard_seq_no) + "/" + util::to_hex(r.ledger.ledger_hash) + ".blob"; + const std::string file_path = ledger::ledger_fs.physical_path(fs_sess_name, file_vpath); + std::string blob_msg; + const int fd = open(file_path.data(), O_RDONLY); + + // If file does not exist, skip this leadger. (it means there are no input/output data for this leadger) + if (fd == -1 && errno == ENOENT) + continue; + + if (fd != -1 && util::read_from_fd(fd, blob_msg, util::HP_VERSION_HEADER_SIZE) > 0) + { + ledger_blob raw_data; + if (msg::fbuf::ledgermsg::create_ledger_blob_from_msg(raw_data, blob_msg, raw_inputs, raw_outputs) != -1) + { + if (raw_inputs) + raw_data.inputs.swap(*r.raw_inputs); + + if (raw_outputs) + raw_data.outputs.swap(*r.raw_outputs); + + close(fd); + continue; + } + } + + if (fd != -1) + close(fd); + + // Reaching this point means loop has encountered an error. + return -1; + } + + return 0; + } + /** * Get the ledger record by seq no. * @param ledger Ledger structure to populate (if match found)). @@ -63,7 +130,7 @@ namespace ledger::query // Construct shard path based on provided ledger seq no. const uint64_t shard_seq_no = (q.seq_no - 1) / ledger::PRIMARY_SHARD_SIZE; - const std::string db_vpath = std::string(ledger::PRIMARY_DIR).append("/").append(std::to_string(shard_seq_no)).append("/").append(ledger::DATABASE); + const std::string db_vpath = std::string(ledger::PRIMARY_DIR) + "/" + std::to_string(shard_seq_no) + "/" + ledger::DATABASE; const std::string dbpath = ledger::ledger_fs.physical_path(fs_sess_name, db_vpath); if (!util::is_file_exists(dbpath)) diff --git a/src/ledger/ledger_query.hpp b/src/ledger/ledger_query.hpp index 4c6c10f9..0f0ab2e0 100644 --- a/src/ledger/ledger_query.hpp +++ b/src/ledger/ledger_query.hpp @@ -16,18 +16,26 @@ namespace ledger::query bool raw_outputs = false; }; + typedef std::map> blob_map; + struct query_result_record { ledger::ledger_record ledger; - // TODO: - // RawInputs field. - // RawOutputs field. + std::optional raw_inputs; + std::optional raw_outputs; + }; + + struct user_buffer_collection + { + std::string pubkey; // Binary user pubkey. + std::vector buffers; // List of binary data buffers. }; typedef std::variant query_request; typedef std::variant> query_result; const query_result execute(std::string_view user_pubkey, const query_request &q); + int fill_blob_data(std::vector &records, const bool raw_inputs, const bool raw_outputs, const std::string &fs_sess_name); int get_ledger_by_seq_no(ledger_record &ledger, const seq_no_query &q, const std::string &fs_sess_name); } diff --git a/src/msg/bson/usrmsg_bson.cpp b/src/msg/bson/usrmsg_bson.cpp index 3e75828b..b41fc9dd 100644 --- a/src/msg/bson/usrmsg_bson.cpp +++ b/src/msg/bson/usrmsg_bson.cpp @@ -224,7 +224,7 @@ namespace msg::usrmsg::bson encoder.key(msg::usrmsg::FLD_RESULTS); encoder.begin_array(); - populate_query_results(encoder, std::get>(result)); + populate_ledger_query_results(encoder, std::get>(result)); encoder.end_array(); encoder.end_object(); encoder.flush(); @@ -419,10 +419,10 @@ namespace msg::usrmsg::bson bool raw_outputs = false; for (auto &val : d[msg::usrmsg::FLD_INCLUDE].array_range()) { - if (val == msg::usrmsg::QUERY_INCLUDE_RAW_INPUTS) + if (val == msg::usrmsg::FLD_RAW_INPUTS) raw_inputs = true; - else if (val == msg::usrmsg::QUERY_INCLUDE_RAW_OUTPUTS) - raw_outputs = false; + else if (val == msg::usrmsg::FLD_RAW_OUTPUTS) + raw_outputs = true; } auto ¶ms_field = d[msg::usrmsg::FLD_PARAMS]; @@ -464,7 +464,7 @@ namespace msg::usrmsg::bson } } - void populate_query_results(jsoncons::bson::bson_bytes_encoder &encoder, const std::vector &results) + void populate_ledger_query_results(jsoncons::bson::bson_bytes_encoder &encoder, const std::vector &results) { for (const ledger::query::query_result_record &r : results) { @@ -487,8 +487,44 @@ namespace msg::usrmsg::bson encoder.byte_string_value(r.ledger.input_hash); encoder.key(msg::usrmsg::FLD_OUTPUT_HASH); encoder.byte_string_value(r.ledger.output_hash); + + // If raw inputs or outputs is not requested, we don't include that field at all in the response. + // Otherwise the field will always contain an array (empty array if no data). + + if (r.raw_inputs) + { + encoder.key(msg::usrmsg::FLD_RAW_INPUTS); + populate_ledger_blob_map(encoder, *r.raw_inputs); + } + + if (r.raw_outputs) + { + encoder.key(msg::usrmsg::FLD_RAW_OUTPUTS); + populate_ledger_blob_map(encoder, *r.raw_outputs); + } + encoder.end_object(); } } + void populate_ledger_blob_map(jsoncons::bson::bson_bytes_encoder &encoder, const ledger::query::blob_map &blob_map) + { + encoder.begin_array(); + for (const auto &[pubkey, blobs] : blob_map) + { + encoder.begin_object(); + + encoder.key(msg::usrmsg::FLD_PUBKEY); + encoder.byte_string_value(pubkey); + encoder.key(msg::usrmsg::FLD_BLOBS); + encoder.begin_array(); + for (const std::string &blob : blobs) + encoder.byte_string_value(blob); + encoder.end_array(); + + encoder.end_object(); + } + encoder.end_array(); + } + } // namespace msg::usrmsg::bson \ No newline at end of file diff --git a/src/msg/bson/usrmsg_bson.hpp b/src/msg/bson/usrmsg_bson.hpp index ef460de0..89e53a9b 100644 --- a/src/msg/bson/usrmsg_bson.hpp +++ b/src/msg/bson/usrmsg_bson.hpp @@ -43,7 +43,9 @@ namespace msg::usrmsg::bson void populate_output_hash_array(jsoncons::bson::bson_bytes_encoder &encoder, const util::merkle_hash_node &node); - void populate_query_results(jsoncons::bson::bson_bytes_encoder &encoder, const std::vector &results); + void populate_ledger_query_results(jsoncons::bson::bson_bytes_encoder &encoder, const std::vector &results); + + void populate_ledger_blob_map(jsoncons::bson::bson_bytes_encoder &encoder, const ledger::query::blob_map &blob_map); } // namespace msg::usrmsg::bson diff --git a/src/msg/fbuf/ledger_helpers.cpp b/src/msg/fbuf/ledger_helpers.cpp index fc938e8c..d7022639 100644 --- a/src/msg/fbuf/ledger_helpers.cpp +++ b/src/msg/fbuf/ledger_helpers.cpp @@ -54,4 +54,49 @@ namespace msg::fbuf::ledgermsg builder.Finish(blob); // Finished building message content to get serialised content. } + + const int create_ledger_blob_from_msg(ledger::ledger_blob &blob_data, const std::string &msg, const bool read_inputs, const bool read_outputs) + { + // Verify ledger blob using flatbuffer verifier + flatbuffers::Verifier verifier((uint8_t *)msg.data(), msg.size(), 16, 100); + if (!VerifyLedgerBlobBuffer(verifier)) + { + LOG_ERROR << "Ledger blob flatbuffer verification failed."; + return -1; + } + + const auto ledger_msg = ledgermsg::GetLedgerBlob(msg.data()); + blob_data.ledger_hash = flatbuf_bytes_to_hash(ledger_msg->ledger_hash()); + + if (read_inputs) + { + std::vector inputs; + for (const auto collection : *ledger_msg->raw_inputs()) + { + for (const auto input_msg : *collection->inputs()) + { + inputs.push_back(std::string(flatbuf_bytes_to_sv(input_msg->input()))); + } + + blob_data.inputs.emplace(std::string(flatbuf_bytes_to_sv(collection->pubkey())), std::move(inputs)); + } + } + + if (read_outputs) + { + std::vector outputs; + for (const auto collection : *ledger_msg->raw_outputs()) + { + for (const auto output_msg : *collection->outputs()) + { + outputs.push_back(std::string(flatbuf_bytes_to_sv(output_msg->output()))); + } + + blob_data.outputs.emplace(std::string(flatbuf_bytes_to_sv(collection->pubkey())), std::move(outputs)); + } + } + + return 0; + } + } // namespace msg::fbuf::ledgermsg diff --git a/src/msg/fbuf/ledger_helpers.hpp b/src/msg/fbuf/ledger_helpers.hpp index d92531d4..cf3f51e5 100644 --- a/src/msg/fbuf/ledger_helpers.hpp +++ b/src/msg/fbuf/ledger_helpers.hpp @@ -7,9 +7,10 @@ namespace msg::fbuf::ledgermsg { - void create_ledger_blob_msg_from_ledger_blob(flatbuffers::FlatBufferBuilder &builder, const ledger::ledger_blob &ledger_blob); + const int create_ledger_blob_from_msg(ledger::ledger_blob &blob_data, const std::string &msg, const bool read_inputs, const bool read_outputs); + } // namespace msg::fbuf::ledgermsg #endif \ No newline at end of file diff --git a/src/msg/json/usrmsg_json.cpp b/src/msg/json/usrmsg_json.cpp index f2ebc302..296652c2 100644 --- a/src/msg/json/usrmsg_json.cpp +++ b/src/msg/json/usrmsg_json.cpp @@ -451,7 +451,7 @@ namespace msg::usrmsg::json msg += msg::usrmsg::FLD_RESULTS; msg += "\":["; if (result.index() == 1) - populate_query_results(msg, std::get>(result)); + populate_ledger_query_results(msg, std::get>(result)); msg += "]}"; } @@ -760,10 +760,10 @@ namespace msg::usrmsg::json bool raw_outputs = false; for (auto &val : d[msg::usrmsg::FLD_INCLUDE].array_range()) { - if (val == msg::usrmsg::QUERY_INCLUDE_RAW_INPUTS) + if (val == msg::usrmsg::FLD_RAW_INPUTS) raw_inputs = true; - else if (val == msg::usrmsg::QUERY_INCLUDE_RAW_OUTPUTS) - raw_outputs = false; + else if (val == msg::usrmsg::FLD_RAW_OUTPUTS) + raw_outputs = true; } auto ¶ms_field = d[msg::usrmsg::FLD_PARAMS]; @@ -838,10 +838,12 @@ namespace msg::usrmsg::json } } - void populate_query_results(std::vector &msg, const std::vector &results) + void populate_ledger_query_results(std::vector &msg, const std::vector &results) { - for (const ledger::query::query_result_record &r : results) + for (size_t i = 0; i < results.size(); i++) { + const ledger::query::query_result_record &r = results[i]; + msg += "{\""; msg += msg::usrmsg::FLD_SEQ_NO; msg += SEP_COLON_NOQUOTE; @@ -878,8 +880,56 @@ namespace msg::usrmsg::json msg += msg::usrmsg::FLD_OUTPUT_HASH; msg += SEP_COLON; msg += util::to_hex(r.ledger.output_hash); - msg += "\"}"; + msg += "\""; + + // If raw inputs or outputs is not requested, we don't include that field at all in the response. + // Otherwise the field will always contain an array (empty array if no data). + + if (r.raw_inputs) + { + msg += SEP_COMMA_NOQUOTE; + msg += msg::usrmsg::FLD_RAW_INPUTS; + msg += SEP_COLON_NOQUOTE; + populate_ledger_blob_map(msg, *r.raw_inputs); + } + + if (r.raw_outputs) + { + msg += SEP_COMMA_NOQUOTE; + msg += msg::usrmsg::FLD_RAW_OUTPUTS; + msg += SEP_COLON_NOQUOTE; + populate_ledger_blob_map(msg, *r.raw_outputs); + } + + msg += (i == (results.size() - 1) ? "}" : "},"); } } + void populate_ledger_blob_map(std::vector &msg, const ledger::query::blob_map &blob_map) + { + msg += "["; + for (auto itr = blob_map.begin(); itr != blob_map.end();) + { + msg += "{\""; + msg += msg::usrmsg::FLD_PUBKEY; + msg += SEP_COLON; + msg += util::to_hex(itr->first); + msg += SEP_COMMA; + msg += msg::usrmsg::FLD_BLOBS; + msg += "\":["; + + const std::vector &blobs = itr->second; + for (size_t i = 0; i < blobs.size(); i++) + { + msg += "\""; + msg += util::to_hex(blobs[i]); + msg += (i == (blobs.size() - 1) ? "\"" : "\","); + } + + itr++; + msg += (itr == blob_map.end() ? "]}" : "]},"); + } + msg += "]"; + } + } // namespace msg::usrmsg::json \ No newline at end of file diff --git a/src/msg/json/usrmsg_json.hpp b/src/msg/json/usrmsg_json.hpp index b805d64d..8341b7b6 100644 --- a/src/msg/json/usrmsg_json.hpp +++ b/src/msg/json/usrmsg_json.hpp @@ -49,7 +49,9 @@ namespace msg::usrmsg::json void populate_output_hash_array(std::vector &msg, const util::merkle_hash_node &node); - void populate_query_results(std::vector &msg, const std::vector &results); + void populate_ledger_query_results(std::vector &msg, const std::vector &results); + + void populate_ledger_blob_map(std::vector &msg, const ledger::query::blob_map &blob_map); } // namespace msg::usrmsg::json diff --git a/src/msg/usrmsg_common.hpp b/src/msg/usrmsg_common.hpp index e2b6a571..bfb25456 100644 --- a/src/msg/usrmsg_common.hpp +++ b/src/msg/usrmsg_common.hpp @@ -54,6 +54,9 @@ namespace msg::usrmsg constexpr const char *FLD_USER_HASH = "user_hash"; constexpr const char *FLD_INPUT_HASH = "input_hash"; constexpr const char *FLD_OUTPUT_HASH = "output_hash"; + constexpr const char *FLD_RAW_INPUTS = "raw_inputs"; + constexpr const char *FLD_RAW_OUTPUTS = "raw_outputs"; + constexpr const char *FLD_BLOBS = "blobs"; // Message types constexpr const char *MSGTYPE_USER_CHALLENGE = "user_challenge"; @@ -85,8 +88,6 @@ namespace msg::usrmsg constexpr const char *REASON_NONCE_OVERFLOW = "nonce_overflow"; constexpr const char *REASON_ROUND_INPUTS_OVERFLOW = "round_inputs_overflow"; constexpr const char *QUERY_FILTER_BY_SEQ_NO = "seq_no"; - constexpr const char *QUERY_INCLUDE_RAW_INPUTS = "raw_inputs"; - constexpr const char *QUERY_INCLUDE_RAW_OUTPUTS = "raw_outputs"; } // namespace msg::usrmsg diff --git a/src/util/util.cpp b/src/util/util.cpp index af8d8e16..d99276b6 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -368,17 +368,18 @@ namespace util * Reads from a given file discriptor. * @param fd File descriptor to be read. * @param buf String buffer to be populated. + * @param offset Begin offset of the file to read. * @return Returns number of bytes read in a successful read and -1 on error. */ - int read_from_fd(const int fd, std::string &buf) + int read_from_fd(const int fd, std::string &buf, const off_t offset) { struct stat st; if (fstat(fd, &st) == -1) return -1; - buf.resize(st.st_size); + buf.resize(st.st_size - offset); - return pread(fd, buf.data(), buf.size(), 0); + return pread(fd, buf.data(), buf.size(), offset); } /** diff --git a/src/util/util.hpp b/src/util/util.hpp index 6fbc8102..889d32a2 100644 --- a/src/util/util.hpp +++ b/src/util/util.hpp @@ -72,7 +72,7 @@ namespace util const std::string get_name(std::string_view path); - int read_from_fd(const int fd, std::string &buf); + int read_from_fd(const int fd, std::string &buf, const off_t offset = 0); int set_lock(const int fd, struct flock &lock, const bool is_rwlock, const off_t start, const off_t len);