From 62c91ad90cd6f2942b483deb19f3b24749b395c3 Mon Sep 17 00:00:00 2001 From: adheeb-adb <63283463+adheeb-adb@users.noreply.github.com> Date: Fri, 29 Jan 2021 18:06:31 +0530 Subject: [PATCH] sqlite wrapper for ledger database. (#228) Added wrapper methods to create ledger table and insert records for sqlite database. --- CMakeLists.txt | 3 + dev-setup.sh | 3 + src/crypto.cpp | 22 ++++ src/crypto.hpp | 2 + src/ledger.cpp | 7 +- src/ledger.hpp | 2 +- src/ledger/ledger_sample.cpp | 119 +++++++++++++++++ src/ledger/ledger_sample.hpp | 12 ++ src/ledger/sqlite.cpp | 249 +++++++++++++++++++++++++++++++++++ src/ledger/sqlite.hpp | 104 +++++++++++++++ src/pchheader.hpp | 1 + 11 files changed, 522 insertions(+), 2 deletions(-) create mode 100644 src/ledger/ledger_sample.cpp create mode 100644 src/ledger/ledger_sample.hpp create mode 100644 src/ledger/sqlite.cpp create mode 100644 src/ledger/sqlite.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e4b103bf..02e82797 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,8 @@ add_executable(hpcore src/usr/usr.cpp src/usr/read_req.cpp src/ledger.cpp + src/ledger/sqlite.cpp + src/ledger/ledger_sample.cpp src/consensus.cpp src/main.cpp ) @@ -68,6 +70,7 @@ target_link_libraries(hpcore libblake3.so libboost_stacktrace_backtrace.a backtrace + sqlite3 ${CMAKE_DL_LIBS} # Needed for stacktrace support ) add_dependencies(hpcore diff --git a/dev-setup.sh b/dev-setup.sh index f0a50386..e7f90fc2 100755 --- a/dev-setup.sh +++ b/dev-setup.sh @@ -107,6 +107,9 @@ rm 1.1.5.tar.gz && rm -r plog-1.1.5 # Boost stacktrace sudo apt-get install -y libboost-stacktrace-dev +# Sqlite +sudo apt-get install -y sqlite3 libsqlite3-dev + # Update linker library cache. sudo ldconfig diff --git a/src/crypto.cpp b/src/crypto.cpp index 88d1bbfc..cc6673a0 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -175,6 +175,28 @@ namespace crypto return hash; } + /** + * Generates blake3 hash for the given string set using stream hashing. + */ + std::string get_hash(const std::set &sw_set) + { + std::string hash; + hash.resize(BLAKE3_OUT_LEN); + + // Init stream hashing. + blake3_hasher hasher; + blake3_hasher_init(&hasher); + + // Hash is generated only using message in contract output struct. + for (std::string_view sw : sw_set) + blake3_hasher_update(&hasher, reinterpret_cast(sw.data()), sw.length()); + + // Get the final hash. + blake3_hasher_finalize(&hasher, reinterpret_cast(hash.data()), hash.length()); + + return hash; + } + std::string generate_uuid() { std::string rand_bytes; diff --git a/src/crypto.hpp b/src/crypto.hpp index 99ce3a7d..d88654ae 100644 --- a/src/crypto.hpp +++ b/src/crypto.hpp @@ -31,6 +31,8 @@ namespace crypto std::string get_hash(const std::vector &sw_vect); + std::string get_hash(const std::set &sw_set); + std::string generate_uuid(); } // namespace crypto diff --git a/src/ledger.cpp b/src/ledger.cpp index f1e2f217..9eae7a55 100644 --- a/src/ledger.cpp +++ b/src/ledger.cpp @@ -7,6 +7,7 @@ #include "msg/fbuf/p2pmsg_helpers.hpp" #include "hplog.hpp" #include "ledger.hpp" +#include "ledger/ledger_sample.hpp" namespace p2pmsg = msg::fbuf::p2pmsg; @@ -337,8 +338,12 @@ namespace ledger * @param proposal Consensus-reached Stage 3 proposal. * @param raw_inputs Raw inputs that are going to store. */ - int save_ledger(const p2p::proposal &proposal, const std::unordered_map raw_inputs) + int save_ledger(const p2p::proposal &proposal, const std::unordered_map &raw_inputs) { + // This is used as a sample to create ledger sqlite database, + // Later this callee method can be called directly from consensus on ledger storage implementations. + // ledger::ledger_sample::save_ledger(proposal); + uint64_t seq_no = 0; std::string hash; if (extract_lcl(proposal.lcl, seq_no, hash) == -1) diff --git a/src/ledger.hpp b/src/ledger.hpp index f7cf5cde..db996c1c 100644 --- a/src/ledger.hpp +++ b/src/ledger.hpp @@ -93,7 +93,7 @@ namespace ledger const std::pair get_ledger_cache_top(); - int save_ledger(const p2p::proposal &proposal, const std::unordered_map raw_inputs); + int save_ledger(const p2p::proposal &proposal, const std::unordered_map &raw_inputs); void remove_old_ledgers(const uint64_t led_seq_no); diff --git a/src/ledger/ledger_sample.cpp b/src/ledger/ledger_sample.cpp new file mode 100644 index 00000000..66c002f7 --- /dev/null +++ b/src/ledger/ledger_sample.cpp @@ -0,0 +1,119 @@ + +#include "ledger_sample.hpp" +#include "../crypto.hpp" +#include "../util/util.hpp" +#include "../msg/fbuf/ledger_helpers.hpp" +#include "../msg/fbuf/common_helpers.hpp" + +// Currently this namespace is added for sqlite testing, later this can be modified and renamed as 'ledger::ledger_sample' -> 'ledger' for ledger implementations. +namespace ledger::ledger_sample +{ + /** + * Create and save ledger record from the given proposal message. + * @param proposal Consensus-reached Stage 3 proposal. + */ + int save_ledger(const p2p::proposal &proposal) + { + sqlite3 *db; + + // For testing purpose a database file is created in directory root. + if (sqlite::open_db("ledger.sqlite", &db) == -1) + { + sqlite3_close(db); + return -1; + } + + if (!sqlite::is_ledger_table_exist(db) && sqlite::create_ledger_table(db) == -1) + { + sqlite3_close(db); + return -1; + } + + uint64_t seq_no = 0; + std::string hash; + if (extract_lcl(proposal.lcl, seq_no, hash) == -1) + { + // lcl records should follow [ledger sequnce numer]-[lcl hex] format. + LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger."; + return -1; + } + + seq_no++; // New lcl sequence number. + + // Serialize lcl using flatbuffer ledger block schema. + flatbuffers::FlatBufferBuilder builder(1024); + msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no); + + // Get binary hash of the serialized lcl. + std::string_view ledger_str_buf = msg::fbuf::flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize()); + const std::string lcl_hash = crypto::get_hash(ledger_str_buf); + + // Get binary hash of users and inputs. + const std::string user_hash = crypto::get_hash(proposal.users); + const std::string input_hash = crypto::get_hash(proposal.input_hashes); + + const std::string seq_no_str = std::to_string(seq_no); + const std::string time_str = std::to_string(proposal.time); + + // Contruct binary string for data hash. + std::string data; + data.reserve(seq_no_str.size() + time_str.size() + (32 * 5)); + data.append(seq_no_str); + data.append(time_str); + data.append(proposal.state_hash.to_string_view()); + data.append(proposal.patch_hash.to_string_view()); + data.append(user_hash); + data.append(input_hash); + data.append(proposal.output_hash); + + // Get binary hash of data. + const std::string data_hash = crypto::get_hash(data); + + // Construct ledger struct. + // Hashes are stored as hex string; + const sqlite::ledger ledger( + seq_no, + proposal.time, + util::to_hex(lcl_hash), + hash, + util::to_hex(data_hash), + util::to_hex(proposal.state_hash.to_string_view()), + util::to_hex(proposal.patch_hash.to_string_view()), + util::to_hex(user_hash), + util::to_hex(input_hash), + util::to_hex(proposal.output_hash)); + + if (sqlite::insert_ledger_row(db, ledger) == -1) + { + sqlite3_close(db); + return -1; + } + + sqlite3_close(db); + + return 0; + } + + int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash) + { + if (lcl == GENESIS_LEDGER) + { + seq_no = 0; + hash = lcl.substr(2); + return 0; + } + + const size_t pos = lcl.find("-"); + if (pos == std::string::npos) + return -1; + + if (util::stoull(lcl.substr(0, pos), seq_no) == -1) + return -1; + + hash = lcl.substr(pos + 1); + if (hash.size() != 64) + return -1; + + return 0; + } +} // namespace ledger::ledger_sample \ No newline at end of file diff --git a/src/ledger/ledger_sample.hpp b/src/ledger/ledger_sample.hpp new file mode 100644 index 00000000..eef4c1de --- /dev/null +++ b/src/ledger/ledger_sample.hpp @@ -0,0 +1,12 @@ +#include "../p2p/p2p.hpp" +#include "sqlite.hpp" + +namespace ledger::ledger_sample +{ + constexpr const char *GENESIS_LEDGER = "0-genesis"; + + int save_ledger(const p2p::proposal &proposal); + + int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash); + +} // namespace ledger::ledger_sample \ No newline at end of file diff --git a/src/ledger/sqlite.cpp b/src/ledger/sqlite.cpp new file mode 100644 index 00000000..36c33790 --- /dev/null +++ b/src/ledger/sqlite.cpp @@ -0,0 +1,249 @@ +#include "sqlite.hpp" + +namespace ledger::sqlite +{ + constexpr const char *LEDGER_TABLE = "ledger"; + constexpr const char *LEDGER_COLUMNS = "seq_no, time, ledger_hash, prev_ledger_hash, data_hash, state_hash, patch_hash, user_hash, input_hash, output_hash"; + constexpr const char *COLUMN_DATA_TYPES[]{"INT", "TEXT"}; + constexpr const char *CREATE_TABLE = "CREATE TABLE "; + constexpr const char *INSERT_INTO = "INSERT INTO "; + constexpr const char *PRIMARY_KEY = "PRIMARY KEY"; + constexpr const char *NOT_NULL = "NOT NULL"; + constexpr const char *VALUES = "VALUES"; + constexpr const char *SELECT_ALL = "SELECT * FROM "; + constexpr const char *SQLITE_MASTER = "sqlite_master"; + constexpr const char *WHERE = " WHERE "; + constexpr const char *AND = " AND "; + + /** + * Opens a connection to a given databse and give the db pointer. + * @param db_name Database name to be connected. + * @param db Pointer to the db pointer which is to be connected and pointed. + * @returns returns 0 on success, or -1 on error. + */ + int open_db(std::string_view db_name, sqlite3 **db) + { + if (sqlite3_open(db_name.data(), db) != SQLITE_OK) + { + LOG_ERROR << "Can't open database: " << sqlite3_errmsg(*db); + return -1; + } + return 0; + } + + /** + * Executes given sql query. + * @param db Pointer to the db. + * @param sql Sql query to be executed. + * @param callback Callback funcion which is called for each result row. + * @param callback_first_arg First data argumat to be parced to the callback (void pointer). + * @returns returns 0 on success, or -1 on error. + */ + int exec_sql(sqlite3 *db, std::string_view sql, int (*callback)(void *, int, char **, char **), void *callback_first_arg) + { + char *err_msg; + if (sqlite3_exec(db, sql.data(), callback, (callback != NULL ? (void *)callback_first_arg : NULL), &err_msg) != SQLITE_OK) + { + LOG_ERROR << "SQL error occured: " << err_msg; + sqlite3_free(err_msg); + return -1; + } + return 0; + } + + /** + * Create a table with given table info. + * @param db Pointer to the db. + * @param table_name Table name to be created. + * @param column_info Column info of the table. + * @returns returns 0 on success, or -1 on error. + */ + int create_table(sqlite3 *db, std::string_view table_name, const std::vector &column_info) + { + std::string sql; + sql.append(CREATE_TABLE); + sql.append(table_name); + sql.append(" ("); + + for (auto itr = column_info.begin(); itr != column_info.end(); ++itr) + { + sql.append(itr->name); + sql.append(" "); + sql.append(COLUMN_DATA_TYPES[itr->column_type]); + + if (itr->is_key) + { + sql.append(" "); + sql.append(PRIMARY_KEY); + } + + if (!itr->is_null) + { + sql.append(" "); + sql.append(NOT_NULL); + } + + if (itr != column_info.end() - 1) + sql.append(","); + } + sql.append(")"); + + /* Execute SQL statement */ + return exec_sql(db, sql); + } + + /** + * Insert values to a table. + * @param db Pointer to the db. + * @param table_name Table name to be populated. + * @param column_names_string Comma seperated string of colums (eg: "col_1,col_2,..."). + * @param value_strings Vector of comma seperated values (wrap in single quotes for TEXT type) (eg: ["r1val1,'r1val2',...", "r2val1,'r2val2',..."]). + * @returns returns 0 on success, or -1 on error. + */ + int insert_values(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, const std::vector &value_strings) + { + std::string sql; + + sql.append(INSERT_INTO); + sql.append(table_name); + sql.append("("); + sql.append(column_names_string); + sql.append(") "); + sql.append(VALUES); + + for (auto itr = value_strings.begin(); itr != value_strings.end(); ++itr) + { + sql.append("("); + sql.append(*itr); + sql.append(")"); + + if (itr != value_strings.end() - 1) + sql.append(","); + } + + /* Execute SQL statement */ + return exec_sql(db, sql); + } + + /** + * Insert a value row to a table. + * @param db Pointer to the db. + * @param table_name Table name to be populated. + * @param column_names_string Comma seperated string of colums (eg: "col_1,col_2,..."). + * @param value_string comma seperated values as per column order (wrap in single quotes for TEXT type) (eg: "r1val1,'r1val2',..."). + * @returns returns 0 on success, or -1 on error. + */ + int insert_value(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, std::string_view value_string) + { + std::string sql; + // Reserving the space for the query before construction. + sql.reserve(sizeof(INSERT_INTO) + table_name.size() + column_names_string.size() + sizeof(VALUES) + value_string.size() + 5); + + sql.append(INSERT_INTO); + sql.append(table_name); + sql.append("("); + sql.append(column_names_string); + sql.append(") "); + sql.append(VALUES); + sql.append("("); + sql.append(value_string); + sql.append(")"); + + /* Execute SQL statement */ + return exec_sql(db, sql); + } + + /** + * Checks whether table exist in the database. + * @param db Pointer to the db. + * @param table_name Table name to be checked. + * @returns returns true is exist, otherwise false. + */ + bool is_table_exists(sqlite3 *db, std::string_view table_name) + { + std::string sql; + // Reserving the space for the query before construction. + sql.reserve(sizeof(SELECT_ALL) + sizeof(SQLITE_MASTER) + sizeof(WHERE) + sizeof(AND) + table_name.size() + 19); + + sql.append(SELECT_ALL); + sql.append(SQLITE_MASTER); + sql.append(WHERE); + sql.append("type='table'"); + sql.append(AND); + sql.append("name='"); + sql.append(table_name); + sql.append("'"); + + sqlite3_stmt *stmt; + + if (sqlite3_prepare_v2(db, sql.data(), -1, &stmt, 0) == SQLITE_OK && + stmt != NULL && sqlite3_step(stmt) == SQLITE_ROW) + { + sqlite3_reset(stmt); + return true; + } + + sqlite3_reset(stmt); + return false; + } + + /** + * Creates a table for ledger records. + * @param db Pointer to the db. + * @returns returns 0 on success, or -1 on error. + */ + int create_ledger_table(sqlite3 *db) + { + std::vector column_info{ + table_column_info("seq_no", COLUMN_DATA_TYPE::INT, true), + table_column_info("time", COLUMN_DATA_TYPE::INT), + table_column_info("ledger_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("prev_ledger_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("data_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("state_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("patch_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("user_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("input_hash", COLUMN_DATA_TYPE::TEXT), + table_column_info("output_hash", COLUMN_DATA_TYPE::TEXT)}; + + if (create_table(db, LEDGER_TABLE, column_info) == -1) + return -1; + + return 0; + } + + /** + * Inserts a ledger record. + * @param db Pointer to the db. + * @param ledger Ledger struct to be inserted. + * @returns returns 0 on success, or -1 on error. + */ + int insert_ledger_row(sqlite3 *db, const ledger &ledger) + { + std::string value_string = std::to_string(ledger.seq_no) + "," + + std::to_string(ledger.time) + "," + + "'" + ledger.ledger_hash_hex + "'," + + "'" + ledger.prev_ledger_hash_hex + "'," + + "'" + ledger.data_hash_hex + "'," + + "'" + ledger.state_hash_hex + "'," + + "'" + ledger.patch_hash_hex + "'," + + "'" + ledger.user_hash_hex + "'," + + "'" + ledger.input_hash_hex + "'," + + "'" + ledger.output_hash_hex + "'"; + + if (insert_value(db, LEDGER_TABLE, LEDGER_COLUMNS, value_string) == -1) + return -1; + + return 0; + } + + /** + * Checks whether ledger table exist. + * @param db Pointer to the db. + * @returns returns true is exist, otherwise false. + */ + bool is_ledger_table_exist(sqlite3 *db) + { + return is_table_exists(db, LEDGER_TABLE); + } +} // namespace ledger::sqlite \ No newline at end of file diff --git a/src/ledger/sqlite.hpp b/src/ledger/sqlite.hpp new file mode 100644 index 00000000..e5941dbe --- /dev/null +++ b/src/ledger/sqlite.hpp @@ -0,0 +1,104 @@ +#ifndef _LEDGER_SQLITE_ +#define _LEDGER_SQLITE_ + +#include "../pchheader.hpp" + +namespace ledger::sqlite +{ + /** + * Define an enum and a string array for the column data types. + * Any column data type that needs to be supportes should be added to both the 'COLUMN_DATA_TYPE' enum and the 'column_data_type' array in its respective order. + */ + enum COLUMN_DATA_TYPE + { + INT, + TEXT + }; + + /** + * Struct of table column information. + * { + * string name Name of the column. + * column_type Data type of the column. + * is_key Whether column is a key. + * is_null Whether column is nullable. + * } + */ + struct table_column_info + { + std::string name; + COLUMN_DATA_TYPE column_type; + bool is_key; + bool is_null; + + table_column_info(std::string_view name, const COLUMN_DATA_TYPE &column_type, const bool is_key = false, const bool is_null = false) + : name(name), column_type(column_type), is_key(is_key), is_null(is_null) + { + } + }; + + /** + * Struct for ledger feilds. + * All the hashes are stored as hex strings. + */ + struct ledger + { + uint64_t seq_no; + uint64_t time; + std::string ledger_hash_hex; + std::string prev_ledger_hash_hex; + std::string data_hash_hex; + std::string state_hash_hex; + std::string patch_hash_hex; + std::string user_hash_hex; + std::string input_hash_hex; + std::string output_hash_hex; + + ledger( + const uint64_t seq_no, + const uint64_t time, + std::string_view ledger_hash_hex, + std::string_view prev_ledger_hash_hex, + std::string_view data_hash_hex, + std::string_view state_hash_hex, + std::string_view patch_hash_hex, + std::string_view user_hash_hex, + std::string_view input_hash_hex, + std::string_view output_hash_hex) + : seq_no(seq_no), + time(time), + ledger_hash_hex(ledger_hash_hex), + prev_ledger_hash_hex(prev_ledger_hash_hex), + data_hash_hex(data_hash_hex), + state_hash_hex(state_hash_hex), + patch_hash_hex(patch_hash_hex), + user_hash_hex(user_hash_hex), + input_hash_hex(input_hash_hex), + output_hash_hex(output_hash_hex) + { + } + }; + + // Generic methods. + int open_db(std::string_view db_name, sqlite3 **db); + + int exec_sql(sqlite3 *db, std::string_view sql, int (*callback)(void *, int, char **, char **) = NULL, void *callback_first_arg = NULL); + + int create_table(sqlite3 *db, std::string_view table_name, const std::vector &column_info); + + int insert_values(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, const std::vector &value_strings); + + int insert_value(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, std::string_view value_string); + + bool is_table_exists(sqlite3 *db, std::string_view table_name); + + // Ledger specific methdods. + int create_ledger_table(sqlite3 *db); + + int insert_ledger_row(sqlite3 *db, const ledger &ledger); + + bool is_ledger_table_exist(sqlite3 *db); + +} // namespace ledger::sqlite + +#endif \ No newline at end of file diff --git a/src/pchheader.hpp b/src/pchheader.hpp index ca58f163..15043ddf 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -48,5 +48,6 @@ #include #include #include +#include #endif \ No newline at end of file