ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

sqlite wrapper for ledger database. (#228)

Browse Source 
Added wrapper methods to create ledger table and insert records for sqlite database.
This commit is contained in:
adheeb-adb
2021-01-29 18:06:31 +05:30
committed by GitHub
parent 3d36912c25
commit 62c91ad90c
11 changed files with 522 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -59,6 +59,8 @@ add_executable(hpcore
src/usr/usr.cpp
src/usr/read_req.cpp
src/ledger.cpp
src/ledger/sqlite.cpp
src/ledger/ledger_sample.cpp
src/consensus.cpp
src/main.cpp
)
@@ -68,6 +70,7 @@ target_link_libraries(hpcore
libblake3.so
libboost_stacktrace_backtrace.a
backtrace
sqlite3
${CMAKE_DL_LIBS} # Needed for stacktrace support
)
add_dependencies(hpcore

3
dev-setup.sh
View File

@@ -107,6 +107,9 @@ rm 1.1.5.tar.gz && rm -r plog-1.1.5
# Boost stacktrace
sudo apt-get install -y libboost-stacktrace-dev
# Sqlite
sudo apt-get install -y sqlite3 libsqlite3-dev
# Update linker library cache.
sudo ldconfig

22
src/crypto.cpp
View File

@@ -175,6 +175,28 @@ namespace crypto
return hash;
}
/**
* Generates blake3 hash for the given string set using stream hashing.
*/
std::string get_hash(const std::set<std::string> &sw_set)
{
std::string hash;
hash.resize(BLAKE3_OUT_LEN);
// Init stream hashing.
blake3_hasher hasher;
blake3_hasher_init(&hasher);
// Hash is generated only using message in contract output struct.
for (std::string_view sw : sw_set)
blake3_hasher_update(&hasher, reinterpret_cast<const unsigned char *>(sw.data()), sw.length());
// Get the final hash.
blake3_hasher_finalize(&hasher, reinterpret_cast<unsigned char *>(hash.data()), hash.length());
return hash;
}
std::string generate_uuid()
{
std::string rand_bytes;

2
src/crypto.hpp
View File

@@ -31,6 +31,8 @@ namespace crypto
std::string get_hash(const std::vector<std::string_view> &sw_vect);
std::string get_hash(const std::set<std::string> &sw_set);
std::string generate_uuid();
} // namespace crypto

7
src/ledger.cpp
View File

@@ -7,6 +7,7 @@
#include "msg/fbuf/p2pmsg_helpers.hpp"
#include "hplog.hpp"
#include "ledger.hpp"
#include "ledger/ledger_sample.hpp"
namespace p2pmsg = msg::fbuf::p2pmsg;
@@ -337,8 +338,12 @@ namespace ledger
* @param proposal Consensus-reached Stage 3 proposal.
* @param raw_inputs Raw inputs that are going to store.
*/
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> raw_inputs)
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> &raw_inputs)
{
// This is used as a sample to create ledger sqlite database,
// Later this callee method can be called directly from consensus on ledger storage implementations.
// ledger::ledger_sample::save_ledger(proposal);
uint64_t seq_no = 0;
std::string hash;
if (extract_lcl(proposal.lcl, seq_no, hash) == -1)

2
src/ledger.hpp
View File

@@ -93,7 +93,7 @@ namespace ledger
const std::pair<uint64_t, std::string> get_ledger_cache_top();
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> raw_inputs);
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> &raw_inputs);
void remove_old_ledgers(const uint64_t led_seq_no);

119
src/ledger/ledger_sample.cpp Normal file
View File

@@ -0,0 +1,119 @@
#include "ledger_sample.hpp"
#include "../crypto.hpp"
#include "../util/util.hpp"
#include "../msg/fbuf/ledger_helpers.hpp"
#include "../msg/fbuf/common_helpers.hpp"
// Currently this namespace is added for sqlite testing, later this can be modified and renamed as 'ledger::ledger_sample' -> 'ledger' for ledger implementations.
namespace ledger::ledger_sample
{
/**
* Create and save ledger record from the given proposal message.
* @param proposal Consensus-reached Stage 3 proposal.
*/
int save_ledger(const p2p::proposal &proposal)
{
sqlite3 *db;
// For testing purpose a database file is created in directory root.
if (sqlite::open_db("ledger.sqlite", &db) == -1)
{
sqlite3_close(db);
return -1;
}
if (!sqlite::is_ledger_table_exist(db) && sqlite::create_ledger_table(db) == -1)
{
sqlite3_close(db);
return -1;
}
uint64_t seq_no = 0;
std::string hash;
if (extract_lcl(proposal.lcl, seq_no, hash) == -1)
{
// lcl records should follow [ledger sequnce numer]-[lcl hex] format.
LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger.";
return -1;
}
seq_no++; // New lcl sequence number.
// Serialize lcl using flatbuffer ledger block schema.
flatbuffers::FlatBufferBuilder builder(1024);
msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no);
// Get binary hash of the serialized lcl.
std::string_view ledger_str_buf = msg::fbuf::flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize());
const std::string lcl_hash = crypto::get_hash(ledger_str_buf);
// Get binary hash of users and inputs.
const std::string user_hash = crypto::get_hash(proposal.users);
const std::string input_hash = crypto::get_hash(proposal.input_hashes);
const std::string seq_no_str = std::to_string(seq_no);
const std::string time_str = std::to_string(proposal.time);
// Contruct binary string for data hash.
std::string data;
data.reserve(seq_no_str.size() + time_str.size() + (32 * 5));
data.append(seq_no_str);
data.append(time_str);
data.append(proposal.state_hash.to_string_view());
data.append(proposal.patch_hash.to_string_view());
data.append(user_hash);
data.append(input_hash);
data.append(proposal.output_hash);
// Get binary hash of data.
const std::string data_hash = crypto::get_hash(data);
// Construct ledger struct.
// Hashes are stored as hex string;
const sqlite::ledger ledger(
seq_no,
proposal.time,
util::to_hex(lcl_hash),
hash,
util::to_hex(data_hash),
util::to_hex(proposal.state_hash.to_string_view()),
util::to_hex(proposal.patch_hash.to_string_view()),
util::to_hex(user_hash),
util::to_hex(input_hash),
util::to_hex(proposal.output_hash));
if (sqlite::insert_ledger_row(db, ledger) == -1)
{
sqlite3_close(db);
return -1;
}
sqlite3_close(db);
return 0;
}
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash)
{
if (lcl == GENESIS_LEDGER)
{
seq_no = 0;
hash = lcl.substr(2);
return 0;
}
const size_t pos = lcl.find("-");
if (pos == std::string::npos)
return -1;
if (util::stoull(lcl.substr(0, pos), seq_no) == -1)
return -1;
hash = lcl.substr(pos + 1);
if (hash.size() != 64)
return -1;
return 0;
}
} // namespace ledger::ledger_sample

12
src/ledger/ledger_sample.hpp Normal file
View File

@@ -0,0 +1,12 @@
#include "../p2p/p2p.hpp"
#include "sqlite.hpp"
namespace ledger::ledger_sample
{
constexpr const char *GENESIS_LEDGER = "0-genesis";
int save_ledger(const p2p::proposal &proposal);
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash);
} // namespace ledger::ledger_sample

249
src/ledger/sqlite.cpp Normal file
View File

@@ -0,0 +1,249 @@
#include "sqlite.hpp"
namespace ledger::sqlite
{
constexpr const char *LEDGER_TABLE = "ledger";
constexpr const char *LEDGER_COLUMNS = "seq_no, time, ledger_hash, prev_ledger_hash, data_hash, state_hash, patch_hash, user_hash, input_hash, output_hash";
constexpr const char *COLUMN_DATA_TYPES[]{"INT", "TEXT"};
constexpr const char *CREATE_TABLE = "CREATE TABLE ";
constexpr const char *INSERT_INTO = "INSERT INTO ";
constexpr const char *PRIMARY_KEY = "PRIMARY KEY";
constexpr const char *NOT_NULL = "NOT NULL";
constexpr const char *VALUES = "VALUES";
constexpr const char *SELECT_ALL = "SELECT * FROM ";
constexpr const char *SQLITE_MASTER = "sqlite_master";
constexpr const char *WHERE = " WHERE ";
constexpr const char *AND = " AND ";
/**
* Opens a connection to a given databse and give the db pointer.
* @param db_name Database name to be connected.
* @param db Pointer to the db pointer which is to be connected and pointed.
* @returns returns 0 on success, or -1 on error.
*/
int open_db(std::string_view db_name, sqlite3 **db)
{
if (sqlite3_open(db_name.data(), db) != SQLITE_OK)
{
LOG_ERROR << "Can't open database: " << sqlite3_errmsg(*db);
return -1;
}
return 0;
}
/**
* Executes given sql query.
* @param db Pointer to the db.
* @param sql Sql query to be executed.
* @param callback Callback funcion which is called for each result row.
* @param callback_first_arg First data argumat to be parced to the callback (void pointer).
* @returns returns 0 on success, or -1 on error.
*/
int exec_sql(sqlite3 *db, std::string_view sql, int (*callback)(void *, int, char **, char **), void *callback_first_arg)
{
char *err_msg;
if (sqlite3_exec(db, sql.data(), callback, (callback != NULL ? (void *)callback_first_arg : NULL), &err_msg) != SQLITE_OK)
{
LOG_ERROR << "SQL error occured: " << err_msg;
sqlite3_free(err_msg);
return -1;
}
return 0;
}
/**
* Create a table with given table info.
* @param db Pointer to the db.
* @param table_name Table name to be created.
* @param column_info Column info of the table.
* @returns returns 0 on success, or -1 on error.
*/
int create_table(sqlite3 *db, std::string_view table_name, const std::vector<table_column_info> &column_info)
{
std::string sql;
sql.append(CREATE_TABLE);
sql.append(table_name);
sql.append(" (");
for (auto itr = column_info.begin(); itr != column_info.end(); ++itr)
{
sql.append(itr->name);
sql.append(" ");
sql.append(COLUMN_DATA_TYPES[itr->column_type]);
if (itr->is_key)
{
sql.append(" ");
sql.append(PRIMARY_KEY);
}
if (!itr->is_null)
{
sql.append(" ");
sql.append(NOT_NULL);
}
if (itr != column_info.end() - 1)
sql.append(",");
}
sql.append(")");
/* Execute SQL statement */
return exec_sql(db, sql);
}
/**
* Insert values to a table.
* @param db Pointer to the db.
* @param table_name Table name to be populated.
* @param column_names_string Comma seperated string of colums (eg: "col_1,col_2,...").
* @param value_strings Vector of comma seperated values (wrap in single quotes for TEXT type) (eg: ["r1val1,'r1val2',...", "r2val1,'r2val2',..."]).
* @returns returns 0 on success, or -1 on error.
*/
int insert_values(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, const std::vector<std::string> &value_strings)
{
std::string sql;
sql.append(INSERT_INTO);
sql.append(table_name);
sql.append("(");
sql.append(column_names_string);
sql.append(") ");
sql.append(VALUES);
for (auto itr = value_strings.begin(); itr != value_strings.end(); ++itr)
{
sql.append("(");
sql.append(*itr);
sql.append(")");
if (itr != value_strings.end() - 1)
sql.append(",");
}
/* Execute SQL statement */
return exec_sql(db, sql);
}
/**
* Insert a value row to a table.
* @param db Pointer to the db.
* @param table_name Table name to be populated.
* @param column_names_string Comma seperated string of colums (eg: "col_1,col_2,...").
* @param value_string comma seperated values as per column order (wrap in single quotes for TEXT type) (eg: "r1val1,'r1val2',...").
* @returns returns 0 on success, or -1 on error.
*/
int insert_value(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, std::string_view value_string)
{
std::string sql;
// Reserving the space for the query before construction.
sql.reserve(sizeof(INSERT_INTO) + table_name.size() + column_names_string.size() + sizeof(VALUES) + value_string.size() + 5);
sql.append(INSERT_INTO);
sql.append(table_name);
sql.append("(");
sql.append(column_names_string);
sql.append(") ");
sql.append(VALUES);
sql.append("(");
sql.append(value_string);
sql.append(")");
/* Execute SQL statement */
return exec_sql(db, sql);
}
/**
* Checks whether table exist in the database.
* @param db Pointer to the db.
* @param table_name Table name to be checked.
* @returns returns true is exist, otherwise false.
*/
bool is_table_exists(sqlite3 *db, std::string_view table_name)
{
std::string sql;
// Reserving the space for the query before construction.
sql.reserve(sizeof(SELECT_ALL) + sizeof(SQLITE_MASTER) + sizeof(WHERE) + sizeof(AND) + table_name.size() + 19);
sql.append(SELECT_ALL);
sql.append(SQLITE_MASTER);
sql.append(WHERE);
sql.append("type='table'");
sql.append(AND);
sql.append("name='");
sql.append(table_name);
sql.append("'");
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql.data(), -1, &stmt, 0) == SQLITE_OK &&
stmt != NULL && sqlite3_step(stmt) == SQLITE_ROW)
{
sqlite3_reset(stmt);
return true;
}
sqlite3_reset(stmt);
return false;
}
/**
* Creates a table for ledger records.
* @param db Pointer to the db.
* @returns returns 0 on success, or -1 on error.
*/
int create_ledger_table(sqlite3 *db)
{
std::vector<table_column_info> column_info{
table_column_info("seq_no", COLUMN_DATA_TYPE::INT, true),
table_column_info("time", COLUMN_DATA_TYPE::INT),
table_column_info("ledger_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("prev_ledger_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("data_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("state_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("patch_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("user_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("input_hash", COLUMN_DATA_TYPE::TEXT),
table_column_info("output_hash", COLUMN_DATA_TYPE::TEXT)};
if (create_table(db, LEDGER_TABLE, column_info) == -1)
return -1;
return 0;
}
/**
* Inserts a ledger record.
* @param db Pointer to the db.
* @param ledger Ledger struct to be inserted.
* @returns returns 0 on success, or -1 on error.
*/
int insert_ledger_row(sqlite3 *db, const ledger &ledger)
{
std::string value_string = std::to_string(ledger.seq_no) + "," +
std::to_string(ledger.time) + "," +
"'" + ledger.ledger_hash_hex + "'," +
"'" + ledger.prev_ledger_hash_hex + "'," +
"'" + ledger.data_hash_hex + "'," +
"'" + ledger.state_hash_hex + "'," +
"'" + ledger.patch_hash_hex + "'," +
"'" + ledger.user_hash_hex + "'," +
"'" + ledger.input_hash_hex + "'," +
"'" + ledger.output_hash_hex + "'";
if (insert_value(db, LEDGER_TABLE, LEDGER_COLUMNS, value_string) == -1)
return -1;
return 0;
}
/**
* Checks whether ledger table exist.
* @param db Pointer to the db.
* @returns returns true is exist, otherwise false.
*/
bool is_ledger_table_exist(sqlite3 *db)
{
return is_table_exists(db, LEDGER_TABLE);
}
} // namespace ledger::sqlite

104
src/ledger/sqlite.hpp Normal file
View File

@@ -0,0 +1,104 @@
#ifndef _LEDGER_SQLITE_
#define _LEDGER_SQLITE_
#include "../pchheader.hpp"
namespace ledger::sqlite
{
/**
* Define an enum and a string array for the column data types.
* Any column data type that needs to be supportes should be added to both the 'COLUMN_DATA_TYPE' enum and the 'column_data_type' array in its respective order.
*/
enum COLUMN_DATA_TYPE
{
INT,
TEXT
};
/**
* Struct of table column information.
* {
* string name Name of the column.
* column_type Data type of the column.
* is_key Whether column is a key.
* is_null Whether column is nullable.
* }
*/
struct table_column_info
{
std::string name;
COLUMN_DATA_TYPE column_type;
bool is_key;
bool is_null;
table_column_info(std::string_view name, const COLUMN_DATA_TYPE &column_type, const bool is_key = false, const bool is_null = false)
: name(name), column_type(column_type), is_key(is_key), is_null(is_null)
{
}
};
/**
* Struct for ledger feilds.
* All the hashes are stored as hex strings.
*/
struct ledger
{
uint64_t seq_no;
uint64_t time;
std::string ledger_hash_hex;
std::string prev_ledger_hash_hex;
std::string data_hash_hex;
std::string state_hash_hex;
std::string patch_hash_hex;
std::string user_hash_hex;
std::string input_hash_hex;
std::string output_hash_hex;
ledger(
const uint64_t seq_no,
const uint64_t time,
std::string_view ledger_hash_hex,
std::string_view prev_ledger_hash_hex,
std::string_view data_hash_hex,
std::string_view state_hash_hex,
std::string_view patch_hash_hex,
std::string_view user_hash_hex,
std::string_view input_hash_hex,
std::string_view output_hash_hex)
: seq_no(seq_no),
time(time),
ledger_hash_hex(ledger_hash_hex),
prev_ledger_hash_hex(prev_ledger_hash_hex),
data_hash_hex(data_hash_hex),
state_hash_hex(state_hash_hex),
patch_hash_hex(patch_hash_hex),
user_hash_hex(user_hash_hex),
input_hash_hex(input_hash_hex),
output_hash_hex(output_hash_hex)
{
}
};
// Generic methods.
int open_db(std::string_view db_name, sqlite3 **db);
int exec_sql(sqlite3 *db, std::string_view sql, int (*callback)(void *, int, char **, char **) = NULL, void *callback_first_arg = NULL);
int create_table(sqlite3 *db, std::string_view table_name, const std::vector<table_column_info> &column_info);
int insert_values(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, const std::vector<std::string> &value_strings);
int insert_value(sqlite3 *db, std::string_view table_name, std::string_view column_names_string, std::string_view value_string);
bool is_table_exists(sqlite3 *db, std::string_view table_name);
// Ledger specific methdods.
int create_ledger_table(sqlite3 *db);
int insert_ledger_row(sqlite3 *db, const ledger &ledger);
bool is_ledger_table_exist(sqlite3 *db);
} // namespace ledger::sqlite
#endif

1
src/pchheader.hpp
View File

@@ -48,5 +48,6 @@
#include <concurrentqueue.h>
#include <plog/Log.h>
#include <plog/Appenders/ColorConsoleAppender.h>
#include <sqlite3.h>
#endif