ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revamped ledger input/output storage and query execution. (#285)

Browse Source 
- Ledger sqlite db improvements.
- Removed flatbuffers from ledger blob data storage.
- Added consensus private mode blob data filtration.
This commit is contained in:
Ravin Perera
2021-04-11 11:45:14 +05:30
committed by GitHub
parent 0a3183a3d6
commit 3941f5f6c6
34 changed files with 1201 additions and 1203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
src/ledger/sqlite.cpp
View File

@@ -5,10 +5,16 @@
namespace ledger::sqlite
{
constexpr const char *LEDGER_TABLE = "ledger";
constexpr const char *HP_VERSION_TABLE = "hp";
constexpr const char *HP_VERSION_COLUMN = "hp_version";
constexpr const char *USERS_TABLE = "users";
constexpr const char *INPUTS_TABLE = "inputs";
constexpr const char *OUTPUTS_TABLE = "outputs";
constexpr const char *HP_TABLE = "hp";
constexpr const char *LEDGER_VERSION_COLUMN = "ledger_version";
constexpr const char *COLUMN_DATA_TYPES[]{"INT", "TEXT", "BLOB"};
constexpr const char *CREATE_TABLE = "CREATE TABLE IF NOT EXISTS ";
constexpr const char *CREATE_INDEX = "CREATE INDEX ";
constexpr const char *CREATE_UNIQUE_INDEX = "CREATE UNIQUE INDEX ";
constexpr const char *JOURNAL_MODE_OFF = "PRAGMA journal_mode=OFF";
constexpr const char *INSERT_INTO = "INSERT INTO ";
constexpr const char *PRIMARY_KEY = "PRIMARY KEY";
constexpr const char *NOT_NULL = "NOT NULL";
@@ -19,13 +25,26 @@ namespace ledger::sqlite
constexpr const char *AND = " AND ";
constexpr const char *SELECT_LAST_LEDGER = "SELECT * FROM ledger ORDER BY seq_no DESC LIMIT 1";
constexpr const char *SELECT_LEDGER_BY_SEQ_NO = "SELECT * FROM ledger WHERE seq_no=? LIMIT 1";
constexpr const char *SELECT_INPUTS_BY_SEQ_NO = "SELECT * FROM inputs WHERE ledger_seq_no=?";
constexpr const char *SELECT_OUTPUTS_BY_SEQ_NO = "SELECT * FROM outputs WHERE ledger_seq_no=?";
constexpr const char *INSERT_INTO_LEDGER = "INSERT INTO ledger("
"seq_no, time, ledger_hash, prev_ledger_hash, data_hash,"
"state_hash, patch_hash, user_hash, input_hash, output_hash"
") VALUES(?,?,?,?,?,?,?,?,?,?)";
constexpr const char *INSERT_INTO_USERS = "INSERT INTO users(ledger_seq_no, pubkey) VALUES(?,?)";
constexpr const char *INSERT_INTO_USER_INPUTS = "INSERT INTO inputs(ledger_seq_no, pubkey, hash, nonce,"
" blob_offset, blob_size) VALUES(?,?,?,?,?,?)";
constexpr const char *INSERT_INTO_USER_OUTPUTS = "INSERT INTO outputs(ledger_seq_no, pubkey, hash,"
" blob_offset, blob_count) VALUES(?,?,?,?,?)";
#define PUBKEY_SIZE 33
#define BIND_H32_BLOB(idx, field) (field.size() == sizeof(util::h32) && sqlite3_bind_blob(stmt, idx, field.data(), sizeof(util::h32), SQLITE_STATIC) == SQLITE_OK)
#define BIND_PUBKEY_BLOB(idx, field) (field.size() == PUBKEY_SIZE && sqlite3_bind_blob(stmt, idx, field.data(), PUBKEY_SIZE, SQLITE_STATIC) == SQLITE_OK)
#define BIND_BLOB(idx, field) (field.size() > 0 && sqlite3_bind_blob(stmt, idx, field.data(), field.size(), SQLITE_STATIC) == SQLITE_OK)
#define GET_H32_BLOB(idx) std::string((char *)sqlite3_column_blob(stmt, idx), sizeof(util::h32))
#define GET_PUBKEY_BLOB(idx) std::string((char *)sqlite3_column_blob(stmt, idx), PUBKEY_SIZE)
/**
* Opens a connection to a given databse and give the db pointer.
@@ -33,15 +52,22 @@ namespace ledger::sqlite
* @param db Pointer to the db pointer which is to be connected and pointed.
* @returns returns 0 on success, or -1 on error.
*/
int open_db(std::string_view db_name, sqlite3 **db)
int open_db(std::string_view db_name, sqlite3 **db, const bool read_only)
{
int ret;
if ((ret = sqlite3_open(db_name.data(), db)) != SQLITE_OK)
const int flags = read_only ? SQLITE_OPEN_READONLY : (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
if ((ret = sqlite3_open_v2(db_name.data(), db, flags, 0)) != SQLITE_OK)
{
*db = NULL;
LOG_ERROR << "Can't open database: " << ret << ", " << sqlite3_errmsg(*db);
return -1;
}
// We turn off journaling for the db because we don't need transacion support.
// Journaling mode introduces lot of extra underyling file system operations which causes lot of overhead.
if (exec_sql(*db, JOURNAL_MODE_OFF) == -1)
return -1;
return 0;
}
@@ -75,9 +101,7 @@ namespace ledger::sqlite
int create_table(sqlite3 *db, std::string_view table_name, const std::vector<table_column_info> &column_info)
{
std::string sql;
sql.append(CREATE_TABLE);
sql.append(table_name);
sql.append(" (");
sql.append(CREATE_TABLE).append(table_name).append(" (");
for (auto itr = column_info.begin(); itr != column_info.end(); ++itr)
{
@@ -102,8 +126,32 @@ namespace ledger::sqlite
}
sql.append(")");
/* Execute SQL statement */
return exec_sql(db, sql);
const int ret = exec_sql(db, sql);
if (ret == -1)
LOG_ERROR << "Error when creating sqlite table " << table_name;
return ret;
}
int create_index(sqlite3 *db, std::string_view table_name, std::string_view column_names, const bool is_unique)
{
std::string index_name = std::string("idx_").append(table_name).append("_").append(column_names);
std::replace(index_name.begin(), index_name.end(), ',', '_');
std::string sql;
sql.append(is_unique ? CREATE_UNIQUE_INDEX : CREATE_INDEX)
.append(index_name)
.append(" ON ")
.append(table_name)
.append("(")
.append(column_names)
.append(")");
const int ret = exec_sql(db, sql);
if (ret == -1)
LOG_ERROR << "Error when creating sqlite index '" << index_name << "' in table " << table_name;
return ret;
}
/**
@@ -210,6 +258,9 @@ namespace ledger::sqlite
*/
int close_db(sqlite3 **db)
{
if (*db == NULL)
return 0;
if (sqlite3_close(*db) != SQLITE_OK)
{
LOG_ERROR << "Can't close database: " << sqlite3_errmsg(*db);
@@ -221,13 +272,13 @@ namespace ledger::sqlite
}
/**
* Creates a table for ledger records.
* Sets up a blank ledger database.
* @param db Pointer to the db.
* @returns returns 0 on success, or -1 on error.
*/
int create_ledger_table(sqlite3 *db)
int initialize_ledger_db(sqlite3 *db)
{
const std::vector<table_column_info> column_info{
const std::vector<table_column_info> columns{
table_column_info("seq_no", COLUMN_DATA_TYPE::INT, true),
table_column_info("time", COLUMN_DATA_TYPE::INT),
table_column_info("ledger_hash", COLUMN_DATA_TYPE::BLOB),
@@ -239,29 +290,86 @@ namespace ledger::sqlite
table_column_info("input_hash", COLUMN_DATA_TYPE::BLOB),
table_column_info("output_hash", COLUMN_DATA_TYPE::BLOB)};
if (create_table(db, LEDGER_TABLE, column_info) == -1)
if (create_table(db, LEDGER_TABLE, columns) == -1 ||
create_index(db, LEDGER_TABLE, "time", true) == -1 ||
create_index(db, LEDGER_TABLE, "ledger_hash", true) == -1)
return -1;
return 0;
}
/**
* Create and update the hp table from the hp version when creating a new shard.
* Sets up a blank ledger raw data database.
* @param db Pointer to the db.
* @param version Hp version.
* @returns returns 0 on success, or -1 on error.
*/
int initialize_ledger_raw_db(sqlite3 *db)
{
// Users table.
{
const std::vector<table_column_info> user_columns{
table_column_info("ledger_seq_no", COLUMN_DATA_TYPE::INT),
table_column_info("pubkey", COLUMN_DATA_TYPE::BLOB)};
if (create_table(db, USERS_TABLE, user_columns) == -1 ||
create_index(db, USERS_TABLE, "ledger_seq_no", false) == -1 ||
create_index(db, USERS_TABLE, "pubkey", false) == -1)
return -1;
}
// Inputs table.
{
const std::vector<table_column_info> input_columns{
table_column_info("ledger_seq_no", COLUMN_DATA_TYPE::INT),
table_column_info("pubkey", COLUMN_DATA_TYPE::BLOB),
table_column_info("hash", COLUMN_DATA_TYPE::BLOB),
table_column_info("nonce", COLUMN_DATA_TYPE::BLOB),
table_column_info("blob_offset", COLUMN_DATA_TYPE::INT),
table_column_info("blob_size", COLUMN_DATA_TYPE::INT)};
if (create_table(db, INPUTS_TABLE, input_columns) == -1 ||
create_index(db, INPUTS_TABLE, "ledger_seq_no", false) == -1 ||
create_index(db, INPUTS_TABLE, "hash", false) == -1 ||
create_index(db, INPUTS_TABLE, "ledger_seq_no,pubkey", false) == -1)
return -1;
}
// Outputs table.
{
const std::vector<table_column_info> input_columns{
table_column_info("ledger_seq_no", COLUMN_DATA_TYPE::INT),
table_column_info("pubkey", COLUMN_DATA_TYPE::BLOB),
table_column_info("hash", COLUMN_DATA_TYPE::BLOB),
table_column_info("blob_offset", COLUMN_DATA_TYPE::INT),
table_column_info("blob_count", COLUMN_DATA_TYPE::INT)};
if (create_table(db, OUTPUTS_TABLE, input_columns) == -1 ||
create_index(db, OUTPUTS_TABLE, "ledger_seq_no", false) == -1 ||
create_index(db, OUTPUTS_TABLE, "hash", false) == -1 ||
create_index(db, OUTPUTS_TABLE, "ledger_seq_no,pubkey", false) == -1)
return -1;
}
return 0;
}
/**
* Create and update the hp system table when creating a new shard.
* @param db Pointer to the db.
* @param version Version string to be placed in the table.
* @returns returns 0 on success, or -1 on error.
*
*/
int create_hp_version_table_and_update(sqlite3 *db, std::string_view version)
int create_hp_table(sqlite3 *db, std::string_view version)
{
const std::vector<table_column_info> column_info{
table_column_info(HP_VERSION_COLUMN, COLUMN_DATA_TYPE::TEXT)};
table_column_info(LEDGER_VERSION_COLUMN, COLUMN_DATA_TYPE::TEXT)};
if (create_table(db, HP_VERSION_TABLE, column_info) == -1)
if (create_table(db, HP_TABLE, column_info) == -1)
return -1;
const std::string value_string = "\"" + std::string(version) + "\"";
if (insert_row(db, HP_VERSION_TABLE, HP_VERSION_COLUMN, value_string) == -1)
if (insert_row(db, HP_TABLE, LEDGER_VERSION_COLUMN, value_string) == -1)
return -1;
return 0;
@@ -293,18 +401,106 @@ namespace ledger::sqlite
return 0;
}
LOG_ERROR << "Error inserting ledger record. " << sqlite3_errmsg(db);
sqlite3_finalize(stmt);
return -1;
}
/**
* Checks whether ledger table exist.
* @param db Pointer to the db.
* @returns returns true is exist, otherwise false.
*/
bool is_ledger_table_exist(sqlite3 *db)
sqlite3_stmt *prepare_user_insert(sqlite3 *db)
{
return is_table_exists(db, LEDGER_TABLE);
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, INSERT_INTO_USERS, -1, &stmt, 0) == SQLITE_OK && stmt != NULL)
return stmt;
return NULL;
}
sqlite3_stmt *prepare_user_input_insert(sqlite3 *db)
{
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, INSERT_INTO_USER_INPUTS, -1, &stmt, 0) == SQLITE_OK && stmt != NULL)
return stmt;
return NULL;
}
sqlite3_stmt *prepare_user_output_insert(sqlite3 *db)
{
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, INSERT_INTO_USER_OUTPUTS, -1, &stmt, 0) == SQLITE_OK && stmt != NULL)
return stmt;
LOG_ERROR << "Prepare sqlite statement failed.";
return NULL;
}
int insert_user_record(sqlite3_stmt *stmt, const uint64_t ledger_seq_no, std::string_view pubkey)
{
if (stmt == NULL)
{
LOG_ERROR << "Sqlite statement null.";
return -1;
}
if (sqlite3_reset(stmt) == SQLITE_OK &&
sqlite3_bind_int64(stmt, 1, ledger_seq_no) == SQLITE_OK &&
BIND_PUBKEY_BLOB(2, pubkey) &&
sqlite3_step(stmt) == SQLITE_DONE)
{
return 0;
}
LOG_ERROR << "Error inserting user record.";
return -1;
}
int insert_user_input_record(sqlite3_stmt *stmt, const uint64_t ledger_seq_no, std::string_view pubkey,
std::string_view hash, std::string_view nonce, const uint64_t blob_offset, const uint64_t blob_size)
{
if (stmt == NULL)
{
LOG_ERROR << "Sqlite statement null.";
return -1;
}
if (sqlite3_reset(stmt) == SQLITE_OK &&
sqlite3_bind_int64(stmt, 1, ledger_seq_no) == SQLITE_OK &&
BIND_PUBKEY_BLOB(2, pubkey) &&
BIND_H32_BLOB(3, hash) &&
BIND_BLOB(4, nonce) &&
sqlite3_bind_int64(stmt, 5, blob_offset) == SQLITE_OK &&
sqlite3_bind_int64(stmt, 6, blob_size) == SQLITE_OK &&
sqlite3_step(stmt) == SQLITE_DONE)
{
return 0;
}
LOG_ERROR << "Error inserting user input record.";
return -1;
}
int insert_user_output_record(sqlite3_stmt *stmt, const uint64_t ledger_seq_no, std::string_view pubkey,
std::string_view hash, const uint64_t blob_offset, const uint64_t output_count)
{
if (stmt == NULL)
{
LOG_ERROR << "Sqlite statement null.";
return -1;
}
if (sqlite3_reset(stmt) == SQLITE_OK &&
sqlite3_bind_int64(stmt, 1, ledger_seq_no) == SQLITE_OK &&
BIND_PUBKEY_BLOB(2, pubkey) &&
BIND_H32_BLOB(3, hash) &&
sqlite3_bind_int64(stmt, 4, blob_offset) == SQLITE_OK &&
sqlite3_bind_int64(stmt, 5, output_count) == SQLITE_OK &&
sqlite3_step(stmt) == SQLITE_DONE)
{
return 0;
}
LOG_ERROR << "Error inserting user output record.";
return -1;
}
/**
@@ -325,7 +521,7 @@ namespace ledger::sqlite
return 0;
}
LOG_ERROR << "Error when querying last ledger from db.";
LOG_ERROR << "Error when querying last ledger from db. " << sqlite3_errmsg(db);
sqlite3_finalize(stmt);
return -1;
}
@@ -358,7 +554,45 @@ namespace ledger::sqlite
}
}
LOG_ERROR << "Error when querying ledger by seq no. from db.";
LOG_ERROR << "Error when querying ledger by seq no. from db. " << sqlite3_errmsg(db);
sqlite3_finalize(stmt);
return -1;
}
int get_user_inputs_by_seq_no(sqlite3 *db, const uint64_t seq_no, std::vector<ledger::ledger_user_input> &inputs)
{
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, SELECT_INPUTS_BY_SEQ_NO, -1, &stmt, 0) == SQLITE_OK && stmt != NULL &&
sqlite3_bind_int64(stmt, 1, seq_no) == SQLITE_OK)
{
while (sqlite3_step(stmt) == SQLITE_ROW)
inputs.push_back(populate_user_input_from_sql_record(stmt));
sqlite3_finalize(stmt);
return 0;
}
LOG_ERROR << "Error when querying ledger inputs by seq no. from db. " << sqlite3_errmsg(db);
sqlite3_finalize(stmt);
return -1;
}
int get_user_outputs_by_seq_no(sqlite3 *db, const uint64_t seq_no, std::vector<ledger::ledger_user_output> &outputs)
{
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, SELECT_OUTPUTS_BY_SEQ_NO, -1, &stmt, 0) == SQLITE_OK && stmt != NULL &&
sqlite3_bind_int64(stmt, 1, seq_no) == SQLITE_OK)
{
while (sqlite3_step(stmt) == SQLITE_ROW)
outputs.push_back(populate_user_output_from_sql_record(stmt));
sqlite3_finalize(stmt);
return 0;
}
LOG_ERROR << "Error when querying ledger outputs by seq no. from db. " << sqlite3_errmsg(db);
sqlite3_finalize(stmt);
return -1;
}
@@ -377,4 +611,27 @@ namespace ledger::sqlite
ledger.output_hash = GET_H32_BLOB(9);
}
ledger::ledger_user_input populate_user_input_from_sql_record(sqlite3_stmt *stmt)
{
ledger::ledger_user_input inp;
inp.ledger_seq_no = sqlite3_column_int64(stmt, 0);
inp.pubkey = GET_PUBKEY_BLOB(1);
inp.hash = GET_H32_BLOB(2);
// inp.nonce = GET_BLOB(3);
inp.blob_offset = sqlite3_column_int64(stmt, 4);
inp.blob_size = sqlite3_column_int64(stmt, 5);
return inp;
}
ledger::ledger_user_output populate_user_output_from_sql_record(sqlite3_stmt *stmt)
{
ledger::ledger_user_output out;
out.ledger_seq_no = sqlite3_column_int64(stmt, 0);
out.pubkey = GET_PUBKEY_BLOB(1);
out.hash = GET_H32_BLOB(2);
out.blob_offset = sqlite3_column_int64(stmt, 3);
out.blob_count = sqlite3_column_int64(stmt, 4);
return out;
}
} // namespace ledger::sqlite