ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Request historical shards when max shard range is increased (#266)

Browse Source 
* Remove and request historical shards at the startup

* Shard history requesting only at the first consensus round

* Removed test log

* Skip max shard seq no file in removing loop

* Updated the code comments

* Persisting condition changed

* Fixed code comment typos

* Fixed code comment typos

* Resolved PR comments

* Halt consensus until completing only the latest shard sync

* Added meaningful comments

* Resolved PR comments and updated hpfs binary

* Logic enhancement and cleanup

* Cleanup the code comment
This commit is contained in:
Chalith Desaman
2021-03-12 12:13:46 +05:30
committed by GitHub
parent b45cdb999d
commit 00a3da9a2b
8 changed files with 102 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/ledger/ledger.cpp
View File

@@ -274,7 +274,7 @@ namespace ledger
/**
* Remove old shards that exceeds max shard range from file system.
* @param led_shard_no minimum shard number to be in history.
* @param led_shard_no Minimum shard number to be in history.
*/
void remove_old_shards(const uint64_t led_shard_no, std::string_view shard_parent_dir)
{
@@ -297,6 +297,76 @@ namespace ledger
}
}
/**
* Cleanup and request historical shards according to the max we can keep.
* @param shard_seq_no Latest shard sequence number.
* @param shard_parent_dir Shard parent directory.
*/
void persist_shard_history(const uint64_t shard_seq_no, std::string_view shard_parent_dir)
{
// Skip if shard cleanup and requesting has been already done.
if ((shard_parent_dir == PRIMARY_DIR && ctx.primary_shards_persisted) || (shard_parent_dir == BLOB_DIR && ctx.blob_shards_persisted))
return;
// Set persisted flag to true. So this cleanup won't get executed again.
shard_parent_dir == PRIMARY_DIR ? ctx.primary_shards_persisted = true : ctx.blob_shards_persisted = true;
const std::string shard_dir_path = std::string(ledger_fs.physical_path(hpfs::RW_SESSION_NAME, shard_parent_dir));
const uint64_t max_shard_count = shard_dir_path == PRIMARY_DIR ? conf::cfg.node.history_config.max_primary_shards : conf::cfg.node.history_config.max_blob_shards;
const std::list<std::string> shard_list = util::fetch_dir_entries(shard_dir_path);
// Skip the sequence no file from the count.
uint64_t shard_count = shard_list.size() - 1;
// First, In history custom mode remove all the historical shards which are older than the min we can keep.
if (conf::cfg.node.history == conf::HISTORY::CUSTOM && shard_seq_no >= max_shard_count)
{
for (const std::string &shard : shard_list)
{
// Skip the sequence no file.
if (("/" + shard) == SHARD_SEQ_NO_FILENAME)
continue;
uint64_t seq_no;
if (util::stoull(shard, seq_no) != -1 && seq_no <= (shard_seq_no - max_shard_count))
{
const std::string shard_path = std::string(shard_dir_path).append("/").append(shard);
if (util::is_dir_exists(shard_path) && util::remove_directory_recursively(shard_path) == -1)
LOG_ERROR << errno << ": Error deleting shard: " << shard;
else
shard_count--;
}
}
}
// In full history mode request for all the historical nodes if not exists, Otherwise request if max count haven't reached
if (shard_seq_no >= shard_count && (conf::cfg.node.history == conf::HISTORY::FULL || shard_count < max_shard_count))
{
const uint64_t seq_no = shard_seq_no - shard_count;
const std::string prev_shard_hash_file_path = shard_dir_path + "/" + std::to_string(seq_no + 1) + PREV_SHARD_HASH_FILENAME;
const int fd = open(prev_shard_hash_file_path.c_str(), O_RDONLY | O_CLOEXEC);
if (fd == -1)
{
LOG_DEBUG << "Cannot read " << prev_shard_hash_file_path;
return;
}
util::h32 prev_shard_hash_from_file;
// Start reading hash excluding hp_version header.
const int res = pread(fd, &prev_shard_hash_from_file, sizeof(util::h32), util::HP_VERSION_HEADER_SIZE);
close(fd);
if (res == -1)
{
LOG_ERROR << errno << ": Error reading hash file. " << prev_shard_hash_file_path;
return;
}
const std::string sync_name = (shard_parent_dir == PRIMARY_DIR ? "primary" : "blob") + std::string(" shard ") + std::to_string(seq_no);
const std::string shard_path = std::string(shard_parent_dir).append("/").append(std::to_string(seq_no));
ledger_sync_worker.set_target_push_back(hpfs::sync_target{sync_name, prev_shard_hash_from_file, shard_path, hpfs::BACKLOG_ITEM_TYPE::DIR});
}
}
/**
* Save raw data from the consensused proposal. A blob file is only created if there is any user inputs or contract outputs
* to save disk space.