ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Code cleanup and naming consistency improvement. (#73)

Browse Source
This commit is contained in:
Ravin Perera
2019-12-19 19:22:17 +05:30
committed by GitHub
parent b475e2796e
commit 7255dae84c
23 changed files with 594 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cluster-create.sh
View File

@@ -48,7 +48,7 @@ do
node -p "JSON.stringify({...require('./tmp.json'), \
binary: '/usr/local/bin/node', \
binargs: './bin/contract.js', \
appbill: 'appbill', \
appbill: '', \
appbillargs: '', \
peerport: ${peerport}, \
pubport: ${pubport}, \

22
src/bill/appbill.cpp
View File

@@ -2,7 +2,6 @@
App bill default implementation 1
MSB of return value is reserved for appbill error
bits 1-8 indicate whether or not public keys 0-6 passed appbill check
(to be completed)
*/
@@ -577,10 +576,17 @@ int check_mode(int argc, char** argv, int print_balances) {
return 128;
}
if (correct_for_ed_keys(argc, argv, 2, 0))
if (argc == 0 || argc % 2 == 1 && !print_balances) {
fprintf(stderr, "appbill check mode requires a public key%s\n", (print_balances ? "" : " and an amount to check against"));
return 128;
}
if (correct_for_ed_keys(argc, argv, (print_balances ? 1 : 2), 0))
return 128;
for (int i = 0; i < argc; i+=2) {
for (int i = 0; i < argc; i+= ( print_balances ? 1 : 2 )) {
// check the pubkey
for (char* x = argv[i];; ++x) {
if ( x - argv[i] == KEY_SIZE*2 && *x != '\0' ) {
@@ -598,6 +604,9 @@ int check_mode(int argc, char** argv, int print_balances) {
return 128;
}
if (print_balances)
continue;
// check the bytecount
for (char* x = argv[i+1]; *x != '\0'; ++x) {
if (*x >= '0' && *x <= '9')
@@ -627,13 +636,14 @@ int check_mode(int argc, char** argv, int print_balances) {
bits[i] = 0;
// loop keys, check balances
for (int i = 0, j = 0; i < argc; i+=2, ++j) {
for (int i = 0, j = 0; i < argc; i+=( print_balances ? 1 : 2 ), ++j) {
// convert the argv from hex to binary
uint8_t key[32];
key_from_hex((uint8_t*)argv[i], key);
uint32_t bytecount = 0;
sscanf(argv[i+1], "%d", &bytecount);
if (!print_balances)
sscanf(argv[i+1], "%d", &bytecount);
int error = 0;
uint64_t balance = 0;
@@ -702,4 +712,4 @@ int main(int argc, char** argv) {
fprintf(stderr, "unknown mode, execution should not reach here\n");
return 128;
}
}

6
src/conf.cpp
View File

@@ -82,7 +82,7 @@ int create_contract()
boost::filesystem::create_directories(ctx.configdir);
boost::filesystem::create_directories(ctx.histdir);
boost::filesystem::create_directories(ctx.statedir);
boost::filesystem::create_directories(ctx.statehistdir);
boost::filesystem::create_directories(ctx.state_hist_dir);
//Create config file with default settings.
@@ -148,7 +148,7 @@ void set_contract_dir_paths(std::string exepath, std::string basedir)
ctx.tlscertfile = ctx.configdir + "/tlscert.pem";
ctx.histdir = basedir + "/hist";
ctx.statedir = basedir + "/state";
ctx.statehistdir = basedir + "/statehist";
ctx.state_hist_dir = basedir + "/statehist";
ctx.logdir = basedir + "/log";
}
@@ -517,7 +517,7 @@ int validate_contract_dir_paths()
ctx.configfile,
ctx.histdir,
ctx.statedir,
ctx.statehistdir,
ctx.state_hist_dir,
ctx.tlskeyfile,
ctx.tlscertfile};

2
src/conf.hpp
View File

@@ -30,7 +30,7 @@ struct contract_ctx
std::string contractdir; // Contract base directory full path
std::string histdir; // Contract history dir full path
std::string statedir; // Contract executing state dir full path (This is the fuse mount point)
std::string statehistdir; // Contract state history dir full path
std::string state_hist_dir; // Contract state history dir full path
std::string logdir; // Contract log dir full path
std::string configdir; // Contract config dir full path
std::string configfile; // Full path to the contract config file

5
src/cons/cons.cpp
View File

@@ -170,6 +170,9 @@ void consensus()
if (should_request_history)
{
// TODO: If we are in a lcl fork condition try to rollback state with the help of
// state_restore to rollback state checkpoints before requesting new state.
//handle minority going forward when boostrapping cluster.
//Here we are mimicking invalid min ledger scenario.
if (majority_lcl == GENESIS_LEDGER)
@@ -410,7 +413,7 @@ bool verify_appbill_check(std::string_view pubkey, const size_t input_len)
if (pid == 0)
{
// before execution chdir into a valid the latest state data directory that contains an appbill.table
chdir(statefs::current_ctx.datadir.c_str());
chdir(statefs::current_ctx.data_dir.c_str());
int ret = execv(execv_args[0], execv_args);
LOG_ERR << "Appbill process execv failed: " << ret;
return false;

50
src/cons/state_handler.cpp
View File

@@ -25,7 +25,14 @@ std::list<backlog_item> pending_requests;
// List of submitted requests we are awaiting responses for, keyed by expected response hash.
std::unordered_map<hasher::B2H, backlog_item, hasher::B2H_std_key_hasher> submitted_requests;
void request_state_from_peer(const std::string &path, const bool is_file, const std::string &lcl, const int32_t block_id, const hasher::B2H expected_hash)
/**
* Sends a state request to a random peer.
* @param path Requested file or dir path.
* @param is_file Whether the requested path if a file or dir.
* @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1.
* @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different.
*/
void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hasher::B2H expected_hash)
{
p2p::state_request sr;
sr.parent_path = path;
@@ -34,33 +41,40 @@ void request_state_from_peer(const std::string &path, const bool is_file, const
sr.expected_hash = expected_hash;
p2p::peer_outbound_message msg(std::make_unique<flatbuffers::FlatBufferBuilder>(1024));
fbschema::p2pmsg::create_msg_from_state_request(msg.builder(), sr, lcl);
fbschema::p2pmsg::create_msg_from_state_request(msg.builder(), sr, ctx.lcl);
p2p::send_message_to_random_peer(msg); //todo: send to a node that hold the majority state to improve reliability of retrieving state.
}
/**
* Creats the reply message for a given state request.
* @param msg The peer outbound message reference to build up the reply message.
* @param sr The state request which should be replied to.
*/
int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr)
{
// If block_id > -1 this means this is a file block data request.
if (sr.block_id > -1)
{
std::vector<uint8_t> blocks;
if (statefs::get_block(blocks, sr.parent_path, sr.block_id, sr.expected_hash) == -1)
// Vector to hold the block bytes. Normally block size is constant BLOCK_SIZE (4MB), but the
// last block of a file may have a smaller size.
std::vector<uint8_t> block;
if (statefs::get_block(block, sr.parent_path, sr.block_id, sr.expected_hash) == -1)
return -1;
p2p::block_response resp;
resp.path = sr.parent_path;
resp.block_id = sr.block_id;
resp.hash = sr.expected_hash;
resp.data = std::string_view(reinterpret_cast<const char *>(blocks.data()), blocks.size());
resp.data = std::string_view(reinterpret_cast<const char *>(block.data()), block.size());
fbschema::p2pmsg::create_msg_from_block_response(msg.builder(), resp, ctx.lcl);
}
else
{
// File state request means we have to reply with the file block hash map.
if (sr.is_file)
{
std::vector<uint8_t> existing_block_hashmap;
if (statefs::get_block_hash_map(existing_block_hashmap, sr.parent_path, sr.expected_hash) == -1)
return -1;
@@ -68,8 +82,8 @@ int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_requ
}
else
{
// If the state request is for a directory we need to reply with the file system entries and their hashes inside that dir.
std::unordered_map<std::string, p2p::state_fs_hash_entry> existing_fs_entries;
if (statefs::get_fs_entry_hashes(existing_fs_entries, sr.parent_path, sr.expected_hash) == -1)
return -1;
@@ -80,6 +94,11 @@ int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_requ
return 0;
}
/**
* Initiates state sync process by setting up context variables and sending the initial state request.
* @param state_hash_to_request Peer's expected state hash. If peer doesn't have this as its state hash the
* request will be ignord.
*/
void start_state_sync(const hasher::B2H state_hash_to_request)
{
{
@@ -98,13 +117,16 @@ void start_state_sync(const hasher::B2H state_hash_to_request)
submit_request(backlog_item{BACKLOG_ITEM_TYPE::DIR, "/", -1, state_hash_to_request});
}
/**
* Runs the state sync loop.
*/
int run_state_sync_iterator()
{
while (true)
{
util::sleep(SYNC_LOOP_WAIT);
// TODO: Also bypass peer session handler responses if not syncing.
// TODO: Also bypass peer session handler state responses if we're not syncing.
if (!ctx.is_state_syncing)
continue;
@@ -188,16 +210,22 @@ int run_state_sync_iterator()
return 0;
}
/**
* Submits a pending state request to the peer.
*/
void submit_request(const backlog_item &request)
{
LOG_DBG << "Submitting state request. type:" << request.type << " path:" << request.path << " blockid:" << request.block_id;
LOG_DBG << "Submitting state request. type:" << request.type << " path:" << request.path << " block_id:" << request.block_id;
submitted_requests.try_emplace(request.expected_hash, request);
const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR;
request_state_from_peer(request.path, is_file, ctx.lcl, request.block_id, request.expected_hash);
request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash);
}
/**
* Process state file system entry response for a directory.
*/
int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp)
{
std::unordered_map<std::string, p2p::state_fs_hash_entry> state_fs_entry_list;

2
src/cons/state_handler.hpp
View File

@@ -33,7 +33,7 @@ extern std::list<std::string> candidate_state_responses;
int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr);
void request_state_from_peer(const std::string &path, const bool is_file, const std::string &lcl, const int32_t block_id, const hasher::B2H expected_hash);
void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hasher::B2H expected_hash);
void start_state_sync(const hasher::B2H state_hash_to_request);

2
src/main.cpp
View File

@@ -192,7 +192,7 @@ int main(int argc, char **argv)
LOG_INFO << "Operating mode: "
<< (conf::cfg.mode == conf::OPERATING_MODE::OBSERVING ? "Observing" : "Proposing");
statefs::init(conf::ctx.statehistdir);
statefs::init(conf::ctx.state_hist_dir);
if (p2p::init() != 0 || usr::init() != 0 || cons::init() != 0)
return -1;

24
src/p2p/p2p.hpp
View File

@@ -59,26 +59,30 @@ struct npl_message
std::string data;
};
// Represents a state request sent to a peer.
struct state_request
{
std::string parent_path;
bool is_file;
int32_t block_id;
hasher::B2H expected_hash;
std::string parent_path; // The requested file or dir path.
bool is_file; // Whether the path is a file or dir.
int32_t block_id; // Block id of the file if we are requesting for file block. Otherwise -1.
hasher::B2H expected_hash; // The expected hash of the requested result.
};
// Represents state file system entry.
struct state_fs_hash_entry
{
bool is_file;
hasher::B2H hash;
bool is_file; // Whether this is a file or dir.
hasher::B2H hash; // Hash of the file or dir.
};
// Represents a file block data resposne.
struct block_response
{
std::string path;
uint32_t block_id;
std::string_view data;
hasher::B2H hash;
std::string path; // Path of the file.
uint32_t block_id; // Id of the block where the data belongs to.
std::string_view data; // The block data.
hasher::B2H hash; // Hash of the bloc data.
};
struct message_collection

6
src/proc/proc.cpp
View File

@@ -182,7 +182,7 @@ int start_state_monitor()
// Fill process args.
char *execv_args[4];
execv_args[0] = conf::ctx.statemonexepath.data();
execv_args[1] = conf::ctx.statehistdir.data();
execv_args[1] = conf::ctx.state_hist_dir.data();
execv_args[2] = conf::ctx.statedir.data();
execv_args[3] = NULL;
@@ -215,8 +215,8 @@ int stop_state_monitor()
// Update the hash tree.
hasher::B2H statehash = hasher::B2H_empty;
statefs::hashtree_builder htreebuilder(statefs::get_statedir_context());
if (htreebuilder.generate(statehash) != 0)
statefs::hashtree_builder htree_builder(statefs::get_state_dir_context());
if (htree_builder.generate(statehash) != 0)
return -1;
cons::ctx.curr_hash_state = std::string(reinterpret_cast<const char *>(&statehash), hasher::HASH_SIZE);

322
src/statefs/hashmap_builder.cpp
View File

@@ -7,11 +7,23 @@
namespace statefs
{
hashmap_builder::hashmap_builder(const statedir_context &ctx) : ctx(ctx)
/**
* Hashmap builder class is responsible for updating file hash based on the modified blocks of a file.
*/
hashmap_builder::hashmap_builder(const state_dir_context &ctx) : ctx(ctx)
{
}
int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath, const std::string filerelpath, const std::map<uint32_t, hasher::B2H> &changedblocks)
/**
* Generates/updates the block hash map for a file and updates the parent dir hash accordingly as well.
* @param parent_dir_hash Hash of the parent directory. This will be updated of the file hash was updated.
* @param filepath Full path to the actual state file.
* @param file_relpath The relative path to the state file from the state data directory.
* @param changed_blocks Index of changed blocks and the new hashes to be used as a hint.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::generate_hashmap_for_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &file_relpath, const std::map<uint32_t, hasher::B2H> &changed_blocks)
{
// We attempt to avoid a full rebuild of the block hash map file when possible.
// For this optimisation, both the block hash map (.bhmap) file and the
@@ -30,39 +42,39 @@ int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const
LOG_ERR << errno << ": Open failed " << filepath;
return -1;
}
const off_t filelength = lseek(orifd, 0, SEEK_END);
uint32_t blockcount = ceil((double)filelength / (double)BLOCK_SIZE);
const off_t file_length = lseek(orifd, 0, SEEK_END);
const uint32_t block_count = ceil((double)file_length / (double)BLOCK_SIZE);
// Attempt to read the existing block hash map file.
std::string bhmapfile;
std::vector<char> bhmapdata;
if (read_blockhashmap(bhmapdata, bhmapfile, filerelpath) == -1)
std::string bhmap_file;
std::vector<char> bhmap_data;
if (read_block_hashmap(bhmap_data, bhmap_file, file_relpath) == -1)
{
close(orifd);
return -1;
}
hasher::B2H oldfilehash = hasher::B2H_empty;
if (!bhmapdata.empty())
memcpy(&oldfilehash, bhmapdata.data(), hasher::HASH_SIZE);
hasher::B2H old_file_hash = hasher::B2H_empty;
if (!bhmap_data.empty())
memcpy(&old_file_hash, bhmap_data.data(), hasher::HASH_SIZE);
// Array to contain the updated block hashes. Slot 0 is for the root hash.
// Allocating hash array on the heap to avoid filling limited stack space.
std::unique_ptr<hasher::B2H[]> hashes = std::make_unique<hasher::B2H[]>(1 + blockcount);
const size_t hashes_size = (1 + blockcount) * hasher::HASH_SIZE;
std::unique_ptr<hasher::B2H[]> hashes = std::make_unique<hasher::B2H[]>(1 + block_count);
const size_t hashes_size = (1 + block_count) * hasher::HASH_SIZE;
if (changedblocks.empty())
if (changed_blocks.empty())
{
// Attempt to read the delta block index file.
std::map<uint32_t, hasher::B2H> bindex;
uint32_t original_blockcount;
if (get_blockindex(bindex, original_blockcount, filerelpath) == -1)
uint32_t original_block_count;
if (get_delta_block_index(bindex, original_block_count, file_relpath) == -1)
{
close(orifd);
return -1;
}
if (update_hashes_with_backup_blockhints(hashes.get(), hashes_size, filerelpath, orifd, blockcount, original_blockcount, bindex, bhmapdata) == -1)
if (update_hashes_with_backup_block_hints(hashes.get(), hashes_size, file_relpath, orifd, block_count, original_block_count, bindex, bhmap_data) == -1)
{
close(orifd);
return -1;
@@ -70,7 +82,7 @@ int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const
}
else
{
if (update_hashes_with_changed_blockhints(hashes.get(), hashes_size, filerelpath, orifd, blockcount, changedblocks, bhmapdata) == -1)
if (update_hashes_with_changed_block_hints(hashes.get(), hashes_size, file_relpath, orifd, block_count, changed_blocks, bhmap_data) == -1)
{
close(orifd);
return -1;
@@ -79,35 +91,42 @@ int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const
close(orifd);
if (write_blockhashmap(bhmapfile, hashes.get(), hashes_size) == -1)
if (write_block_hashmap(bhmap_file, hashes.get(), hashes_size) == -1)
return -1;
if (update_hashtree_entry(parentdirhash, !bhmapdata.empty(), oldfilehash, hashes[0], bhmapfile, filerelpath) == -1)
if (update_hashtree_entry(parent_dir_hash, !bhmap_data.empty(), old_file_hash, hashes[0], bhmap_file, file_relpath) == -1)
return -1;
return 0;
}
int hashmap_builder::read_blockhashmap(std::vector<char> &bhmapdata, std::string &bhmapfile, const std::string &relpath)
/**
* Reads the block hash map of a given data file into the provided vector.
* @param bhmap_data Vector to copy the block hash map contents.
* @param bhmap_file The full path to the block hash map file pointed to by the relative path.
* @param relpath The relative path of the actual data file.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::read_block_hashmap(std::vector<char> &bhmap_data, std::string &bhmap_file, const std::string &relpath)
{
bhmapfile.reserve(ctx.blockhashmapdir.length() + relpath.length() + HASHMAP_EXT_LEN);
bhmapfile.append(ctx.blockhashmapdir).append(relpath).append(HASHMAP_EXT);
bhmap_file.reserve(ctx.block_hashmap_dir.length() + relpath.length() + BLOCK_HASHMAP_EXT_LEN);
bhmap_file.append(ctx.block_hashmap_dir).append(relpath).append(BLOCK_HASHMAP_EXT);
if (boost::filesystem::exists(bhmapfile))
if (boost::filesystem::exists(bhmap_file))
{
int hmapfd = open(bhmapfile.c_str(), O_RDONLY);
int hmapfd = open(bhmap_file.c_str(), O_RDONLY);
if (hmapfd == -1)
{
LOG_ERR << errno << ": Open failed " << bhmapfile;
LOG_ERR << errno << ": Open failed " << bhmap_file;
return -1;
}
off_t size = lseek(hmapfd, 0, SEEK_END);
bhmapdata.resize(size);
bhmap_data.resize(size);
if (pread(hmapfd, bhmapdata.data(), size, 0) == -1)
if (pread(hmapfd, bhmap_data.data(), size, 0) == -1)
{
LOG_ERR << errno << ": Read failed " << bhmapfile;
LOG_ERR << errno << ": Read failed " << bhmap_file;
close(hmapfd);
return -1;
}
@@ -117,7 +136,7 @@ int hashmap_builder::read_blockhashmap(std::vector<char> &bhmapdata, std::string
else
{
// Create directory tree if not exist so we are able to create the hashmap files.
boost::filesystem::path hmapsubdir = boost::filesystem::path(bhmapfile).parent_path();
boost::filesystem::path hmapsubdir = boost::filesystem::path(bhmap_file).parent_path();
if (created_bhmapsubdirs.count(hmapsubdir.string()) == 0)
{
boost::filesystem::create_directories(hmapsubdir);
@@ -128,99 +147,118 @@ int hashmap_builder::read_blockhashmap(std::vector<char> &bhmapdata, std::string
return 0;
}
int hashmap_builder::get_blockindex(std::map<uint32_t, hasher::B2H> &idxmap, uint32_t &totalblockcount, const std::string &filerelpath)
/**
* Reads the delta block index of a file.
* @param idxmap Map to copy the block index contents (block id --> hash).
* @param total_block_count Reference to hold the total block count of the original data file.
* @param file_relpath Relative path to the data file.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::get_delta_block_index(std::map<uint32_t, hasher::B2H> &idxmap, uint32_t &total_block_count, const std::string &file_relpath)
{
std::string bindexfile;
bindexfile.reserve(ctx.deltadir.length() + filerelpath.length() + BLOCKINDEX_EXT_LEN);
bindexfile.append(ctx.deltadir).append(filerelpath).append(BLOCKINDEX_EXT);
std::string bindex_file;
bindex_file.reserve(ctx.delta_dir.length() + file_relpath.length() + BLOCK_INDEX_EXT_LEN);
bindex_file.append(ctx.delta_dir).append(file_relpath).append(BLOCK_INDEX_EXT);
if (boost::filesystem::exists(bindexfile))
if (boost::filesystem::exists(bindex_file))
{
std::ifstream infile(bindexfile, std::ios::binary | std::ios::ate);
std::streamsize idxsize = infile.tellg();
infile.seekg(0, std::ios::beg);
std::ifstream in_file(bindex_file, std::ios::binary | std::ios::ate);
std::streamsize idx_size = in_file.tellg();
in_file.seekg(0, std::ios::beg);
// Read the block index file into a vector.
std::vector<char> bindex(idxsize);
if (infile.read(bindex.data(), idxsize))
std::vector<char> bindex(idx_size);
if (in_file.read(bindex.data(), idx_size))
{
// First 8 bytes contain the original file length.
off_t orifilelen;
memcpy(&orifilelen, bindex.data(), 8);
totalblockcount = ceil((double)orifilelen / (double)BLOCK_SIZE);
total_block_count = ceil((double)orifilelen / (double)BLOCK_SIZE);
// Skip the first 8 bytes and loop through index entries.
for (uint32_t idxoffset = 8; idxoffset < bindex.size();)
for (uint32_t idx_offset = 8; idx_offset < bindex.size();)
{
// Read the block no. (4 bytes) of where this block is from in the original file.
uint32_t blockno = 0;
memcpy(&blockno, bindex.data() + idxoffset, 4);
idxoffset += 12; // Skip the cached block offset (8 bytes)
uint32_t block_no = 0;
memcpy(&block_no, bindex.data() + idx_offset, 4);
idx_offset += 12; // Skip the cached block offset (8 bytes)
// Read the block hash (32 bytes).
hasher::B2H hash;
memcpy(&hash, bindex.data() + idxoffset, 32);
idxoffset += 32;
memcpy(&hash, bindex.data() + idx_offset, 32);
idx_offset += 32;
idxmap.try_emplace(blockno, hash);
idxmap.try_emplace(block_no, hash);
}
}
else
{
LOG_ERR << errno << ": Read failed " << bindexfile;
LOG_ERR << errno << ": Read failed " << bindex_file;
return -1;
}
infile.close();
in_file.close();
}
return 0;
}
int hashmap_builder::update_hashes_with_backup_blockhints(
hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd,
const uint32_t blockcount, const uint32_t original_blockcount, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmapdata)
/**
* Updates the hash map with the use of delta backup block ids.
* @param hashes Pointer to the hash array to copy the block hashes after the update.
* @param hashes_size Byte length of the hashes array.
* @param relpath Relative path of the data file.
* @param orifd An open file descriptor to the data file.
* @param block_count Block count of the updated file.
* @param original_block_count Original block count before the update.
* @param bindex Delta backup block index map.
* @param bhmap_data Contents of the existing block hash map.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::update_hashes_with_backup_block_hints(
hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, const uint32_t block_count,
const uint32_t original_block_count, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmap_data)
{
uint32_t nohint_blockstart = 0;
// If both existing delta block index and block hash map is available, we can just overlay the
// changed block hashes (mentioned in the delta block index) on top of the old block hashes.
// This would prevent unncessarily hashing lot of blocks.
if (!bhmapdata.empty() && !bindex.empty())
if (!bhmap_data.empty() && !bindex.empty())
{
// Load old hashes.
memcpy(hashes, bhmapdata.data(), hashes_size < bhmapdata.size() ? hashes_size : bhmapdata.size());
memcpy(hashes, bhmap_data.data(), hashes_size < bhmap_data.size() ? hashes_size : bhmap_data.size());
// Refer to the block index and rehash the changed blocks.
for (const auto [blockid, oldhash] : bindex)
for (const auto [block_id, old_hash] : bindex)
{
// If the blockid from the block index is no longer there, that means the current file is
// If the block_id from the block index is no longer there, that means the current file is
// shorter than the previous version. So we can stop hashing at this point.
if (blockid >= blockcount)
if (block_id >= block_count)
break;
if (compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1)
if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1)
return -1;
}
// If the current file has more blocks than the previous version, we need to hash those
// additional blocks as well.
if (blockcount > original_blockcount)
nohint_blockstart = original_blockcount;
if (block_count > original_block_count)
nohint_blockstart = original_block_count;
else
nohint_blockstart = blockcount; // No additional blocks remaining.
nohint_blockstart = block_count; // No additional blocks remaining.
}
//Hash any additional blocks that has to be hashed without the guidance of block index.
for (uint32_t blockid = nohint_blockstart; blockid < blockcount; blockid++)
for (uint32_t block_id = nohint_blockstart; block_id < block_count; block_id++)
{
if (compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1)
if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1)
return -1;
}
// Calculate the new file hash: filehash = HASH(filename + XOR(block hashes))
hasher::B2H filehash = hasher::B2H_empty;
for (int i = 1; i <= blockcount; i++)
for (int i = 1; i <= block_count; i++)
filehash ^= hashes[i];
// Rehash the file hash with filename included.
@@ -231,9 +269,20 @@ int hashmap_builder::update_hashes_with_backup_blockhints(
return 0;
}
int hashmap_builder::update_hashes_with_changed_blockhints(
/**
* Updates the hash map with the use of list of updated block ids.
* @param hashes Pointer to the hash array to copy the block hashes after the update.
* @param hashes_size Byte length of the hashes array.
* @param relpath Relative path of the data file.
* @param orifd An open file descriptor to the data file.
* @param block_count Block count of the updated file.
* @param bindex Map of updated block ids and new hashes.
* @param bhmap_data Contents of the existing block hash map.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::update_hashes_with_changed_block_hints(
hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd,
const uint32_t blockcount, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmapdata)
const uint32_t block_count, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmap_data)
{
// If both existing delta block index and block hash map is available, we can just overlay the
// changed block hashes (mentioned in the delta block index) on top of the old block hashes.
@@ -242,19 +291,19 @@ int hashmap_builder::update_hashes_with_changed_blockhints(
if (!bindex.empty())
{
// Load old hashes if exists.
if (!bhmapdata.empty())
memcpy(hashes, bhmapdata.data(), hashes_size < bhmapdata.size() ? hashes_size : bhmapdata.size());
if (!bhmap_data.empty())
memcpy(hashes, bhmap_data.data(), hashes_size < bhmap_data.size() ? hashes_size : bhmap_data.size());
// Refer to the block index and overlay the new hash into the hashes array.
for (const auto [blockid, newhash] : bindex)
hashes[blockid + 1] = newhash;
for (const auto [block_id, new_hash] : bindex)
hashes[block_id + 1] = new_hash;
// If the block hash map didn't existed, we need to calculate and fill the unchanged block hashes from the actual file.
if (bhmapdata.empty())
if (bhmap_data.empty())
{
for (uint32_t blockid = 0; blockid < blockcount; blockid++)
for (uint32_t block_id = 0; block_id < block_count; block_id++)
{
if (bindex.count(blockid) == 0 && compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1)
if (bindex.count(block_id) == 0 && compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1)
return -1;
}
}
@@ -262,16 +311,16 @@ int hashmap_builder::update_hashes_with_changed_blockhints(
else
{
// If we don't have the changed block index, we have to hash the entire file blocks again.
for (uint32_t blockid = 0; blockid < blockcount; blockid++)
for (uint32_t block_id = 0; block_id < block_count; block_id++)
{
if (compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1)
if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1)
return -1;
}
}
// Calculate the new file hash: filehash = HASH(filename + XOR(block hashes))
hasher::B2H filehash = hasher::B2H_empty;
for (int i = 1; i <= blockcount; i++)
for (int i = 1; i <= block_count; i++)
filehash ^= hashes[i];
// Rehash the file hash with filename included.
@@ -282,42 +331,57 @@ int hashmap_builder::update_hashes_with_changed_blockhints(
return 0;
}
int hashmap_builder::compute_blockhash(hasher::B2H &hash, uint32_t blockid, int filefd, const std::string &relpath)
/**
* Calculates the hash of the specified block id of a file.
* @param hash Reference to assign the calculated hash.
* @param block_id Id of the block to be hashed.
* @param filefd Open file descriptor for the state data file.
* @param relpath Relative path of the state data file.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::compute_blockhash(hasher::B2H &hash, const uint32_t block_id, const int filefd, const std::string &relpath)
{
// Allocating block buffer on the heap to avoid filling limited stack space.
std::unique_ptr<char[]> blockbuf = std::make_unique<char[]>(BLOCK_SIZE);
const off_t blockoffset = BLOCK_SIZE * blockid;
size_t bytesread = pread(filefd, blockbuf.get(), BLOCK_SIZE, blockoffset);
if (bytesread == -1)
std::unique_ptr<char[]> block_buf = std::make_unique<char[]>(BLOCK_SIZE);
const off_t block_offset = BLOCK_SIZE * block_id;
size_t bytes_read = pread(filefd, block_buf.get(), BLOCK_SIZE, block_offset);
if (bytes_read == -1)
{
LOG_ERR << errno << ": Read failed " << relpath;
return -1;
}
hash = hasher::hash(&blockoffset, 8, blockbuf.get(), bytesread);
hash = hasher::hash(&block_offset, 8, block_buf.get(), bytes_read);
return 0;
}
int hashmap_builder::write_blockhashmap(const std::string &bhmapfile, const hasher::B2H *hashes, const off_t hashes_size)
/**
* Saves the block hash map into the relevant .bhmap file.
* @param bhmap_file Full path to the block hash map file.
* @param hashes Pointer to the hashes array containing the root hash and block hashes.
* @param hashes_size Byte length of the hashes array.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::write_block_hashmap(const std::string &bhmap_file, const hasher::B2H *hashes, const off_t hashes_size)
{
int hmapfd = open(bhmapfile.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS);
int hmapfd = open(bhmap_file.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS);
if (hmapfd == -1)
{
LOG_ERR << errno << ": Open failed " << bhmapfile;
LOG_ERR << errno << ": Open failed " << bhmap_file;
return -1;
}
// Write the updated hash list into the block hash map file.
if (pwrite(hmapfd, hashes, hashes_size, 0) == -1)
{
LOG_ERR << errno << ": Write failed " << bhmapfile;
LOG_ERR << errno << ": Write failed " << bhmap_file;
close(hmapfd);
return -1;
}
if (ftruncate(hmapfd, hashes_size) == -1)
{
LOG_ERR << errno << ": Truncate failed " << bhmapfile;
LOG_ERR << errno << ": Truncate failed " << bhmap_file;
close(hmapfd);
return -1;
}
@@ -325,91 +389,113 @@ int hashmap_builder::write_blockhashmap(const std::string &bhmapfile, const hash
close(hmapfd);
}
int hashmap_builder::update_hashtree_entry(hasher::B2H &parentdirhash, const bool oldbhmap_exists, const hasher::B2H oldfilehash, const hasher::B2H newfilehash, const std::string &bhmapfile, const std::string &relpath)
/**
* Updates a file hash and adjust parent dir hash of the hash tree.
* @param parent_dir_hash Current hash of the parent dir. This will be assigned the new hash after the update.
* @param old_bhmap_exists Whether the block hash map file already exists or not.
* @param old_file_hash Old file hash. (0000 if this is a new file)
* @param new_file_hash New file hash.
* @param bhmap_file Full path to the block hash map file.
* @param relpath Relative path to the state data file.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash,
const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath)
{
std::string hardlinkdir(ctx.hashtreedir);
const std::string relpathdir = boost::filesystem::path(relpath).parent_path().string();
std::string hardlink_dir(ctx.hashtree_dir);
const std::string relpath_dir = boost::filesystem::path(relpath).parent_path().string();
hardlinkdir.append(relpathdir);
if (relpathdir != "/")
hardlinkdir.append("/");
hardlink_dir.append(relpath_dir);
if (relpath_dir != "/")
hardlink_dir.append("/");
std::stringstream newhlpath;
newhlpath << hardlinkdir << newfilehash << ".rh";
std::stringstream new_hl_path;
new_hl_path << hardlink_dir << new_file_hash << ".rh";
if (oldbhmap_exists)
// TODO: Even though we maintain hardlinks named after the file hash, we don't actually utilize them elsewhere.
// The intention is to be able to get a hash listing of the entire directory. Such ability is useful to serve state
// requests. However since state requests need the file name along with the hash we have to resort to iterating each
// .bhmap file and reading the file hash from first 32 bytes.
if (old_bhmap_exists)
{
// Rename the existing hard link if old block hash map existed.
// We thereby assume the old hard link also existed.
std::stringstream oldhlpath;
oldhlpath << hardlinkdir << oldfilehash << ".rh";
if (rename(oldhlpath.str().c_str(), newhlpath.str().c_str()) == -1)
oldhlpath << hardlink_dir << old_file_hash << ".rh";
if (rename(oldhlpath.str().c_str(), new_hl_path.str().c_str()) == -1)
return -1;
// Subtract the old root hash and add the new root hash from the parent dir hash.
parentdirhash ^= oldfilehash;
parentdirhash ^= newfilehash;
parent_dir_hash ^= old_file_hash;
parent_dir_hash ^= new_file_hash;
}
else
{
// Create a new hard link with new root hash as the name.
if (link(bhmapfile.c_str(), newhlpath.str().c_str()) == -1)
if (link(bhmap_file.c_str(), new_hl_path.str().c_str()) == -1)
return -1;
// Add the new root hash to parent hash.
parentdirhash ^= newfilehash;
parent_dir_hash ^= new_file_hash;
}
return 0;
}
int hashmap_builder::remove_hashmapfile(hasher::B2H &parentdirhash, const std::string &bhmapfile)
/**
* Removes an existing block hash map file. Caled when deleting a state data file.
* @param parent_dir_hash Current hash of the parent dir. This will be assigned the new hash after the update.
* @param Full path to the block hash map file.
* @return 0 on success. -1 on failure.
*/
int hashmap_builder::remove_hashmap_file(hasher::B2H &parent_dir_hash, const std::string &bhmap_file)
{
if (boost::filesystem::exists(bhmapfile))
if (boost::filesystem::exists(bhmap_file))
{
int hmapfd = open(bhmapfile.data(), O_RDONLY);
int hmapfd = open(bhmap_file.data(), O_RDONLY);
if (hmapfd == -1)
{
LOG_ERR << errno << ": Open failed " << bhmapfile;
LOG_ERR << errno << ": Open failed " << bhmap_file;
return -1;
}
hasher::B2H filehash;
if (read(hmapfd, &filehash, hasher::HASH_SIZE) == -1)
{
LOG_ERR << errno << ": Read failed " << bhmapfile;
LOG_ERR << errno << ": Read failed " << bhmap_file;
close(hmapfd);
return -1;
}
// Delete the .bhmap file.
if (remove(bhmapfile.c_str()) == -1)
if (remove(bhmap_file.c_str()) == -1)
{
LOG_ERR << errno << ": Delete failed " << bhmapfile;
LOG_ERR << errno << ": Delete failed " << bhmap_file;
close(hmapfd);
return -1;
}
// Delete the hardlink of the .bhmap file.
std::string hardlinkdir(ctx.hashtreedir);
const std::string relpath = get_relpath(bhmapfile, ctx.blockhashmapdir);
const std::string relpathdir = boost::filesystem::path(relpath).parent_path().string();
std::string hardlink_dir(ctx.hashtree_dir);
const std::string relpath = get_relpath(bhmap_file, ctx.block_hashmap_dir);
const std::string relpath_dir = boost::filesystem::path(relpath).parent_path().string();
hardlinkdir.append(relpathdir);
if (relpathdir != "/")
hardlinkdir.append("/");
hardlink_dir.append(relpath_dir);
if (relpath_dir != "/")
hardlink_dir.append("/");
std::stringstream hlpath;
hlpath << hardlinkdir << filehash << ".rh";
hlpath << hardlink_dir << filehash << ".rh";
if (remove(hlpath.str().c_str()) == -1)
{
LOG_ERR << errno << ": Delete failed for halrd link " << filehash << " of " << bhmapfile;
LOG_ERR << errno << ": Delete failed for hard link " << filehash << " of " << bhmap_file;
close(hmapfd);
return -1;
}
// XOR parent dir hash with file hash so the file hash gets removed from parent dir hash.
parentdirhash ^= filehash;
parent_dir_hash ^= filehash;
close(hmapfd);
}

26
src/statefs/hashmap_builder.hpp
View File

@@ -11,26 +11,26 @@ namespace statefs
class hashmap_builder
{
private:
const statedir_context ctx;
const state_dir_context ctx;
// List of new block hash map sub directories created during the session.
std::unordered_set<std::string> created_bhmapsubdirs;
int read_blockhashmap(std::vector<char> &bhmapdata, std::string &hmapfile, const std::string &relpath);
int get_blockindex(std::map<uint32_t, hasher::B2H> &idxmap, uint32_t &totalblockcount, const std::string &filerelpath);
int update_hashes_with_backup_blockhints(
int read_block_hashmap(std::vector<char> &bhmap_data, std::string &hmapfile, const std::string &relpath);
int get_delta_block_index(std::map<uint32_t, hasher::B2H> &idxmap, uint32_t &total_block_count, const std::string &file_relpath);
int update_hashes_with_backup_block_hints(
hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd,
const uint32_t blockcount, const uint32_t original_blockcount, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmapdata);
int update_hashes_with_changed_blockhints(
const uint32_t block_count, const uint32_t original_block_count, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmap_data);
int update_hashes_with_changed_block_hints(
hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd,
const uint32_t blockcount, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmapdata);
int compute_blockhash(hasher::B2H &hash, uint32_t blockid, int filefd, const std::string &relpath);
int write_blockhashmap(const std::string &bhmapfile, const hasher::B2H *hashes, const off_t hashes_size);
int update_hashtree_entry(hasher::B2H &parentdirhash, const bool oldbhmap_exists, const hasher::B2H oldfilehash, const hasher::B2H newfilehash, const std::string &bhmapfile, const std::string &relpath);
const uint32_t block_count, const std::map<uint32_t, hasher::B2H> &bindex, const std::vector<char> &bhmap_data);
int compute_blockhash(hasher::B2H &hash, const uint32_t block_id, const int filefd, const std::string &relpath);
int write_block_hashmap(const std::string &bhmap_file, const hasher::B2H *hashes, const off_t hashes_size);
int update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash, const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath);
public:
hashmap_builder(const statedir_context &ctx);
int generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath, const std::string filerelpath, const std::map<uint32_t, hasher::B2H> &changedblocks);
int remove_hashmapfile(hasher::B2H &parentdirhash, const std::string &filepath);
hashmap_builder(const state_dir_context &ctx);
int generate_hashmap_for_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &file_relpath, const std::map<uint32_t, hasher::B2H> &changed_blocks);
int remove_hashmap_file(hasher::B2H &parent_dir_hash, const std::string &filepath);
};
} // namespace statefs

204
src/statefs/hashtree_builder.cpp
View File

@@ -6,187 +6,187 @@
namespace statefs
{
hashtree_builder::hashtree_builder(const statedir_context &ctx) : ctx(ctx), hmapbuilder(ctx)
hashtree_builder::hashtree_builder(const state_dir_context &ctx) : ctx(ctx), hmapbuilder(ctx)
{
force_rebuild_all = false;
hintmode = false;
hint_mode = false;
}
int hashtree_builder::generate(hasher::B2H &roothash)
int hashtree_builder::generate(hasher::B2H &root_hash)
{
// Load modified file path hints if available.
populate_hintpaths_from_idxfile(IDX_TOUCHEDFILES);
populate_hintpaths_from_idxfile(IDX_NEWFILES);
hintmode = !hintpaths.empty();
populate_hint_paths_from_idx_file(IDX_TOUCHED_FILES);
populate_hint_paths_from_idx_file(IDX_NEW_FILES);
hint_mode = !hint_paths.empty();
return traverse_and_generate(roothash);
return traverse_and_generate(root_hash);
}
int hashtree_builder::generate(hasher::B2H &roothash, const bool force_all)
int hashtree_builder::generate(hasher::B2H &root_hash, const bool force_all)
{
force_rebuild_all = force_all;
if (force_rebuild_all)
{
boost::filesystem::remove_all(ctx.blockhashmapdir);
boost::filesystem::remove_all(ctx.hashtreedir);
boost::filesystem::remove_all(ctx.block_hashmap_dir);
boost::filesystem::remove_all(ctx.hashtree_dir);
boost::filesystem::create_directories(ctx.blockhashmapdir);
boost::filesystem::create_directories(ctx.hashtreedir);
boost::filesystem::create_directories(ctx.block_hashmap_dir);
boost::filesystem::create_directories(ctx.hashtree_dir);
}
return traverse_and_generate(roothash);
return traverse_and_generate(root_hash);
}
int hashtree_builder::generate(hasher::B2H &roothash, const std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> &touchedfiles)
int hashtree_builder::generate(hasher::B2H &root_hash, const std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> &touched_files)
{
hintmode = true;
fileblockindex = touchedfiles;
for (const auto &[relpath, bindex] : touchedfiles)
insert_hintpath(relpath);
hint_mode = true;
file_block_index = touched_files;
for (const auto &[relpath, bindex] : touched_files)
insert_hint_path(relpath);
return traverse_and_generate(roothash);
return traverse_and_generate(root_hash);
}
int hashtree_builder::traverse_and_generate(hasher::B2H &roothash)
int hashtree_builder::traverse_and_generate(hasher::B2H &root_hash)
{
// Load current root hash if exist.
const std::string dirhashfile = ctx.hashtreedir + "/" + DIRHASH_FNAME;
roothash = get_existingdirhash(dirhashfile);
const std::string dir_hash_file = ctx.hashtree_dir + "/" + DIR_HASH_FNAME;
root_hash = get_existing_dir_hash(dir_hash_file);
traversel_rootdir = ctx.datadir;
traversel_rootdir = ctx.data_dir;
removal_mode = false;
if (update_hashtree(roothash) != 0)
if (update_hashtree(root_hash) != 0)
return -1;
// If there are any remaining hint files directly under this directory, that means
// those files are no longer there. So we need to delete the corresponding .bhmap and rh files
// and adjust the directory hash accordingly.
if (hintmode && !hintpaths.empty())
if (hint_mode && !hint_paths.empty())
{
traversel_rootdir = ctx.blockhashmapdir;
traversel_rootdir = ctx.block_hashmap_dir;
removal_mode = true;
if (update_hashtree(roothash) != 0)
if (update_hashtree(root_hash) != 0)
return -1;
}
return 0;
}
int hashtree_builder::update_hashtree(hasher::B2H &roothash)
int hashtree_builder::update_hashtree(hasher::B2H &root_hash)
{
hintpath_map::iterator hintdir_itr = hintpaths.end();
if (!should_process_dir(hintdir_itr, traversel_rootdir))
hintpath_map::iterator hint_dir_itr = hint_paths.end();
if (!should_process_dir(hint_dir_itr, traversel_rootdir))
return 0;
if (update_hashtree_fordir(roothash, traversel_rootdir, hintdir_itr, true) != 0)
if (update_hashtree_fordir(root_hash, traversel_rootdir, hint_dir_itr, true) != 0)
return -1;
return 0;
}
int hashtree_builder::update_hashtree_fordir(hasher::B2H &parentdirhash, const std::string &dirpath, const hintpath_map::iterator hintdir_itr, const bool isrootlevel)
int hashtree_builder::update_hashtree_fordir(hasher::B2H &parent_dir_hash, const std::string &dirpath, const hintpath_map::iterator hint_dir_itr, const bool is_root_level)
{
const std::string htreedirpath = switch_basepath(dirpath, traversel_rootdir, ctx.hashtreedir);
const std::string htree_dirpath = switch_base_path(dirpath, traversel_rootdir, ctx.hashtree_dir);
// Load current dir hash if exist.
const std::string dirhashfile = htreedirpath + "/" + DIRHASH_FNAME;
hasher::B2H dirhash = get_existingdirhash(dirhashfile);
const std::string dir_hash_file = htree_dirpath + "/" + DIR_HASH_FNAME;
hasher::B2H dir_hash = get_existing_dir_hash(dir_hash_file);
// Remember the dir hash before we mutate it.
hasher::B2H original_dirhash = dirhash;
hasher::B2H original_dir_hash = dir_hash;
// Iterate files/subdirs inside this dir.
const boost::filesystem::directory_iterator itrend;
for (boost::filesystem::directory_iterator itr(dirpath); itr != itrend; itr++)
const boost::filesystem::directory_iterator itr_end;
for (boost::filesystem::directory_iterator itr(dirpath); itr != itr_end; itr++)
{
const bool isdir = boost::filesystem::is_directory(itr->path());
const std::string pathstr = itr->path().string();
const bool is_dir = boost::filesystem::is_directory(itr->path());
const std::string path_str = itr->path().string();
if (isdir)
if (is_dir)
{
hintpath_map::iterator hintsubdir_itr = hintpaths.end();
if (!should_process_dir(hintsubdir_itr, pathstr))
hintpath_map::iterator hint_subdir_itr = hint_paths.end();
if (!should_process_dir(hint_subdir_itr, path_str))
continue;
if (update_hashtree_fordir(dirhash, pathstr, hintsubdir_itr, false) != 0)
if (update_hashtree_fordir(dir_hash, path_str, hint_subdir_itr, false) != 0)
return -1;
}
else
{
if (!should_process_file(hintdir_itr, pathstr))
if (!should_process_file(hint_dir_itr, path_str))
continue;
if (process_file(dirhash, pathstr, htreedirpath) != 0)
if (process_file(dir_hash, path_str, htree_dirpath) != 0)
return -1;
}
}
// If there are no more files in the hint dir, delete the hint dir entry as well.
if (hintdir_itr != hintpaths.end() && hintdir_itr->second.empty())
hintpaths.erase(hintdir_itr);
if (hint_dir_itr != hint_paths.end() && hint_dir_itr->second.empty())
hint_paths.erase(hint_dir_itr);
// In removalmode, we check whether the dir is empty. If so we remove the dir as well.
if (removal_mode && boost::filesystem::is_empty(dirpath))
{
// We remove the dirs if we are below root level only.
// Otherwise we only remove root dir.hash file.
if (!isrootlevel)
if (!is_root_level)
{
boost::filesystem::remove_all(dirpath);
boost::filesystem::remove_all(htreedirpath);
boost::filesystem::remove_all(htree_dirpath);
}
else
{
boost::filesystem::remove(dirhashfile);
boost::filesystem::remove(dir_hash_file);
}
// Subtract the original dir hash from the parent dir hash.
parentdirhash ^= original_dirhash;
parent_dir_hash ^= original_dir_hash;
}
else if (dirhash != original_dirhash)
else if (dir_hash != original_dir_hash)
{
// If dir hash has changed, write it back to dir hash file.
if (save_dirhash(dirhashfile, dirhash) == -1)
if (save_dir_hash(dir_hash_file, dir_hash) == -1)
return -1;
// Also update the parent dir hash by subtracting the old hash and adding the new hash.
parentdirhash ^= original_dirhash;
parentdirhash ^= dirhash;
parent_dir_hash ^= original_dir_hash;
parent_dir_hash ^= dir_hash;
}
else
{
parentdirhash = dirhash;
parent_dir_hash = dir_hash;
}
return 0;
}
hasher::B2H hashtree_builder::get_existingdirhash(const std::string &dirhashfile)
hasher::B2H hashtree_builder::get_existing_dir_hash(const std::string &dir_hash_file)
{
// Load current dir hash if exist.
hasher::B2H dirhash = hasher::B2H_empty;
int dirhashfd = open(dirhashfile.c_str(), O_RDONLY);
if (dirhashfd > 0)
hasher::B2H dir_hash = hasher::B2H_empty;
int dir_hash_fd = open(dir_hash_file.c_str(), O_RDONLY);
if (dir_hash_fd > 0)
{
read(dirhashfd, &dirhash, hasher::HASH_SIZE);
close(dirhashfd);
read(dir_hash_fd, &dir_hash, hasher::HASH_SIZE);
close(dir_hash_fd);
}
return dirhash;
return dir_hash;
}
int hashtree_builder::save_dirhash(const std::string &dirhashfile, hasher::B2H dirhash)
int hashtree_builder::save_dir_hash(const std::string &dir_hash_file, hasher::B2H dir_hash)
{
int dirhashfd = open(dirhashfile.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS);
if (dirhashfd == -1)
int dir_hash_fd = open(dir_hash_file.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS);
if (dir_hash_fd == -1)
return -1;
if (write(dirhashfd, &dirhash, hasher::HASH_SIZE) == -1)
if (write(dir_hash_fd, &dir_hash, hasher::HASH_SIZE) == -1)
{
close(dirhashfd);
close(dir_hash_fd);
return -1;
}
close(dirhashfd);
close(dir_hash_fd);
return 0;
}
@@ -195,17 +195,17 @@ inline bool hashtree_builder::should_process_dir(hintpath_map::iterator &dir_itr
if (force_rebuild_all)
return true;
return (hintmode ? get_hinteddir_match(dir_itr, dirpath) : true);
return (hint_mode ? get_hinteddir_match(dir_itr, dirpath) : true);
}
bool hashtree_builder::should_process_file(const hintpath_map::iterator hintdir_itr, const std::string filepath)
bool hashtree_builder::should_process_file(const hintpath_map::iterator hint_dir_itr, const std::string filepath)
{
if (force_rebuild_all)
return true;
if (hintmode)
if (hint_mode)
{
if (hintdir_itr == hintpaths.end())
if (hint_dir_itr == hint_paths.end())
return false;
std::string relpath = get_relpath(filepath, traversel_rootdir);
@@ -213,82 +213,82 @@ bool hashtree_builder::should_process_file(const hintpath_map::iterator hintdir_
// If in removal mode, we are traversing .bhmap files. Hence we should truncate .bhmap extension
// before we search for the path in file hints.
if (removal_mode)
relpath = relpath.substr(0, relpath.length() - HASHMAP_EXT_LEN);
relpath = relpath.substr(0, relpath.length() - BLOCK_HASHMAP_EXT_LEN);
std::unordered_set<std::string> &hintfiles = hintdir_itr->second;
const auto hintfile_itr = hintfiles.find(relpath);
if (hintfile_itr == hintfiles.end())
std::unordered_set<std::string> &hint_files = hint_dir_itr->second;
const auto hint_file_itr = hint_files.find(relpath);
if (hint_file_itr == hint_files.end())
return false;
// Erase the visiting filepath from hint files.
hintfiles.erase(hintfile_itr);
hint_files.erase(hint_file_itr);
}
return true;
}
int hashtree_builder::process_file(hasher::B2H &parentdirhash, const std::string &filepath, const std::string &htreedirpath)
int hashtree_builder::process_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &htree_dirpath)
{
if (!removal_mode)
{
// Create directory tree if not exist so we are able to create the file root hash files (hard links).
if (created_htreesubdirs.count(htreedirpath) == 0)
if (created_htree_subdirs.count(htree_dirpath) == 0)
{
boost::filesystem::create_directories(htreedirpath);
created_htreesubdirs.emplace(htreedirpath);
boost::filesystem::create_directories(htree_dirpath);
created_htree_subdirs.emplace(htree_dirpath);
}
std::string relpath = get_relpath(filepath, ctx.datadir);
std::map<uint32_t, hasher::B2H> changedblocks = fileblockindex[relpath];
const std::string relpath = get_relpath(filepath, ctx.data_dir);
const std::map<uint32_t, hasher::B2H> &changed_blocks = file_block_index[relpath];
if (hmapbuilder.generate_hashmap_forfile(parentdirhash, filepath, relpath, changedblocks) == -1)
if (hmapbuilder.generate_hashmap_for_file(parent_dir_hash, filepath, relpath, changed_blocks) == -1)
return -1;
}
else
{
if (hmapbuilder.remove_hashmapfile(parentdirhash, filepath) == -1)
if (hmapbuilder.remove_hashmap_file(parent_dir_hash, filepath) == -1)
return -1;
}
return 0;
}
void hashtree_builder::populate_hintpaths_from_idxfile(const char *const idxfile)
void hashtree_builder::populate_hint_paths_from_idx_file(const char *const idxfile)
{
std::ifstream infile(std::string(ctx.deltadir).append(idxfile));
if (!infile.fail())
std::ifstream in_file(std::string(ctx.delta_dir).append(idxfile));
if (!in_file.fail())
{
for (std::string relpath; std::getline(infile, relpath);)
insert_hintpath(relpath);
infile.close();
for (std::string relpath; std::getline(in_file, relpath);)
insert_hint_path(relpath);
in_file.close();
}
}
void hashtree_builder::insert_hintpath(const std::string &relpath)
void hashtree_builder::insert_hint_path(const std::string &relpath)
{
boost::filesystem::path p_relpath(relpath);
std::string parentdir = p_relpath.parent_path().string();
hintpaths[parentdir].emplace(relpath);
std::string parent_dir = p_relpath.parent_path().string();
hint_paths[parent_dir].emplace(relpath);
}
bool hashtree_builder::get_hinteddir_match(hintpath_map::iterator &matchitr, const std::string &dirpath)
bool hashtree_builder::get_hinteddir_match(hintpath_map::iterator &match_itr, const std::string &dirpath)
{
// First check whether there's an exact match. If not check for a partial match.
// Exact match will return the iterator. Partial match or not found will return end() iterator.
const std::string relpath = get_relpath(dirpath, traversel_rootdir);
const auto exactmatchitr = hintpaths.find(relpath);
const auto exact_match_itr = hint_paths.find(relpath);
if (exactmatchitr != hintpaths.end())
if (exact_match_itr != hint_paths.end())
{
matchitr = exactmatchitr;
match_itr = exact_match_itr;
return true;
}
for (auto itr = hintpaths.begin(); itr != hintpaths.end(); itr++)
for (auto itr = hint_paths.begin(); itr != hint_paths.end(); itr++)
{
if (strncmp(relpath.c_str(), itr->first.c_str(), relpath.length()) == 0)
{
// Partial match found.
matchitr = hintpaths.end();
match_itr = hint_paths.end();
return true;
}
}

42
src/statefs/hashtree_builder.hpp
View File

@@ -14,39 +14,39 @@ typedef std::unordered_map<std::string, std::unordered_set<std::string>> hintpat
class hashtree_builder
{
private:
const statedir_context ctx;
const state_dir_context ctx;
hashmap_builder hmapbuilder;
// Hint path map with parent dir as key and list of file paths under each parent dir.
hintpath_map hintpaths;
hintpath_map hint_paths;
bool force_rebuild_all;
bool hintmode;
bool hint_mode;
bool removal_mode;
std::string traversel_rootdir;
std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> fileblockindex;
std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> file_block_index;
// List of new root hash map sub directories created during the session.
std::unordered_set<std::string> created_htreesubdirs;
std::unordered_set<std::string> created_htree_subdirs;
int traverse_and_generate(hasher::B2H &roothash);
int update_hashtree(hasher::B2H &roothash);
int update_hashtree_fordir(hasher::B2H &parentdirhash, const std::string &relpath, const hintpath_map::iterator hintdir_itr, const bool isrootlevel);
int traverse_and_generate(hasher::B2H &root_hash);
int update_hashtree(hasher::B2H &root_hash);
int update_hashtree_fordir(hasher::B2H &parent_dir_hash, const std::string &relpath, const hintpath_map::iterator hint_dir_itr, const bool is_root_level);
hasher::B2H get_existingdirhash(const std::string &dirhashfile);
int save_dirhash(const std::string &dirhashfile, hasher::B2H dirhash);
bool should_process_dir(hintpath_map::iterator &hintsubdir_itr, const std::string &dirpath);
bool should_process_file(const hintpath_map::iterator hintdir_itr, const std::string filepath);
int process_file(hasher::B2H &parentdirhash, const std::string &filepath, const std::string &htreedirpath);
int update_hashtree_entry(hasher::B2H &parentdirhash, const bool oldbhmap_exists, const hasher::B2H oldfilehash, const hasher::B2H newfilehash, const std::string &bhmapfile, const std::string &relpath);
void populate_hintpaths_from_idxfile(const char *const idxfile);
void insert_hintpath(const std::string &relpath);
bool get_hinteddir_match(hintpath_map::iterator &matchitr, const std::string &dirpath);
hasher::B2H get_existing_dir_hash(const std::string &dir_hash_file);
int save_dir_hash(const std::string &dir_hash_file, hasher::B2H dir_hash);
bool should_process_dir(hintpath_map::iterator &hint_subdir_itr, const std::string &dirpath);
bool should_process_file(const hintpath_map::iterator hint_dir_itr, const std::string filepath);
int process_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &htree_dirpath);
int update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash, const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath);
void populate_hint_paths_from_idx_file(const char *const idxfile);
void insert_hint_path(const std::string &relpath);
bool get_hinteddir_match(hintpath_map::iterator &match_itr, const std::string &dirpath);
public:
hashtree_builder(const statedir_context &ctx);
int generate(hasher::B2H &roothash);
int generate(hasher::B2H &roothash, const bool force_all);
int generate(hasher::B2H &roothash, const std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> &touchedfiles);
hashtree_builder(const state_dir_context &ctx);
int generate(hasher::B2H &root_hash);
int generate(hasher::B2H &root_hash, const bool force_all);
int generate(hasher::B2H &root_hash, const std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> &touched_files);
};
} // namespace statefs

48
src/statefs/state_common.cpp
View File

@@ -5,41 +5,41 @@
namespace statefs
{
std::string statehistdir;
statedir_context current_ctx;
std::string state_hist_dir;
state_dir_context current_ctx;
void init(const std::string &statehist_dir_root)
void init(const std::string &state_hist_dir_root)
{
statehistdir = realpath(statehist_dir_root.c_str(), NULL);
state_hist_dir = realpath(state_hist_dir_root.c_str(), NULL);
// Initialize 0 state (current state) directory.
current_ctx = get_statedir_context(0, true);
current_ctx = get_state_dir_context(0, true);
}
std::string get_statedir_root(const int16_t checkpointid)
std::string get_state_dir_root(const int16_t checkpoint_id)
{
return statehistdir + "/" + std::to_string(checkpointid);
return state_hist_dir + "/" + std::to_string(checkpoint_id);
}
statedir_context get_statedir_context(const int16_t checkpointid, const bool createdirs)
state_dir_context get_state_dir_context(const int16_t checkpoint_id, const bool create_dirs)
{
statedir_context ctx;
ctx.rootdir = get_statedir_root(checkpointid);
ctx.datadir = ctx.rootdir + DATA_DIR;
ctx.blockhashmapdir = ctx.rootdir + BHMAP_DIR;
ctx.hashtreedir = ctx.rootdir + HTREE_DIR;
ctx.deltadir = ctx.rootdir + DELTA_DIR;
state_dir_context ctx;
ctx.root_dir = get_state_dir_root(checkpoint_id);
ctx.data_dir = ctx.root_dir + DATA_DIR;
ctx.block_hashmap_dir = ctx.root_dir + BHMAP_DIR;
ctx.hashtree_dir = ctx.root_dir + HTREE_DIR;
ctx.delta_dir = ctx.root_dir + DELTA_DIR;
if (createdirs)
if (create_dirs)
{
if (!boost::filesystem::exists(ctx.datadir))
boost::filesystem::create_directories(ctx.datadir);
if (!boost::filesystem::exists(ctx.blockhashmapdir))
boost::filesystem::create_directories(ctx.blockhashmapdir);
if (!boost::filesystem::exists(ctx.hashtreedir))
boost::filesystem::create_directories(ctx.hashtreedir);
if (!boost::filesystem::exists(ctx.deltadir))
boost::filesystem::create_directories(ctx.deltadir);
if (!boost::filesystem::exists(ctx.data_dir))
boost::filesystem::create_directories(ctx.data_dir);
if (!boost::filesystem::exists(ctx.block_hashmap_dir))
boost::filesystem::create_directories(ctx.block_hashmap_dir);
if (!boost::filesystem::exists(ctx.hashtree_dir))
boost::filesystem::create_directories(ctx.hashtree_dir);
if (!boost::filesystem::exists(ctx.delta_dir))
boost::filesystem::create_directories(ctx.delta_dir);
}
return ctx;
@@ -53,7 +53,7 @@ std::string get_relpath(const std::string &fullpath, const std::string &base_pat
return relpath;
}
std::string switch_basepath(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path)
std::string switch_base_path(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path)
{
return to_base_path + get_relpath(fullpath, from_base_path);
}

44
src/statefs/state_common.hpp
View File

@@ -15,23 +15,23 @@ constexpr int16_t MAX_CHECKPOINTS = 5;
constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB
// Cache block index entry bytes length.
constexpr size_t BLOCKINDEX_ENTRY_SIZE = 44;
constexpr size_t BLOCK_INDEX_ENTRY_SIZE = 44;
// Permissions used when creating block cache and index files.
constexpr int FILE_PERMS = 0644;
const char *const HASHMAP_EXT = ".bhmap";
constexpr size_t HASHMAP_EXT_LEN = 6;
const char *const BLOCK_HASHMAP_EXT = ".bhmap";
constexpr size_t BLOCK_HASHMAP_EXT_LEN = 6;
const char *const BLOCKINDEX_EXT = ".bindex";
constexpr size_t BLOCKINDEX_EXT_LEN = 7;
const char *const BLOCK_INDEX_EXT = ".bindex";
constexpr size_t BLOCK_INDEX_EXT_LEN = 7;
const char *const BLOCKCACHE_EXT = ".bcache";
constexpr size_t BLOCKCACHE_EXT_LEN = 7;
const char *const BLOCK_CACHE_EXT = ".bcache";
constexpr size_t BLOCK_CACHE_EXT_LEN = 7;
const char *const IDX_NEWFILES = "/idxnew.idx";
const char *const IDX_TOUCHEDFILES = "/idxtouched.idx";
const char *const DIRHASH_FNAME = "dir.hash";
const char *const IDX_NEW_FILES = "/idxnew.idx";
const char *const IDX_TOUCHED_FILES = "/idxtouched.idx";
const char *const DIR_HASH_FNAME = "dir.hash";
const char *const DATA_DIR = "/data";
const char *const BHMAP_DIR = "/bhmap";
@@ -41,26 +41,26 @@ const char *const DELTA_DIR = "/delta";
/**
* Context struct to hold all state-related directory paths.
*/
struct statedir_context
struct state_dir_context
{
std::string rootdir; // Directory holding state sub dirs.
std::string datadir; // Directory containing smart contract data.
std::string blockhashmapdir; // Directory containing block hash map files.
std::string hashtreedir; // Directory containing hash tree files (dir.hash and hard links).
std::string deltadir; // Directory containing original smart contract data.
std::string root_dir; // Directory holding state sub dirs.
std::string data_dir; // Directory containing smart contract data.
std::string block_hashmap_dir; // Directory containing block hash map files.
std::string hashtree_dir; // Directory containing hash tree files (dir.hash and hard links).
std::string delta_dir; // Directory containing original smart contract data.
};
// Container directory to contain all checkpoints.
extern std::string statehistdir;
extern std::string state_hist_dir;
// Currently loaded state checkpoint directory context (usually checkpoint 0)
extern statedir_context current_ctx;
extern state_dir_context current_ctx;
void init(const std::string &statehist_dir_root);
std::string get_statedir_root(const int16_t checkpointid);
statedir_context get_statedir_context(int16_t checkpointid = 0, bool createdirs = false);
void init(const std::string &state_hist_dir_root);
std::string get_state_dir_root(const int16_t checkpoint_id);
state_dir_context get_state_dir_context(int16_t checkpoint_id = 0, bool create_dirs = false);
std::string get_relpath(const std::string &fullpath, const std::string &base_path);
std::string switch_basepath(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path);
std::string switch_base_path(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path);
} // namespace statefs

4
src/statefs/state_monitor/fusefs.cpp
View File

@@ -1277,8 +1277,8 @@ int start(const char *arg0, const char *state_hist_dir, const char *fuse_mnt_dir
const bool is_first_run = boost::filesystem::is_empty(state_hist_dir);
statefs::init(state_hist_dir);
statemonitor.ctx = statefs::get_statedir_context();
fs.source = statemonitor.ctx.datadir;
statemonitor.ctx = statefs::get_state_dir_context();
fs.source = statemonitor.ctx.data_dir;
// Create a checkpoint from the second run onwards.
if (!is_first_run)

2
src/statefs/state_monitor/fusefs.hpp
View File

@@ -3,7 +3,7 @@
namespace fusefs
{
int start(const char *arg0, const char *source, const char *mountpoint, const char *deltadir);
int start(const char *arg0, const char *source, const char *mountpoint, const char *delta_dir);
}
#endif

62
src/statefs/state_monitor/state_monitor.cpp
View File

@@ -37,7 +37,7 @@ void state_monitor::create_checkpoint()
int16_t oldest_chkpnt = MAX_CHECKPOINTS * -1;
for (int16_t chkpnt = oldest_chkpnt; chkpnt <= -1; chkpnt++)
{
std::string dir = get_statedir_root(chkpnt);
std::string dir = get_state_dir_root(chkpnt);
if (boost::filesystem::exists(dir))
{
@@ -47,21 +47,21 @@ void state_monitor::create_checkpoint()
}
else
{
std::string dir_shift = get_statedir_root(chkpnt - 1);
std::string dir_shift = get_state_dir_root(chkpnt - 1);
boost::filesystem::rename(dir, dir_shift);
}
}
if (chkpnt == -1)
{
statedir_context ctx = get_statedir_context(0, true);
state_dir_context ctx = get_state_dir_context(0, true);
// Shift 0-state delta dir to -1.
std::string delta_1 = dir + DELTA_DIR;
boost::filesystem::create_directories(delta_1);
boost::filesystem::rename(ctx.deltadir, delta_1);
boost::filesystem::create_directories(ctx.deltadir);
boost::filesystem::rename(ctx.delta_dir, delta_1);
boost::filesystem::create_directories(ctx.delta_dir);
}
}
@@ -331,10 +331,10 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s
if (fi.is_new)
return 0;
uint32_t original_blockcount = ceil((double)fi.original_length / (double)BLOCK_SIZE);
uint32_t original_block_count = ceil((double)fi.original_length / (double)BLOCK_SIZE);
// Check whether we have already cached the entire file.
if (original_blockcount == fi.cached_blockids.size())
if (original_block_count == fi.cached_blockids.size())
return 0;
// Initialize fds and indexes required for caching the file.
@@ -342,7 +342,7 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s
return -1;
// Return if incoming write is outside any of the original blocks.
if (offset > original_blockcount * BLOCK_SIZE)
if (offset > original_block_count * BLOCK_SIZE)
return 0;
uint32_t startblock = offset / BLOCK_SIZE;
@@ -363,20 +363,20 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s
// Read the block being replaced and send to cache file.
// Allocating block buffer on the heap to avoid filling limited stack space.
std::unique_ptr<char[]> blockbuf = std::make_unique<char[]>(BLOCK_SIZE);
off_t blockoffset = BLOCK_SIZE * i;
size_t bytesread = pread(fi.readfd, blockbuf.get(), BLOCK_SIZE, BLOCK_SIZE * i);
if (bytesread < 0)
std::unique_ptr<char[]> block_buf = std::make_unique<char[]>(BLOCK_SIZE);
off_t block_offset = BLOCK_SIZE * i;
size_t bytes_read = pread(fi.readfd, block_buf.get(), BLOCK_SIZE, BLOCK_SIZE * i);
if (bytes_read < 0)
{
std::cerr << errno << ": Read failed " << fi.filepath << "\n";
return -1;
}
// No more bytes to read in this file.
if (bytesread == 0)
if (bytes_read == 0)
return 0;
if (write(fi.cachefd, blockbuf.get(), bytesread) < 0)
if (write(fi.cachefd, block_buf.get(), bytes_read) < 0)
{
std::cerr << errno << ": Write to block cache failed. " << fi.filepath << "\n";
return -1;
@@ -388,8 +388,8 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s
// Entry format: [blocknum(4 bytes) | cacheoffset(8 bytes) | blockhash(32 bytes)]
// Calculate the block hash by combining block offset with block data.
char entrybuf[BLOCKINDEX_ENTRY_SIZE];
hasher::B2H hash = hasher::hash(&blockoffset, 8, blockbuf.get(), bytesread);
char entrybuf[BLOCK_INDEX_ENTRY_SIZE];
hasher::B2H hash = hasher::hash(&block_offset, 8, block_buf.get(), bytes_read);
// Original file block id.
memcpy(entrybuf, &i, 4);
@@ -398,7 +398,7 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s
memcpy(entrybuf + 4, &cacheoffset, 8);
// The block hash.
memcpy(entrybuf + 12, hash.data, 32);
if (write(fi.indexfd, entrybuf, BLOCKINDEX_ENTRY_SIZE) < 0)
if (write(fi.indexfd, entrybuf, BLOCK_INDEX_ENTRY_SIZE) < 0)
{
std::cerr << errno << ": Write to block index failed. " << fi.filepath << "\n";
return -1;
@@ -432,11 +432,11 @@ int state_monitor::prepare_caching(state_file_info &fi)
// Get the path of the file relative to the state dir. We maintain this same reative path for the
// corresponding cache and index files in the cache dir.
std::string relpath = get_relpath(fi.filepath, ctx.datadir);
std::string relpath = get_relpath(fi.filepath, ctx.data_dir);
std::string tmppath;
tmppath.reserve(ctx.deltadir.length() + relpath.length() + BLOCKCACHE_EXT_LEN);
tmppath.append(ctx.deltadir).append(relpath).append(BLOCKCACHE_EXT);
tmppath.reserve(ctx.delta_dir.length() + relpath.length() + BLOCK_CACHE_EXT_LEN);
tmppath.append(ctx.delta_dir).append(relpath).append(BLOCK_CACHE_EXT);
// Create directory tree if not exist so we are able to create the cache and index files.
boost::filesystem::path cachesubdir = boost::filesystem::path(tmppath).parent_path();
@@ -455,7 +455,7 @@ int state_monitor::prepare_caching(state_file_info &fi)
}
// Create and open the block index file.
tmppath.replace(tmppath.length() - BLOCKCACHE_EXT_LEN, BLOCKINDEX_EXT_LEN, BLOCKINDEX_EXT);
tmppath.replace(tmppath.length() - BLOCK_CACHE_EXT_LEN, BLOCK_INDEX_EXT_LEN, BLOCK_INDEX_EXT);
fi.indexfd = open(tmppath.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS);
if (fi.indexfd <= 0)
{
@@ -501,7 +501,7 @@ int state_monitor::write_touched_file_entry(std::string_view filepath)
{
if (touched_fileindex_fd <= 0)
{
std::string index_file = ctx.deltadir + IDX_TOUCHEDFILES;
std::string index_file = ctx.delta_dir + IDX_TOUCHED_FILES;
touched_fileindex_fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS);
if (touched_fileindex_fd <= 0)
{
@@ -511,7 +511,7 @@ int state_monitor::write_touched_file_entry(std::string_view filepath)
}
// Write the relative file path line to the index.
filepath = filepath.substr(ctx.datadir.length(), filepath.length() - ctx.datadir.length());
filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length());
write(touched_fileindex_fd, filepath.data(), filepath.length());
write(touched_fileindex_fd, "\n", 1);
return 0;
@@ -523,7 +523,7 @@ int state_monitor::write_touched_file_entry(std::string_view filepath)
*/
int state_monitor::write_new_file_entry(std::string_view filepath)
{
std::string index_file = ctx.deltadir + IDX_NEWFILES;
std::string index_file = ctx.delta_dir + IDX_NEW_FILES;
int fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS);
if (fd <= 0)
{
@@ -532,7 +532,7 @@ int state_monitor::write_new_file_entry(std::string_view filepath)
}
// Write the relative file path line to the index.
filepath = filepath.substr(ctx.datadir.length(), filepath.length() - ctx.datadir.length());
filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length());
write(fd, filepath.data(), filepath.length());
write(fd, "\n", 1);
close(fd);
@@ -545,19 +545,19 @@ int state_monitor::write_new_file_entry(std::string_view filepath)
*/
void state_monitor::remove_new_file_entry(std::string_view filepath)
{
filepath = filepath.substr(ctx.datadir.length(), filepath.length() - ctx.datadir.length());
filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length());
// We create a copy of the new file index and transfer lines from first file
// to the second file except the line matching the given filepath.
std::string index_file = ctx.deltadir + IDX_NEWFILES;
std::string index_file_tmp = ctx.deltadir + IDX_NEWFILES + ".tmp";
std::string index_file = ctx.delta_dir + IDX_NEW_FILES;
std::string index_file_tmp = ctx.delta_dir + IDX_NEW_FILES + ".tmp";
std::ifstream infile(index_file);
std::ifstream in_file(index_file);
std::ofstream outfile(index_file_tmp);
bool lines_transferred = false;
for (std::string line; std::getline(infile, line);)
for (std::string line; std::getline(in_file, line);)
{
if (line != filepath) // Skip the file being removed.
{
@@ -566,7 +566,7 @@ void state_monitor::remove_new_file_entry(std::string_view filepath)
}
}
infile.close();
in_file.close();
outfile.close();
// Remove the old index.

2
src/statefs/state_monitor/state_monitor.hpp
View File

@@ -62,7 +62,7 @@ private:
void remove_new_file_entry(std::string_view filepath);
public:
statedir_context ctx;
state_dir_context ctx;
void create_checkpoint();
void oncreate(const int fd);
void onopen(const int inodefd, const int flags);

124
src/statefs/state_restore.cpp
View File

@@ -9,40 +9,40 @@ namespace statefs
{
// Look at new files added and delete them if still exist.
void state_restore::delete_newfiles()
void state_restore::delete_new_files()
{
std::string indexfile(ctx.deltadir);
indexfile.append(IDX_NEWFILES);
std::string index_file(ctx.delta_dir);
index_file.append(IDX_NEW_FILES);
std::ifstream infile(indexfile);
for (std::string file; std::getline(infile, file);)
std::ifstream in_file(index_file);
for (std::string file; std::getline(in_file, file);)
{
std::string filepath(ctx.datadir);
std::string filepath(ctx.data_dir);
filepath.append(file);
remove(filepath.c_str());
}
infile.close();
in_file.close();
}
// Look at touched files and restore them.
int state_restore::restore_touchedfiles()
int state_restore::restore_touched_files()
{
std::unordered_set<std::string> processed;
std::string indexfile(ctx.deltadir);
indexfile.append(IDX_TOUCHEDFILES);
std::string index_file(ctx.delta_dir);
index_file.append(IDX_TOUCHED_FILES);
std::ifstream infile(indexfile);
for (std::string file; std::getline(infile, file);)
std::ifstream in_file(index_file);
for (std::string file; std::getline(in_file, file);)
{
// Skip if already processed.
if (processed.count(file) > 0)
continue;
std::vector<char> bindex;
if (read_blockindex(bindex, file) != 0)
if (read_block_index(bindex, file) != 0)
return -1;
if (restore_blocks(file, bindex) != 0)
@@ -52,23 +52,23 @@ int state_restore::restore_touchedfiles()
processed.emplace(file);
}
infile.close();
in_file.close();
return 0;
}
// Read the delta block index.
int state_restore::read_blockindex(std::vector<char> &buffer, std::string_view file)
int state_restore::read_block_index(std::vector<char> &buffer, std::string_view file)
{
std::string bindexfile(ctx.deltadir);
bindexfile.append(file).append(BLOCKINDEX_EXT);
std::ifstream infile(bindexfile, std::ios::binary | std::ios::ate);
std::streamsize idxsize = infile.tellg();
infile.seekg(0, std::ios::beg);
std::string bindex_file(ctx.delta_dir);
bindex_file.append(file).append(BLOCK_INDEX_EXT);
std::ifstream in_file(bindex_file, std::ios::binary | std::ios::ate);
std::streamsize idx_size = in_file.tellg();
in_file.seekg(0, std::ios::beg);
buffer.resize(idxsize);
if (!infile.read(buffer.data(), idxsize))
buffer.resize(idx_size);
if (!in_file.read(buffer.data(), idx_size))
{
LOG_ERR << errno << ": Read failed " << bindexfile;
LOG_ERR << errno << ": Read failed " << bindex_file;
return -1;
}
@@ -78,71 +78,71 @@ int state_restore::read_blockindex(std::vector<char> &buffer, std::string_view f
// Restore blocks mentioned in the delta block index.
int state_restore::restore_blocks(std::string_view file, const std::vector<char> &bindex)
{
int bcachefd = 0, orifilefd = 0;
const char *idxptr = bindex.data();
int bcache_fd = 0, ori_file_fd = 0;
const char *idx_ptr = bindex.data();
// First 8 bytes of the index contains the supposed length of the original file.
off_t originallen = 0;
memcpy(&originallen, idxptr, 8);
off_t original_len = 0;
memcpy(&original_len, idx_ptr, 8);
// Open block cache file.
{
std::string bcachefile(ctx.deltadir);
bcachefile.append(file).append(BLOCKCACHE_EXT);
bcachefd = open(bcachefile.c_str(), O_RDONLY);
if (bcachefd <= 0)
std::string bcache_file(ctx.delta_dir);
bcache_file.append(file).append(BLOCK_CACHE_EXT);
bcache_fd = open(bcache_file.c_str(), O_RDONLY);
if (bcache_fd <= 0)
{
LOG_ERR << errno << ": Open failed " << bcachefile;
LOG_ERR << errno << ": Open failed " << bcache_file;
return -1;
}
}
// Create or Open original file.
{
std::string originalfile(ctx.datadir);
originalfile.append(file);
std::string original_file(ctx.data_dir);
original_file.append(file);
// Create directory tree if not exist so we are able to create the file.
boost::filesystem::path filedir = boost::filesystem::path(originalfile).parent_path();
boost::filesystem::path filedir = boost::filesystem::path(original_file).parent_path();
if (created_dirs.count(filedir.string()) == 0)
{
boost::filesystem::create_directories(filedir);
created_dirs.emplace(filedir.string());
}
orifilefd = open(originalfile.c_str(), O_WRONLY | O_CREAT, FILE_PERMS);
if (orifilefd <= 0)
ori_file_fd = open(original_file.c_str(), O_WRONLY | O_CREAT, FILE_PERMS);
if (ori_file_fd <= 0)
{
LOG_ERR << errno << ": Open failed " << originalfile;
LOG_ERR << errno << ": Open failed " << original_file;
return -1;
}
}
// Restore the blocks as specified in block index.
for (uint32_t idxoffset = 8; idxoffset < bindex.size();)
for (uint32_t idx_offset = 8; idx_offset < bindex.size();)
{
// Find the block no. of where this block is from in the original file.
uint32_t blockno = 0;
memcpy(&blockno, idxptr + idxoffset, 4);
idxoffset += 4;
off_t orifileoffset = blockno * BLOCK_SIZE;
uint32_t block_no = 0;
memcpy(&block_no, idx_ptr + idx_offset, 4);
idx_offset += 4;
off_t ori_file_offset = block_no * BLOCK_SIZE;
// Find the offset where the block is located in the block cache file.
off_t bcacheoffset;
memcpy(&bcacheoffset, idxptr + idxoffset, 8);
idxoffset += 40; // Skip the hash(32)
off_t bcache_offset;
memcpy(&bcache_offset, idx_ptr + idx_offset, 8);
idx_offset += 40; // Skip the hash(32)
// Transfer the cached block to the target file.
copy_file_range(bcachefd, &bcacheoffset, orifilefd, &orifileoffset, BLOCK_SIZE, 0);
copy_file_range(bcache_fd, &bcache_offset, ori_file_fd, &ori_file_offset, BLOCK_SIZE, 0);
}
// If the target file is bigger than the original size, truncate it to the original size.
off_t currentlen = lseek(orifilefd, 0, SEEK_END);
if (currentlen > originallen)
ftruncate(orifilefd, originallen);
off_t current_len = lseek(ori_file_fd, 0, SEEK_END);
if (current_len > original_len)
ftruncate(ori_file_fd, original_len);
close(bcachefd);
close(orifilefd);
close(bcache_fd);
close(ori_file_fd);
return 0;
}
@@ -154,12 +154,12 @@ void state_restore::rewind_checkpoints()
// we need to shift each history delta by 1 place.
// Delete the state 0 (current) delta.
boost::filesystem::remove_all(ctx.deltadir);
boost::filesystem::remove_all(ctx.delta_dir);
int16_t oldest_chkpnt = (MAX_CHECKPOINTS + 1) * -1; // +1 because we maintain one extra checkpoint in case of rollbacks.
for (int16_t chkpnt = -1; chkpnt >= oldest_chkpnt; chkpnt--)
{
std::string dir = get_statedir_root(chkpnt);
std::string dir = get_state_dir_root(chkpnt);
if (boost::filesystem::exists(dir))
{
@@ -167,12 +167,12 @@ void state_restore::rewind_checkpoints()
{
// Shift -1 state delta dir to 0-state and delete -1 dir.
std::string delta_1 = dir + DELTA_DIR;
boost::filesystem::rename(delta_1, ctx.deltadir);
boost::filesystem::rename(delta_1, ctx.delta_dir);
boost::filesystem::remove_all(dir);
}
else
{
std::string dirshift = get_statedir_root(chkpnt + 1);
std::string dirshift = get_state_dir_root(chkpnt + 1);
boost::filesystem::rename(dir, dirshift);
}
}
@@ -180,17 +180,17 @@ void state_restore::rewind_checkpoints()
}
// Rolls back current state to previous state.
int state_restore::rollback(hasher::B2H &roothash)
int state_restore::rollback(hasher::B2H &root_hash)
{
ctx = get_statedir_context();
ctx = get_state_dir_context();
delete_newfiles();
if (restore_touchedfiles() == -1)
delete_new_files();
if (restore_touched_files() == -1)
return -1;
// Update hash tree.
hashtree_builder htreebuilder(ctx);
htreebuilder.generate(roothash);
hashtree_builder htree_builder(ctx);
htree_builder.generate(root_hash);
rewind_checkpoints();

10
src/statefs/state_restore.hpp
View File

@@ -11,16 +11,16 @@ namespace statefs
class state_restore
{
private:
statedir_context ctx;
state_dir_context ctx;
std::unordered_set<std::string> created_dirs;
void delete_newfiles();
int restore_touchedfiles();
int read_blockindex(std::vector<char> &buffer, std::string_view file);
void delete_new_files();
int restore_touched_files();
int read_block_index(std::vector<char> &buffer, std::string_view file);
int restore_blocks(std::string_view file, const std::vector<char> &bindex);
void rewind_checkpoints();
public:
int rollback(hasher::B2H &roothash);
int rollback(hasher::B2H &root_hash);
};
} // namespace statefs

46
src/statefs/state_store.cpp
View File

@@ -17,7 +17,7 @@ std::unordered_map<std::string, std::map<uint32_t, hasher::B2H>> touched_files;
*/
bool is_dir_exists(const std::string &dir_relpath)
{
const std::string full_path = current_ctx.datadir + dir_relpath;
const std::string full_path = current_ctx.data_dir + dir_relpath;
return boost::filesystem::exists(full_path);
}
@@ -34,7 +34,7 @@ int get_fs_entry_hashes(std::unordered_map<std::string, p2p::state_fs_hash_entry
if (expected_hash != hasher::B2H_empty)
{
// Check whether the existing block hash matches expected hash.
const std::string dir_hash_path = current_ctx.hashtreedir + dir_relpath + DIRHASH_FNAME;
const std::string dir_hash_path = current_ctx.hashtree_dir + dir_relpath + DIR_HASH_FNAME;
hasher::B2H existsing_hash;
if (read_file_bytes(&existsing_hash, dir_hash_path.c_str(), 0, hasher::HASH_SIZE) == -1)
@@ -44,7 +44,7 @@ int get_fs_entry_hashes(std::unordered_map<std::string, p2p::state_fs_hash_entry
return -1;
}
const std::string full_path = current_ctx.datadir + dir_relpath;
const std::string full_path = current_ctx.data_dir + dir_relpath;
for (const boost::filesystem::directory_entry &dentry : boost::filesystem::directory_iterator(full_path))
{
const boost::filesystem::path p = dentry.path();
@@ -60,12 +60,12 @@ int get_fs_entry_hashes(std::unordered_map<std::string, p2p::state_fs_hash_entry
if (fs_entry.is_file)
{
hash_path = current_ctx.blockhashmapdir + fsentry_relpath + HASHMAP_EXT;
hash_path = current_ctx.block_hashmap_dir + fsentry_relpath + BLOCK_HASHMAP_EXT;
}
else
{
fsentry_relpath += "/";
hash_path = current_ctx.hashtreedir + fsentry_relpath + DIRHASH_FNAME;
hash_path = current_ctx.hashtree_dir + fsentry_relpath + DIR_HASH_FNAME;
// Skip the directory if it doesn't contain the dir.hash file.
// By that we assume the directory is empty so we're not interested in it.
if (!boost::filesystem::exists(hash_path))
@@ -86,7 +86,7 @@ int get_fs_entry_hashes(std::unordered_map<std::string, p2p::state_fs_hash_entry
*/
int get_block_hash_map(std::vector<uint8_t> &vec, const std::string &file_relpath, const hasher::B2H expected_hash)
{
const std::string bhmap_path = current_ctx.blockhashmapdir + file_relpath + HASHMAP_EXT;
const std::string bhmap_path = current_ctx.block_hashmap_dir + file_relpath + BLOCK_HASHMAP_EXT;
if (expected_hash != hasher::B2H_empty)
{
@@ -119,11 +119,11 @@ int get_block_hash_map(std::vector<uint8_t> &vec, const std::string &file_relpat
*/
int get_file_length(const std::string &file_relpath)
{
std::string full_path = current_ctx.datadir + file_relpath;
std::string full_path = current_ctx.data_dir + file_relpath;
int fd = open(full_path.c_str(), O_RDONLY);
if (fd == -1)
{
LOG_ERR << errno << "Open failed " << full_path;
LOG_ERR << errno << " Open failed " << full_path;
return -1;
}
@@ -142,7 +142,7 @@ int get_block(std::vector<uint8_t> &vec, const std::string &file_relpath, const
// Check whether the existing block hash matches expected hash.
if (expected_hash != hasher::B2H_empty)
{
std::string bhmap_path = current_ctx.blockhashmapdir + file_relpath + HASHMAP_EXT;
std::string bhmap_path = current_ctx.block_hashmap_dir + file_relpath + BLOCK_HASHMAP_EXT;
hasher::B2H existing_hash = hasher::B2H_empty;
if (read_file_bytes(&existing_hash, bhmap_path.c_str(), (block_id + 1) * hasher::HASH_SIZE, hasher::HASH_SIZE) == -1)
@@ -152,7 +152,7 @@ int get_block(std::vector<uint8_t> &vec, const std::string &file_relpath, const
return -1;
}
std::string full_path = current_ctx.datadir + file_relpath;
std::string full_path = current_ctx.data_dir + file_relpath;
vec.resize(BLOCK_SIZE);
int read_bytes = read_file_bytes(vec.data(), full_path.c_str(), block_id * BLOCK_SIZE, BLOCK_SIZE);
@@ -168,7 +168,7 @@ int get_block(std::vector<uint8_t> &vec, const std::string &file_relpath, const
*/
void create_dir(const std::string &dir_relpath)
{
const std::string full_path = current_ctx.datadir + dir_relpath;
const std::string full_path = current_ctx.data_dir + dir_relpath;
boost::filesystem::create_directories(full_path);
}
@@ -178,7 +178,7 @@ void create_dir(const std::string &dir_relpath)
*/
int delete_dir(const std::string &dir_relpath)
{
std::string full_dir_path = current_ctx.datadir + dir_relpath;
std::string full_dir_path = current_ctx.data_dir + dir_relpath;
const boost::filesystem::directory_iterator itr_end;
for (boost::filesystem::directory_iterator itr(full_dir_path); itr != itr_end; itr++)
@@ -192,7 +192,7 @@ int delete_dir(const std::string &dir_relpath)
// Add the deleted file rel path to the touched files list.
touched_files.emplace(
get_relpath(p.string(), current_ctx.datadir),
get_relpath(p.string(), current_ctx.data_dir),
std::map<uint32_t, hasher::B2H>());
}
}
@@ -209,7 +209,7 @@ int delete_dir(const std::string &dir_relpath)
*/
int delete_file(const std::string &file_relpath)
{
std::string full_path = current_ctx.datadir + file_relpath;
std::string full_path = current_ctx.data_dir + file_relpath;
if (!boost::filesystem::remove(full_path))
return -1;
@@ -223,11 +223,11 @@ int delete_file(const std::string &file_relpath)
*/
int truncate_file(const std::string &file_relpath, const size_t newsize)
{
std::string full_path = current_ctx.datadir + file_relpath;
std::string full_path = current_ctx.data_dir + file_relpath;
int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS);
if (fd == -1)
{
LOG_ERR << errno << "Open failed " << full_path;
LOG_ERR << errno << " Open failed " << full_path;
return -1;
}
@@ -252,11 +252,11 @@ int truncate_file(const std::string &file_relpath, const size_t newsize)
*/
int write_block(const std::string &file_relpath, const uint32_t block_id, const void *buf, const size_t len)
{
std::string full_path = current_ctx.datadir + file_relpath;
std::string full_path = current_ctx.data_dir + file_relpath;
int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS);
if (fd == -1)
{
LOG_ERR << errno << "Open failed " << full_path;
LOG_ERR << errno << " Open failed " << full_path;
return -1;
}
@@ -265,7 +265,7 @@ int write_block(const std::string &file_relpath, const uint32_t block_id, const
close(fd);
if (ret == -1)
{
LOG_ERR << errno << "Write failed " << full_path;
LOG_ERR << errno << " Write failed " << full_path;
return -1;
}
@@ -304,7 +304,7 @@ int read_file_bytes(void *buf, const char *filepath, const off_t start, const si
int fd = open(filepath, O_RDONLY);
if (fd == -1)
{
LOG_ERR << errno << "Open failed " << filepath;
LOG_ERR << errno << " Open failed " << filepath;
return -1;
}
@@ -312,7 +312,7 @@ int read_file_bytes(void *buf, const char *filepath, const off_t start, const si
close(fd);
if (read_bytes <= 0)
{
LOG_ERR << errno << "Read failed " << filepath;
LOG_ERR << errno << " Read failed " << filepath;
return -1;
}
@@ -331,7 +331,7 @@ int read_file_bytes_to_end(std::vector<uint8_t> &vec, const char *filepath, cons
int fd = open(filepath, O_RDONLY);
if (fd == -1)
{
LOG_ERR << errno << "Open failed " << filepath;
LOG_ERR << errno << " Open failed " << filepath;
return -1;
}
@@ -346,7 +346,7 @@ int read_file_bytes_to_end(std::vector<uint8_t> &vec, const char *filepath, cons
close(fd);
if (read_bytes <= 0)
{
LOG_ERR << errno << "Read failed " << filepath;
LOG_ERR << errno << " Read failed " << filepath;
return -1;
}
vec.resize(read_bytes);