Ledger close refactor. (#330)

This commit is contained in:
Ravin Perera
2021-07-14 10:09:13 +05:30
committed by GitHub
parent d315f9c316
commit 7ba84e8e7a
12 changed files with 177 additions and 77 deletions

View File

@@ -285,7 +285,7 @@ namespace comm
if (challenge_status == comm::CHALLENGE_STATUS::CHALLENGE_VERIFIED)
{
// Sessions use pubkey hex as unique id (skipping first 2 bytes key type prefix).
return uniqueid.substr(2, 10) + (is_inbound ? ":in" : ":out");
return uniqueid.substr(2, 8) + (is_inbound ? ":in" : ":out");
}
return uniqueid + (is_inbound ? ":in" : ":out");

View File

@@ -105,6 +105,8 @@ namespace consensus
int consensus()
{
// A consensus round consists of 4 stages (0,1,2,3).
// In stage 0 we perform closing of ledger based on previous round stage 3 votes. In stages 1,2,3
// we progressively narrow down the votes based on increasing majority thresholds.
// For a given stage, this function may get visited multiple times due to time-wait conditions.
if (!wait_and_proceed_stage())
@@ -116,6 +118,10 @@ namespace consensus
// arived ones and expired ones.
revise_candidate_proposals(ctx.vote_status == VOTES_SYNCED);
// Attempt to close the ledger after scanning last round stage 3 proposals.
if (ctx.stage == 0)
attempt_ledger_close();
// Get current lcl, state, patch, primary shard and raw shard info.
util::sequence_hash lcl_id = ledger::ctx.get_lcl_id();
util::h32 state_hash = sc::contract_fs.get_parent_hash(sc::STATE_DIR_PATH);
@@ -154,7 +160,7 @@ namespace consensus
{
int new_sync_status = check_sync_status(unl_count, votes, lcl_id);
if (ctx.vote_status != VOTES_SYNCED && new_sync_status == VOTES_UNRELIABLE)
if (ctx.vote_status != VOTES_SYNCED && new_sync_status == VOTES_SYNCED)
{
// If we are just becoming 'in-sync' after being out-of-sync, check the vote status again after the proper
// pruning of candidate proposals. This is because we relax the proposal pruning rules when we are not in sync,
@@ -210,7 +216,7 @@ namespace consensus
const p2p::proposal p = create_stage123_proposal(votes, unl_count, state_hash, patch_hash, last_primary_shard_id, last_raw_shard_id);
broadcast_proposal(p);
// This marks the moment we finish a sync cycle. We are in stage 1 and we ditect that our votes are in sync.
// This marks the moment we finish a sync cycle. We are in stage 1 and we detect that our votes are in sync.
if (ctx.stage == 1 && ctx.sync_ongoing)
{
// Clear any sync recovery pending state if we enter stage 1 while being in sync.
@@ -218,21 +224,6 @@ namespace consensus
status::sync_status_changed(true);
LOG_DEBUG << "Sync recovery completed.";
}
else if (ctx.stage == 3)
{
// Upon successful consensus at stage 3, update the ledger and execute the contract using the consensus proposal.
consensed_user_map consensed_users;
if (prepare_consensed_users(consensed_users, p) == -1 ||
commit_consensus_results(p, consensed_users, patch_hash) == -1)
{
LOG_ERROR << "Error occured in Stage 3 consensus execution.";
// Cleanup obsolete information before next round starts.
cleanup_output_collections();
cleanup_consensed_user_inputs(consensed_users);
}
}
}
// We have finished a consensus stage.
@@ -243,13 +234,76 @@ namespace consensus
return 0;
}
/**
* Scan the stage 3 proposals from last round and create the ledger if all majority critertia are met.
*/
void attempt_ledger_close()
{
std::map<util::h32, uint32_t> hash_votes; // Votes on the proposal hash.
util::h32 self_hash = util::h32_empty;
util::h32 majority_hash = util::h32_empty;
// Find the stage 3 proposal that we have made.
const auto itr = ctx.candidate_proposals.find(conf::cfg.node.public_key);
if (itr == ctx.candidate_proposals.end() || itr->second.stage != 3)
{
LOG_DEBUG << "We haven't proposed to close any ledger.";
return;
}
const p2p::proposal self_prop = itr->second;
// Count votes of all stage 3 proposal hashes.
for (const auto &[pubkey, cp] : ctx.candidate_proposals)
{
if (cp.stage == 3)
increment(hash_votes, cp.root_hash);
}
// Find the winning hash and no. of votes for it.
uint32_t winning_votes = 0;
for (const auto [hash, votes] : hash_votes)
{
if (votes > winning_votes)
{
winning_votes = votes;
majority_hash = hash;
}
}
const uint32_t min_votes_required = ceil(MAJORITY_THRESHOLD * unl::count());
if (winning_votes < min_votes_required)
{
LOG_INFO << "Cannot close ledger. Possible fork condition. won:" << winning_votes << " needed:" << min_votes_required;
return;
}
if (self_prop.root_hash != majority_hash)
{
LOG_INFO << "Cannot close ledger. Our proposal:" << self_prop.root_hash << " does not match with majority:" << majority_hash;
return;
}
LOG_DEBUG << "Closing ledger with proposal:" << self_prop.root_hash;
// Upon successful ledger close condition, update the ledger and execute the contract using the consensus proposal.
consensed_user_map consensed_users;
if (prepare_consensed_users(consensed_users, self_prop) == -1 ||
commit_consensus_results(self_prop, consensed_users) == -1)
{
LOG_ERROR << "Error occured when closing ledger";
// Cleanup obsolete information before next round starts.
cleanup_output_collections();
cleanup_consensed_user_inputs(consensed_users);
}
}
/**
* Performs the consensus finalalization activities with the provided consensused information.
* @param cons_prop The proposal which reached consensus.
* @param consensed_users Set of consensed users and their consensed inputs and outputs.
* @param patch_hash The current config patch hash.
*/
int commit_consensus_results(const p2p::proposal &cons_prop, const consensus::consensed_user_map &consensed_users, const util::h32 &patch_hash)
int commit_consensus_results(const p2p::proposal &cons_prop, const consensus::consensed_user_map &consensed_users)
{
// Creating a ledger while sync ongoing happens when we discover that our ledger votes are in sync at stage 2 or 3. At this point,
// we can create the ledger with majority votes. However we dont't have the raw contract outputs we should have had in the previous ledger
@@ -278,6 +332,7 @@ namespace consensus
dispatch_consensed_user_outputs(consensed_users, lcl_id);
// Apply consensed config patch file changes to the hpcore runtime and hp.cfg.
const util::h32 patch_hash = sc::contract_fs.get_parent_hash(sc::PATCH_FILE_PATH);
if (apply_consensed_patch_file_changes(cons_prop.patch_hash, patch_hash) == -1)
return -1;
@@ -436,21 +491,41 @@ namespace consensus
{
const p2p::proposal &cp = itr->second;
// If we are in sync, only consider this round's proposals which are from current or previous stage.
// Otherwise consider all proposals as long as they are from the same round.
const bool stage_valid = in_sync ? (ctx.stage >= cp.stage && (ctx.stage - cp.stage) <= 1) : true;
const bool keep_candidate = (ctx.round_start_time == cp.time) && stage_valid;
LOG_DEBUG << (keep_candidate ? "Prop--->" : "Erased")
<< " [s" << std::to_string(cp.stage)
<< "] u/i:" << cp.users.size()
<< "/" << cp.input_ordered_hashes.size()
<< " ts:" << cp.time
<< " state:" << cp.state_hash
<< " patch:" << cp.patch_hash
<< " lps:" << cp.last_primary_shard_id
<< " lrs:" << cp.last_raw_shard_id
<< " [from:" << ((cp.pubkey == conf::cfg.node.public_key) ? "self" : util::to_hex(cp.pubkey).substr(2, 10)) << "]"
<< "(" << (cp.recv_timestamp > cp.sent_timestamp ? (cp.recv_timestamp - cp.sent_timestamp) : 0) << "ms)";
// Drop all proposals which are older than previous round.
// If we are not in sync, consider all remaining proposals.
// If we are in sync, consider proposals from previous stage only.
bool keep_candidate = false;
if (ctx.round_start_time >= cp.time && (ctx.round_start_time - cp.time) <= conf::cfg.contract.roundtime)
{
if (!in_sync)
keep_candidate = true;
else
keep_candidate = ctx.round_start_time == cp.time
? ctx.stage >= cp.stage && (ctx.stage - cp.stage) <= 1 // This round previous stage.
: ctx.stage == 0 && cp.stage == 3; // Previous round stage 3.
}
if (keep_candidate)
{
LOG_DEBUG << "[s" << std::to_string(cp.stage)
<< "-" << cp.root_hash
<< "] u/i/t:" << cp.users.size()
<< "/" << cp.input_ordered_hashes.size()
<< "/" << cp.time
<< " s:" << cp.state_hash
<< " p:" << cp.patch_hash
<< " ps:" << cp.last_primary_shard_id
<< " rs:" << cp.last_raw_shard_id
<< " [frm:" << (cp.from_self ? "self" : util::to_hex(cp.pubkey).substr(2, 8))
<< "<" << (cp.recv_timestamp > cp.sent_timestamp ? (cp.recv_timestamp - cp.sent_timestamp) : 0) << "ms]";
}
else
{
LOG_DEBUG << "Erased [s" << std::to_string(cp.stage)
<< "-" << cp.root_hash
<< "] [frm:" << (cp.from_self ? "self" : util::to_hex(cp.pubkey).substr(2, 8)) << "]";
}
if (keep_candidate)
++itr;
@@ -616,7 +691,9 @@ namespace consensus
}
else
{
const uint64_t stage_start = ctx.round_start_time + (ctx.stage * ctx.stage_time);
// Stage start time is calculated based on the duration each stage gets. Stages 1,2,3 gets equal duration as configured
// by stage slice. Stage 0 gets entire remaining time out of the round window.
const uint64_t stage_start = ctx.round_start_time + ctx.stage0_duration + ((ctx.stage - 1) * ctx.stage123_duration);
// Compute stage time wait.
// Node wait between stages to collect enough proposals from previous stages from other nodes.
@@ -688,13 +765,14 @@ namespace consensus
p2pmsg::create_msg_from_proposal(fbuf, p);
p2p::broadcast_message(fbuf, true, false, !conf::cfg.contract.is_consensus_public, 1); // Use high priority send.
LOG_DEBUG << "Proposed <s" << std::to_string(p.stage) << "> u/i:" << p.users.size()
LOG_DEBUG << "Proposed-s" << std::to_string(p.stage)
<< " u/i/t:" << p.users.size()
<< "/" << p.input_ordered_hashes.size()
<< " ts:" << p.time
<< " state:" << p.state_hash
<< " patch:" << p.patch_hash
<< " lps:" << p.last_primary_shard_id
<< " lrs:" << p.last_raw_shard_id;
<< "/" << p.time
<< " s:" << p.state_hash
<< " p:" << p.patch_hash
<< " ps:" << p.last_primary_shard_id
<< " rs:" << p.last_raw_shard_id;
}
/**
@@ -980,10 +1058,10 @@ namespace consensus
}
// Check whether we have received enough votes in total.
const uint32_t min_required = ceil(MAJORITY_THRESHOLD * unl_count);
if (total_ledger_primary_hash_votes < min_required)
const uint32_t min_votes_required = ceil(MAJORITY_THRESHOLD * unl_count);
if (total_ledger_primary_hash_votes < min_votes_required)
{
LOG_INFO << "Not enough peers proposing to perform consensus. votes:" << total_ledger_primary_hash_votes << " needed:" << min_required;
LOG_INFO << "Not enough peers proposing to perform consensus. votes:" << total_ledger_primary_hash_votes << " needed:" << min_votes_required;
return false;
}
@@ -997,10 +1075,9 @@ namespace consensus
}
}
const uint32_t min_wins_required = ceil(MAJORITY_THRESHOLD * ctx.candidate_proposals.size());
if (winning_votes < min_wins_required)
if (winning_votes < 2) // min_votes_required
{
LOG_INFO << "No consensus on last shard hash. Possible fork condition. won:" << winning_votes << " needed:" << min_wins_required;
LOG_INFO << "No consensus on last shard hash. Possible fork condition. won:" << winning_votes << " needed:" << min_votes_required;
return false;
}
else if (ledger::ctx.get_last_primary_shard_id() != majority_primary_shard_id)
@@ -1375,8 +1452,9 @@ namespace consensus
conf::cfg.contract.stage_slice = majority_time_config - (conf::cfg.contract.roundtime * 100);
}
// We allocate configured stage slice for stages 0, 1, 2. Stage 3 gets the entire remaining time from the round window.
ctx.stage_time = conf::cfg.contract.roundtime * conf::cfg.contract.stage_slice / 100;
// We allocate configured stage slice for stages 1, 2, 3. Stage 0 gets the entire remaining time from the round window.
ctx.stage123_duration = conf::cfg.contract.roundtime * conf::cfg.contract.stage_slice / 100;
ctx.stage0_duration = conf::cfg.contract.roundtime - (ctx.stage123_duration * 3);
ctx.stage_reset_wait_threshold = conf::cfg.contract.roundtime / 10;
// We use a time window boundry offset based on contract id to vary the window boundries between

View File

@@ -113,7 +113,8 @@ namespace consensus
uint8_t stage = 1;
uint64_t round_start_time = 0;
uint32_t stage_time = 0; // Time allocated to a consensus stage.
uint32_t stage0_duration = 0; // Time allocated to a consensus stage 0.
uint32_t stage123_duration = 0; // Time allocated to each consensus stage 1,2,3.
uint32_t stage_reset_wait_threshold = 0; // Minimum stage wait time to reset the stage.
uint64_t round_boundry_offset = 0; // Time window boundry offset based on contract id.
uint16_t unreliable_votes_attempts = 0; // No. of times we failed to get reliable votes continously.
@@ -174,7 +175,9 @@ namespace consensus
int consensus();
int commit_consensus_results(const p2p::proposal &cons_prop, const consensus::consensed_user_map &consensed_users, const util::h32 &patch_hash);
void attempt_ledger_close();
int commit_consensus_results(const p2p::proposal &cons_prop, const consensus::consensed_user_map &consensed_users);
int check_sync_status(const size_t unl_count, vote_counter &votes, const util::sequence_hash &lcl_id);
@@ -216,7 +219,7 @@ namespace consensus
uint64_t get_ledger_time_resolution(const uint64_t time);
uint64_t get_stage_time_resolution(const uint64_t time);
uint64_t get_stage_duration_resolution(const uint64_t time);
int execute_contract(const uint64_t time, const consensed_user_map &consensed_users, const util::sequence_hash &lcl_id);

View File

@@ -113,7 +113,7 @@ namespace hpfs
}
}
LOG_DEBUG << "Hpfs " << name << " serve: Sent " << fbufs.size() << " replies to [" << util::to_hex(session_id).substr(2, 10) << "]";
LOG_DEBUG << "Hpfs " << name << " serve: Sent " << fbufs.size() << " replies to [" << util::to_hex(session_id).substr(2, 8) << "]";
}
fs_mount->release_rw_session();

View File

@@ -300,7 +300,7 @@ namespace hpfs
if (is_shutting_down)
return false;
const std::string from = response.first.substr(2, 10); // Sender pubkey.
const std::string from = response.first.substr(2, 8); // Sender pubkey.
const p2pmsg::P2PMsg &msg = *p2pmsg::GetP2PMsg(response.second.data());
const p2pmsg::HpfsResponseMsg &resp_msg = *msg.content_as_HpfsResponseMsg();
@@ -590,7 +590,7 @@ namespace hpfs
request_state_from_peer(request.vpath, is_file, request.block_id, request.expected_hash, target_pubkey);
LOG_DEBUG << "Hpfs " << name << " sync: " << (is_resubmit ? "Re-submitting" : "Submitting")
<< " request to [" << (target_pubkey.empty() ? "" : target_pubkey.substr(2, 10)) << "]. type:" << request.type
<< " request to [" << (target_pubkey.empty() ? "" : target_pubkey.substr(2, 8)) << "]. type:" << request.type
<< " path:" << request.vpath << " block_id:" << request.block_id
<< " hash:" << request.expected_hash;
}

View File

@@ -76,11 +76,10 @@ namespace msg::fbuf::p2pmsg
add(h->hash());
}
const std::string hash()
const util::h32 hash()
{
std::string hash;
hash.resize(BLAKE3_OUT_LEN);
blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t *>(hash.data()), hash.size());
util::h32 hash;
blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t *>(&hash), sizeof(util::h32));
return hash;
}
};

View File

@@ -54,7 +54,11 @@ namespace msg::fbuf::p2pmsg
return p2p::peer_message_info{p2p_msg, p2p_msg->content_type(), p2p_msg->created_on()};
}
bool verify_proposal_msg_trust(const p2p::peer_message_info &mi)
/**
* Validate proposal signature against the hash of proposal fields.
* @return The proposal hash if verification success. Empty hash of verification failed.
*/
const util::h32 verify_proposal_msg_trust(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
@@ -64,10 +68,19 @@ namespace msg::fbuf::p2pmsg
if (!unl::exists(std::string(pubkey)))
{
LOG_DEBUG << "Peer proposal message pubkey verification failed. Not in UNL.";
return false;
return util::h32_empty;
}
// Get hash of proposal data field values and verify the signature against the hash.
const util::h32 hash = hash_proposal_msg(msg);
if (crypto::verify(hash.to_string_view(), flatbuf_bytes_to_sv(msg.sig()), pubkey) == 0)
return hash;
else
return util::h32_empty;
}
const util::h32 hash_proposal_msg(const msg::fbuf::p2pmsg::ProposalMsg &msg)
{
flatbuf_hasher hasher;
hasher.add(msg.stage());
hasher.add(msg.time());
@@ -81,8 +94,7 @@ namespace msg::fbuf::p2pmsg
hasher.add(msg.patch_hash());
hasher.add(msg.last_primary_shard_id());
hasher.add(msg.last_raw_shard_id());
return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), pubkey) == 0;
return hasher.hash();
}
bool verify_npl_msg_trust(const p2p::peer_message_info &mi)
@@ -103,7 +115,8 @@ namespace msg::fbuf::p2pmsg
hasher.add(msg.data());
hasher.add(msg.lcl_id());
return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), pubkey) == 0;
const util::h32 hash = hasher.hash();
return crypto::verify(hash.to_string_view(), flatbuf_bytes_to_sv(msg.sig()), pubkey) == 0;
}
const p2p::peer_challenge create_peer_challenge_from_msg(const p2p::peer_message_info &mi)
@@ -125,12 +138,15 @@ namespace msg::fbuf::p2pmsg
std::string(flatbuf_bytes_to_sv(msg.pubkey()))};
}
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi)
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi, const util::h32 &hash)
{
const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
p2p::proposal p;
p.pubkey = flatbuf_bytes_to_sv(msg.pubkey());
p.root_hash = hash;
p.from_self = p.pubkey == conf::cfg.node.public_key;
p.sent_timestamp = mi.originated_on;
p.recv_timestamp = util::get_epoch_milliseconds();
p.time = msg.time();
@@ -331,7 +347,7 @@ namespace msg::fbuf::p2pmsg
hasher.add(p.last_primary_shard_id);
hasher.add(p.last_raw_shard_id);
return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
return crypto::sign(hasher.hash().to_string_view(), conf::cfg.node.private_key);
}
const std::string generate_npl_signature(std::string_view data, const util::sequence_hash &lcl_id)
@@ -340,7 +356,7 @@ namespace msg::fbuf::p2pmsg
hasher.add(data);
hasher.add(lcl_id);
return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
return crypto::sign(hasher.hash().to_string_view(), conf::cfg.node.private_key);
}
void create_p2p_msg(flatbuffers::FlatBufferBuilder &builder, const msg::fbuf::p2pmsg::P2PMsgContent content_type, const flatbuffers::Offset<void> content)

View File

@@ -14,7 +14,9 @@ namespace msg::fbuf::p2pmsg
const p2p::peer_message_info get_peer_message_info(std::string_view message, const p2p::peer_comm_session *session = NULL);
bool verify_proposal_msg_trust(const p2p::peer_message_info &mi);
const util::h32 verify_proposal_msg_trust(const p2p::peer_message_info &mi);
const util::h32 hash_proposal_msg(const msg::fbuf::p2pmsg::ProposalMsg &msg);
bool verify_npl_msg_trust(const p2p::peer_message_info &mi);
@@ -22,7 +24,7 @@ namespace msg::fbuf::p2pmsg
const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const p2p::peer_message_info &mi);
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi);
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi, const util::h32 &hash);
const p2p::npl_message create_npl_from_msg(const p2p::peer_message_info &mi);

View File

@@ -33,10 +33,11 @@ namespace p2p
int64_t weight = 0;
};
struct proposal
{
std::string pubkey;
util::h32 root_hash; // The proposal root hash (hash of all the proposal consensus fields). Only populated for incoming proposals.
bool from_self; // Whether the proposal was sent by this node itself. Only populated for incoming proposals.
uint64_t sent_timestamp = 0; // The timestamp of the sender when this proposal was sent.
uint64_t recv_timestamp = 0; // The timestamp when we received the proposal. (used for network statistics)
@@ -88,7 +89,7 @@ namespace p2p
// Represents an NPL message sent by a peer.
struct npl_message
{
std::string pubkey; // Peer binary pubkey.
std::string pubkey; // Peer binary pubkey.
util::sequence_hash lcl_id; // lcl of the peer.
std::string data;
};

View File

@@ -174,14 +174,15 @@ namespace p2p
}
else if (mi.type == p2pmsg::P2PMsgContent_ProposalMsg)
{
if (!p2pmsg::verify_proposal_msg_trust(mi))
const util::h32 hash = p2pmsg::verify_proposal_msg_trust(mi);
if (hash == util::h32_empty)
{
session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1);
LOG_DEBUG << "Proposal rejected due to trust failure. " << session.display_name();
return 0;
}
handle_proposal_message(p2pmsg::create_proposal_from_msg(mi));
handle_proposal_message(p2pmsg::create_proposal_from_msg(mi, hash));
}
else if (mi.type == p2pmsg::P2PMsgContent_NplMsg)
{
@@ -298,7 +299,7 @@ namespace p2p
const peer_message_info mi = p2pmsg::get_peer_message_info(message);
if (mi.type == p2pmsg::P2PMsgContent_ProposalMsg)
handle_proposal_message(p2pmsg::create_proposal_from_msg(mi));
handle_proposal_message(p2pmsg::create_proposal_from_msg(mi, hash_proposal_msg(*mi.p2p_msg->content_as_ProposalMsg())));
else if (mi.type == p2pmsg::P2PMsgContent_NonUnlProposalMsg)
handle_nonunl_proposal_message(p2pmsg::create_nonunl_proposal_from_msg(mi));
else if (mi.type == p2pmsg::P2PMsgContent_NplMsg)

View File

@@ -135,7 +135,7 @@ namespace sc::hpfs_log_sync
std::string target_pubkey;
p2p::send_message_to_random_peer(fbuf, target_pubkey, true);
if (!target_pubkey.empty())
LOG_DEBUG << "Hpfs log sync: Requesting from [" << target_pubkey.substr(2, 10) << "]."
LOG_DEBUG << "Hpfs log sync: Requesting from [" << target_pubkey.substr(2, 8) << "]."
<< " min:" << sync_ctx.min_log_record.seq_no
<< " target:" << sync_ctx.target_log_seq_no;

View File

@@ -65,7 +65,7 @@ namespace util
output << std::hex;
const uint8_t *buf = reinterpret_cast<const uint8_t *>(&h);
for (int i = 0; i < 5; i++) // Only print first 5 bytes in hex.
for (int i = 0; i < 4; i++) // Only print first 4 bytes in hex.
output << std::setfill('0') << std::setw(2) << (int)buf[i];
// Reset the ostream flags because we set std::hex at the begining.