ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

P2P Flatbuffers schema refactor. (#255)

Browse Source
This commit is contained in:
Ravin Perera
2021-02-23 13:15:03 +05:30
committed by GitHub
parent 312713c296
commit 137c199b07
36 changed files with 3456 additions and 4211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/msg/fbuf/common_helpers.cpp
View File

@@ -3,123 +3,54 @@
namespace msg::fbuf
{
//---Conversion helpers from flatbuffers data types to std data types---//
//---Flatbuf to std---//
/**
* Returns string_view from flat buffer data pointer and length.
*/
std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length)
std::string_view flatbuf_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length)
{
const char *signature_content_str = reinterpret_cast<const char *>(data);
return std::string_view(signature_content_str, length);
}
/**
* Returns string_view from Flat Buffer vector of bytes.
*/
std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer)
std::string_view flatbuf_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer)
{
return flatbuff_bytes_to_sv(buffer->Data(), buffer->size());
return flatbuf_bytes_to_sv(buffer->Data(), buffer->size());
}
/**
* Returns return string_view from Flat Buffer string.
*/
std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer)
std::string_view flatbuf_str_to_sv(const flatbuffers::String *buffer)
{
return flatbuff_bytes_to_sv(buffer->Data(), buffer->size());
return flatbuf_bytes_to_sv(buffer->Data(), buffer->size());
}
/**
* Returns hash from Flat Buffer vector of bytes.
*/
util::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer)
util::h32 flatbuf_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer)
{
return *reinterpret_cast<const util::h32 *>(buffer->data());
}
/**
* Returns set from Flatbuffer vector of ByteArrays.
*/
const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec)
std::string_view builder_to_string_view(const flatbuffers::FlatBufferBuilder &builder)
{
std::set<std::string> set;
for (auto el : *fbvec)
set.emplace(std::string(flatbuff_bytes_to_sv(el->array())));
return set;
std::string_view msg = std::string_view(
reinterpret_cast<const char *>(builder.GetBufferPointer()), builder.GetSize());
return msg;
}
/**
* Returns a map from Flatbuffer vector of key value pairs.
*/
const std::unordered_map<std::string, const std::string>
flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec)
{
std::unordered_map<std::string, const std::string> map;
map.reserve(fbvec->size());
for (auto el : *fbvec)
map.emplace(flatbuff_bytes_to_sv(el->key()), flatbuff_bytes_to_sv(el->value()));
return map;
}
//---std to Flatbuf---//
//---Conversion helpers from std data types to flatbuffers data types---//
//---These are used in constructing Flatbuffer messages using builders---//
/**
* Returns Flatbuffer bytes vector from string_view.
*/
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
sv_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
{
return builder.CreateVector(reinterpret_cast<const uint8_t *>(sv.data()), sv.size());
}
/**
* Returns Flatbuffer string from string_view.
*/
const flatbuffers::Offset<flatbuffers::String>
sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
sv_to_flatbuf_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
{
return builder.CreateString(sv);
}
/**
* Returns Flatbuffer bytes vector from hash.
*/
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, const util::h32 hash)
hash_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, const util::h32 hash)
{
return builder.CreateVector(reinterpret_cast<const uint8_t *>(&hash), sizeof(util::h32));
}
/**
* Returns Flatbuffer vector of ByteArrays from given set of strings.
*/
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set)
{
std::vector<flatbuffers::Offset<ByteArray>> fbvec;
fbvec.reserve(set.size());
for (std::string_view str : set)
fbvec.push_back(CreateByteArray(builder, sv_to_flatbuff_bytes(builder, str)));
return builder.CreateVector(fbvec);
}
/**
* Returns Flatbuffer vector of key value pairs from given map.
*/
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map)
{
std::vector<flatbuffers::Offset<BytesKeyValuePair>> fbvec;
fbvec.reserve(map.size());
for (auto const &[key, value] : map)
{
fbvec.push_back(CreateBytesKeyValuePair(
builder,
sv_to_flatbuff_bytes(builder, key),
sv_to_flatbuff_bytes(builder, value)));
}
return builder.CreateVector(fbvec);
}
} // namespace msg::fbuf

36
src/msg/fbuf/common_helpers.hpp
View File

@@ -3,46 +3,36 @@
#include "../../pchheader.hpp"
#include "../../util/h32.hpp"
#include "common_schema_generated.h"
#include "p2pmsg_generated.h"
namespace msg::fbuf
{
/**
* This section contains common Flatbuffer message reading/writing helpers.
*/
* This section contains common Flatbuffer message reading/writing helpers.
*/
//---Conversion helpers from flatbuffers data types to std data types---//
//---Flatbuf to std---//
std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
std::string_view flatbuf_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
std::string_view flatbuf_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer);
std::string_view flatbuf_str_to_sv(const flatbuffers::String *buffer);
util::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
util::h32 flatbuf_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
const std::set<std::string>
flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
std::string_view builder_to_string_view(const flatbuffers::FlatBufferBuilder &builder);
const std::unordered_map<std::string, const std::string>
flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec);
//---Conversion helpers from std data types to flatbuffers data types---//
//---std to Flatbuf---//
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
sv_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::String>
sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
sv_to_flatbuf_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, util::h32 hash);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map);
hash_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, util::h32 hash);
} // namespace msg::fbuf

11
src/msg/fbuf/common_schema.fbs
View File

@@ -1,11 +0,0 @@
//IDL file for common types.
namespace msg.fbuf;
table BytesKeyValuePair { //A key, value pair of byte[].
key:[ubyte];
value:[ubyte];
}
table ByteArray { //To represent list of byte arrays
array:[ubyte];
}

146
src/msg/fbuf/common_schema_generated.h
View File

@@ -1,146 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_
#define FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_
#include "flatbuffers/flatbuffers.h"
namespace msg {
namespace fbuf {
struct BytesKeyValuePair;
struct BytesKeyValuePairBuilder;
struct ByteArray;
struct ByteArrayBuilder;
struct BytesKeyValuePair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef BytesKeyValuePairBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_KEY = 4,
VT_VALUE = 6
};
const flatbuffers::Vector<uint8_t> *key() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_KEY);
}
flatbuffers::Vector<uint8_t> *mutable_key() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_KEY);
}
const flatbuffers::Vector<uint8_t> *value() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_VALUE);
}
flatbuffers::Vector<uint8_t> *mutable_value() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_VALUE);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_KEY) &&
verifier.VerifyVector(key()) &&
VerifyOffset(verifier, VT_VALUE) &&
verifier.VerifyVector(value()) &&
verifier.EndTable();
}
};
struct BytesKeyValuePairBuilder {
typedef BytesKeyValuePair Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_key(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> key) {
fbb_.AddOffset(BytesKeyValuePair::VT_KEY, key);
}
void add_value(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> value) {
fbb_.AddOffset(BytesKeyValuePair::VT_VALUE, value);
}
explicit BytesKeyValuePairBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<BytesKeyValuePair> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<BytesKeyValuePair>(end);
return o;
}
};
inline flatbuffers::Offset<BytesKeyValuePair> CreateBytesKeyValuePair(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> key = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> value = 0) {
BytesKeyValuePairBuilder builder_(_fbb);
builder_.add_value(value);
builder_.add_key(key);
return builder_.Finish();
}
inline flatbuffers::Offset<BytesKeyValuePair> CreateBytesKeyValuePairDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *key = nullptr,
const std::vector<uint8_t> *value = nullptr) {
auto key__ = key ? _fbb.CreateVector<uint8_t>(*key) : 0;
auto value__ = value ? _fbb.CreateVector<uint8_t>(*value) : 0;
return msg::fbuf::CreateBytesKeyValuePair(
_fbb,
key__,
value__);
}
struct ByteArray FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef ByteArrayBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_ARRAY = 4
};
const flatbuffers::Vector<uint8_t> *array() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_ARRAY);
}
flatbuffers::Vector<uint8_t> *mutable_array() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_ARRAY);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_ARRAY) &&
verifier.VerifyVector(array()) &&
verifier.EndTable();
}
};
struct ByteArrayBuilder {
typedef ByteArray Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_array(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> array) {
fbb_.AddOffset(ByteArray::VT_ARRAY, array);
}
explicit ByteArrayBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<ByteArray> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<ByteArray>(end);
return o;
}
};
inline flatbuffers::Offset<ByteArray> CreateByteArray(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> array = 0) {
ByteArrayBuilder builder_(_fbb);
builder_.add_array(array);
return builder_.Finish();
}
inline flatbuffers::Offset<ByteArray> CreateByteArrayDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *array = nullptr) {
auto array__ = array ? _fbb.CreateVector<uint8_t>(*array) : 0;
return msg::fbuf::CreateByteArray(
_fbb,
array__);
}
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_

89
src/msg/fbuf/flatbuf_hasher.hpp Normal file
View File

@@ -0,0 +1,89 @@
#ifndef _HP_MSG_FBUF_FBUF_HASHER
#define _HP_MSG_FBUF_FBUF_HASHER
#include "../../pchheader.hpp"
#include "../../util/util.hpp"
#include "p2pmsg_generated.h"
namespace msg::fbuf::p2pmsg
{
class flatbuf_hasher
{
private:
blake3_hasher hasher;
public:
flatbuf_hasher()
{
blake3_hasher_init(&hasher);
}
void add(const uint8_t i)
{
blake3_hasher_update(&hasher, &i, sizeof(uint8_t));
}
void add(const uint32_t i)
{
uint8_t bytes[4];
util::uint32_to_bytes(bytes, i);
blake3_hasher_update(&hasher, bytes, sizeof(bytes));
}
void add(const uint64_t i)
{
uint8_t bytes[8];
util::uint64_to_bytes(bytes, i);
blake3_hasher_update(&hasher, bytes, sizeof(bytes));
}
void add(std::string_view sv)
{
blake3_hasher_update(&hasher, sv.data(), sv.size());
}
void add(const std::set<std::string> &sl)
{
for(const std::string &s : sl)
add(s);
}
void add(const flatbuffers::Vector<uint8_t> *v)
{
blake3_hasher_update(&hasher, v->data(), v->size());
}
void add(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *v)
{
for (const auto el : *v)
add(el->array());
}
void add(const util::h32 &h)
{
add(h.to_string_view());
}
void add(const p2p::sequence_hash &h)
{
add(h.seq_no);
add(h.hash);
}
void add(const SequenceHash *h)
{
add(h->seq_no());
add(h->hash());
}
const std::string hash()
{
std::string hash;
hash.resize(BLAKE3_OUT_LEN);
blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t *>(hash.data()), hash.size());
return hash;
}
};
}
#endif

15
src/msg/fbuf/ledger_helpers.cpp
View File

@@ -1,6 +1,5 @@
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "ledger_schema_generated.h"
#include "ledger_blob_schema_generated.h"
#include "common_helpers.hpp"
#include "ledger_helpers.hpp"
@@ -24,11 +23,11 @@ namespace msg::fbuf::ledgermsg
inputs.reserve(value.size());
for (const auto &input : value)
inputs.push_back(ledgermsg::CreateRawInput(builder, sv_to_flatbuff_bytes(builder, input)));
inputs.push_back(ledgermsg::CreateRawInput(builder, sv_to_flatbuf_bytes(builder, input)));
raw_inputs.push_back(ledgermsg::CreateRawInputCollection(
builder,
sv_to_flatbuff_bytes(builder, key),
sv_to_flatbuf_bytes(builder, key),
builder.CreateVector(inputs)));
}
@@ -38,18 +37,18 @@ namespace msg::fbuf::ledgermsg
outputs.reserve(value.size());
for (const auto &output : value)
outputs.push_back(ledgermsg::CreateRawOutput(builder, sv_to_flatbuff_bytes(builder, output)));
outputs.push_back(ledgermsg::CreateRawOutput(builder, sv_to_flatbuf_bytes(builder, output)));
raw_outputs.push_back(ledgermsg::CreateRawOutputCollection(
builder,
sv_to_flatbuff_bytes(builder, key),
sv_to_flatbuf_bytes(builder, key),
builder.CreateVector(outputs)));
}
flatbuffers::Offset<ledgermsg::LedgerBlob> blob =
ledgermsg::CreateLedgerBlob(
builder,
hash_to_flatbuff_bytes(builder, ledger_blob.ledger_hash),
hash_to_flatbuf_bytes(builder, ledger_blob.ledger_hash),
builder.CreateVector(raw_inputs),
builder.CreateVector(raw_outputs));

1
src/msg/fbuf/ledger_helpers.hpp
View File

@@ -4,7 +4,6 @@
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "../../ledger/ledger.hpp"
#include "ledger_schema_generated.h"
namespace msg::fbuf::ledgermsg
{

17
src/msg/fbuf/ledger_schema.fbs
View File

@@ -1,17 +0,0 @@
include "common_schema.fbs";
namespace msg.fbuf.ledgermsg;
table LedgerBlock {
version:string;
seq_no:uint64;
time:uint64;
lcl:[ubyte];
state_hash:[ubyte];
patch_hash:[ubyte];
users: [ByteArray];
inputs: [ByteArray];
output: [ubyte];
}
root_type LedgerBlock;

244
src/msg/fbuf/ledger_schema_generated.h
View File

@@ -1,244 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
#define FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
#include "flatbuffers/flatbuffers.h"
#include "common_schema_generated.h"
namespace msg {
namespace fbuf {
namespace ledgermsg {
struct LedgerBlock;
struct LedgerBlockBuilder;
struct LedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef LedgerBlockBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_SEQ_NO = 6,
VT_TIME = 8,
VT_LCL = 10,
VT_STATE_HASH = 12,
VT_PATCH_HASH = 14,
VT_USERS = 16,
VT_INPUTS = 18,
VT_OUTPUT = 20
};
const flatbuffers::String *version() const {
return GetPointer<const flatbuffers::String *>(VT_VERSION);
}
flatbuffers::String *mutable_version() {
return GetPointer<flatbuffers::String *>(VT_VERSION);
}
uint64_t seq_no() const {
return GetField<uint64_t>(VT_SEQ_NO, 0);
}
bool mutate_seq_no(uint64_t _seq_no) {
return SetField<uint64_t>(VT_SEQ_NO, _seq_no, 0);
}
uint64_t time() const {
return GetField<uint64_t>(VT_TIME, 0);
}
bool mutate_time(uint64_t _time) {
return SetField<uint64_t>(VT_TIME, _time, 0);
}
const flatbuffers::Vector<uint8_t> *lcl() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
flatbuffers::Vector<uint8_t> *mutable_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
const flatbuffers::Vector<uint8_t> *state_hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_STATE_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_state_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_STATE_HASH);
}
const flatbuffers::Vector<uint8_t> *patch_hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PATCH_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_patch_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PATCH_HASH);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *users() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_USERS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_users() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_USERS);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
}
const flatbuffers::Vector<uint8_t> *output() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
}
flatbuffers::Vector<uint8_t> *mutable_output() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_VERSION) &&
verifier.VerifyString(version()) &&
VerifyField<uint64_t>(verifier, VT_SEQ_NO) &&
VerifyField<uint64_t>(verifier, VT_TIME) &&
VerifyOffset(verifier, VT_LCL) &&
verifier.VerifyVector(lcl()) &&
VerifyOffset(verifier, VT_STATE_HASH) &&
verifier.VerifyVector(state_hash()) &&
VerifyOffset(verifier, VT_PATCH_HASH) &&
verifier.VerifyVector(patch_hash()) &&
VerifyOffset(verifier, VT_USERS) &&
verifier.VerifyVector(users()) &&
verifier.VerifyVectorOfTables(users()) &&
VerifyOffset(verifier, VT_INPUTS) &&
verifier.VerifyVector(inputs()) &&
verifier.VerifyVectorOfTables(inputs()) &&
VerifyOffset(verifier, VT_OUTPUT) &&
verifier.VerifyVector(output()) &&
verifier.EndTable();
}
};
struct LedgerBlockBuilder {
typedef LedgerBlock Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(flatbuffers::Offset<flatbuffers::String> version) {
fbb_.AddOffset(LedgerBlock::VT_VERSION, version);
}
void add_seq_no(uint64_t seq_no) {
fbb_.AddElement<uint64_t>(LedgerBlock::VT_SEQ_NO, seq_no, 0);
}
void add_time(uint64_t time) {
fbb_.AddElement<uint64_t>(LedgerBlock::VT_TIME, time, 0);
}
void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
fbb_.AddOffset(LedgerBlock::VT_LCL, lcl);
}
void add_state_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state_hash) {
fbb_.AddOffset(LedgerBlock::VT_STATE_HASH, state_hash);
}
void add_patch_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> patch_hash) {
fbb_.AddOffset(LedgerBlock::VT_PATCH_HASH, patch_hash);
}
void add_users(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> users) {
fbb_.AddOffset(LedgerBlock::VT_USERS, users);
}
void add_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs) {
fbb_.AddOffset(LedgerBlock::VT_INPUTS, inputs);
}
void add_output(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output) {
fbb_.AddOffset(LedgerBlock::VT_OUTPUT, output);
}
explicit LedgerBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<LedgerBlock> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<LedgerBlock>(end);
return o;
}
};
inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlock(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::String> version = 0,
uint64_t seq_no = 0,
uint64_t time = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state_hash = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> patch_hash = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> users = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output = 0) {
LedgerBlockBuilder builder_(_fbb);
builder_.add_time(time);
builder_.add_seq_no(seq_no);
builder_.add_output(output);
builder_.add_inputs(inputs);
builder_.add_users(users);
builder_.add_patch_hash(patch_hash);
builder_.add_state_hash(state_hash);
builder_.add_lcl(lcl);
builder_.add_version(version);
return builder_.Finish();
}
inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlockDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const char *version = nullptr,
uint64_t seq_no = 0,
uint64_t time = 0,
const std::vector<uint8_t> *lcl = nullptr,
const std::vector<uint8_t> *state_hash = nullptr,
const std::vector<uint8_t> *patch_hash = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *users = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs = nullptr,
const std::vector<uint8_t> *output = nullptr) {
auto version__ = version ? _fbb.CreateString(version) : 0;
auto lcl__ = lcl ? _fbb.CreateVector<uint8_t>(*lcl) : 0;
auto state_hash__ = state_hash ? _fbb.CreateVector<uint8_t>(*state_hash) : 0;
auto patch_hash__ = patch_hash ? _fbb.CreateVector<uint8_t>(*patch_hash) : 0;
auto users__ = users ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*users) : 0;
auto inputs__ = inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*inputs) : 0;
auto output__ = output ? _fbb.CreateVector<uint8_t>(*output) : 0;
return msg::fbuf::ledgermsg::CreateLedgerBlock(
_fbb,
version__,
seq_no,
time,
lcl__,
state_hash__,
patch_hash__,
users__,
inputs__,
output__);
}
inline const msg::fbuf::ledgermsg::LedgerBlock *GetLedgerBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
}
inline const msg::fbuf::ledgermsg::LedgerBlock *GetSizePrefixedLedgerBlock(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
}
inline LedgerBlock *GetMutableLedgerBlock(void *buf) {
return flatbuffers::GetMutableRoot<LedgerBlock>(buf);
}
inline bool VerifyLedgerBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
}
inline bool VerifySizePrefixedLedgerBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
}
inline void FinishLedgerBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedLedgerBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace ledgermsg
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_

154
src/msg/fbuf/p2pmsg.fbs Normal file
View File

@@ -0,0 +1,154 @@
// IDL file for p2p message definitions.
// flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg.fbs
namespace msg.fbuf.p2pmsg;
union P2PMsgContent {
PeerChallengeMsg,
PeerChallengeResponseMsg,
NonUnlProposalMsg,
ProposalMsg,
NplMsg,
HpfsRequestMsg,
HpfsResponseMsg,
PeerRequirementAnnouncementMsg,
PeerCapacityAnnouncementMsg,
PeerListRequestMsg,
PeerListResponseMsg
}
table P2PMsg {
hp_version:string;
created_on:uint64;
content:P2PMsgContent;
}
table PeerChallengeMsg {
contract_id:string;
roundtime:uint32;
challenge:[ubyte];
}
table PeerChallengeResponseMsg {
challenge:[ubyte];
pubkey:[ubyte];
sig:[ubyte];
}
table UserInput {
input_container:[ubyte];
sig:[ubyte];
protocol:uint8;
}
table UserInputGroup {
pubkey:[ubyte];
messages:[UserInput];
}
table NonUnlProposalMsg {
user_inputs:[UserInputGroup];
}
table ProposalMsg {
pubkey:[ubyte]; // Sender pubkey.
sig:[ubyte]; // Signature of the field data.
stage:uint8;
time:uint64;
roundtime:uint32;
nonce: [ubyte];
users:[ByteArray];
input_hashes:[ByteArray];
output_hash:[ubyte];
output_sig:[ubyte];
state_hash: [ubyte];
patch_hash: [ubyte];
last_primary_shard_id:SequenceHash;
last_blob_shard_id: SequenceHash;
// Make sure to update signature generation/verification whenever these fields are changed.
}
table NplMsg {
pubkey:[ubyte]; // Sender pubkey.
sig:[ubyte]; // Signature of the field data.
data:[ubyte];
lcl_id:SequenceHash;
// Make sure to update signature generation/verification whenever these fields are changed.
}
table HpfsRequestMsg {
mount_id: uint32;
parent_path:string;
is_file:bool;
block_id:int32;
expected_hash:[ubyte];
}
union HpfsResponse {
HpfsFileHashMapResponse,
HpfsBlockResponse,
HpfsFsEntryResponse
}
table HpfsResponseMsg{
hash:[ubyte];
path: string;
mount_id: uint32;
content:HpfsResponse;
}
table HpfsFsEntryResponse{
entries: [HpfsFSHashEntry];
}
table HpfsFileHashMapResponse{
file_length:uint64;
hash_map:[ubyte];
}
table HpfsBlockResponse{
block_id:uint32;
data: [ubyte];
}
table HpfsFSHashEntry{
name: string;
is_file: bool;
hash: [ubyte];
}
table PeerRequirementAnnouncementMsg{
need_consensus_msg_forwarding: bool;
}
table PeerCapacityAnnouncementMsg{
available_capacity:int16;
timestamp:uint64;
}
table PeerListRequestMsg{
}
table PeerListResponseMsg{
peer_list: [PeerProperties];
}
table PeerProperties {
host_address:string;
port:uint16;
available_capacity:int16;
timestamp:uint64;
}
table SequenceHash {
seq_no: uint64;
hash: [ubyte];
}
table ByteArray { // To help represent list of byte arrays
array:[ubyte];
}
root_type P2PMsg; //root type for all messages

18
src/msg/fbuf/p2pmsg_container.fbs
View File

@@ -1,18 +0,0 @@
//IDL file for p2p message container schema.
/*Message content need to be serialised and parsed into container, we need two root types in flatbuffs to generat
needed api methods. Since Flatbuff only support one rrot at each IDL files, we had to seperate message schema to 2 files.
*/
include "p2pmsg_content.fbs";
namespace msg.fbuf.p2pmsg;
table Container { //root type for message
version:uint16;
timestamp:uint64;
pubkey:[ubyte];
lcl:[ubyte];
last_primary_shard_id:Sequence_Hash;
signature:[ubyte]; // signature of the message content
content:[ubyte]; // message content: byte array of proposal,npl,etc
}
root_type Container;

208
src/msg/fbuf/p2pmsg_container_generated.h
View File

@@ -1,208 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_
#define FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_
#include "flatbuffers/flatbuffers.h"
#include "common_schema_generated.h"
#include "p2pmsg_content_generated.h"
namespace msg {
namespace fbuf {
namespace p2pmsg {
struct Container;
struct ContainerBuilder;
struct Container FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef ContainerBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_TIMESTAMP = 6,
VT_PUBKEY = 8,
VT_LCL = 10,
VT_LAST_PRIMARY_SHARD_ID = 12,
VT_SIGNATURE = 14,
VT_CONTENT = 16
};
uint16_t version() const {
return GetField<uint16_t>(VT_VERSION, 0);
}
bool mutate_version(uint16_t _version) {
return SetField<uint16_t>(VT_VERSION, _version, 0);
}
uint64_t timestamp() const {
return GetField<uint64_t>(VT_TIMESTAMP, 0);
}
bool mutate_timestamp(uint64_t _timestamp) {
return SetField<uint64_t>(VT_TIMESTAMP, _timestamp, 0);
}
const flatbuffers::Vector<uint8_t> *pubkey() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
flatbuffers::Vector<uint8_t> *mutable_pubkey() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
const flatbuffers::Vector<uint8_t> *lcl() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
flatbuffers::Vector<uint8_t> *mutable_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
const msg::fbuf::p2pmsg::Sequence_Hash *last_primary_shard_id() const {
return GetPointer<const msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
}
msg::fbuf::p2pmsg::Sequence_Hash *mutable_last_primary_shard_id() {
return GetPointer<msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
}
const flatbuffers::Vector<uint8_t> *signature() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_SIGNATURE);
}
flatbuffers::Vector<uint8_t> *mutable_signature() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_SIGNATURE);
}
const flatbuffers::Vector<uint8_t> *content() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CONTENT);
}
flatbuffers::Vector<uint8_t> *mutable_content() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_CONTENT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<uint16_t>(verifier, VT_VERSION) &&
VerifyField<uint64_t>(verifier, VT_TIMESTAMP) &&
VerifyOffset(verifier, VT_PUBKEY) &&
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_LCL) &&
verifier.VerifyVector(lcl()) &&
VerifyOffset(verifier, VT_LAST_PRIMARY_SHARD_ID) &&
verifier.VerifyTable(last_primary_shard_id()) &&
VerifyOffset(verifier, VT_SIGNATURE) &&
verifier.VerifyVector(signature()) &&
VerifyOffset(verifier, VT_CONTENT) &&
verifier.VerifyVector(content()) &&
verifier.EndTable();
}
};
struct ContainerBuilder {
typedef Container Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(uint16_t version) {
fbb_.AddElement<uint16_t>(Container::VT_VERSION, version, 0);
}
void add_timestamp(uint64_t timestamp) {
fbb_.AddElement<uint64_t>(Container::VT_TIMESTAMP, timestamp, 0);
}
void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
fbb_.AddOffset(Container::VT_PUBKEY, pubkey);
}
void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
fbb_.AddOffset(Container::VT_LCL, lcl);
}
void add_last_primary_shard_id(flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id) {
fbb_.AddOffset(Container::VT_LAST_PRIMARY_SHARD_ID, last_primary_shard_id);
}
void add_signature(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature) {
fbb_.AddOffset(Container::VT_SIGNATURE, signature);
}
void add_content(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content) {
fbb_.AddOffset(Container::VT_CONTENT, content);
}
explicit ContainerBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<Container> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Container>(end);
return o;
}
};
inline flatbuffers::Offset<Container> CreateContainer(
flatbuffers::FlatBufferBuilder &_fbb,
uint16_t version = 0,
uint64_t timestamp = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content = 0) {
ContainerBuilder builder_(_fbb);
builder_.add_timestamp(timestamp);
builder_.add_content(content);
builder_.add_signature(signature);
builder_.add_last_primary_shard_id(last_primary_shard_id);
builder_.add_lcl(lcl);
builder_.add_pubkey(pubkey);
builder_.add_version(version);
return builder_.Finish();
}
inline flatbuffers::Offset<Container> CreateContainerDirect(
flatbuffers::FlatBufferBuilder &_fbb,
uint16_t version = 0,
uint64_t timestamp = 0,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<uint8_t> *lcl = nullptr,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
const std::vector<uint8_t> *signature = nullptr,
const std::vector<uint8_t> *content = nullptr) {
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
auto lcl__ = lcl ? _fbb.CreateVector<uint8_t>(*lcl) : 0;
auto signature__ = signature ? _fbb.CreateVector<uint8_t>(*signature) : 0;
auto content__ = content ? _fbb.CreateVector<uint8_t>(*content) : 0;
return msg::fbuf::p2pmsg::CreateContainer(
_fbb,
version,
timestamp,
pubkey__,
lcl__,
last_primary_shard_id,
signature__,
content__);
}
inline const msg::fbuf::p2pmsg::Container *GetContainer(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::p2pmsg::Container>(buf);
}
inline const msg::fbuf::p2pmsg::Container *GetSizePrefixedContainer(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::p2pmsg::Container>(buf);
}
inline Container *GetMutableContainer(void *buf) {
return flatbuffers::GetMutableRoot<Container>(buf);
}
inline bool VerifyContainerBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::p2pmsg::Container>(nullptr);
}
inline bool VerifySizePrefixedContainerBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::p2pmsg::Container>(nullptr);
}
inline void FinishContainerBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::p2pmsg::Container> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedContainerBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::p2pmsg::Container> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace p2pmsg
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_

141
src/msg/fbuf/p2pmsg_content.fbs
View File

@@ -1,141 +0,0 @@
// IDL file for p2p message content schema.
// flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg_content.fbs
include "common_schema.fbs";
namespace msg.fbuf.p2pmsg;
table Peer_Challenge_Message {
contract_id:string;
roundtime:uint32;
challenge:string;
}
table Peer_Challenge_Response_Message {
challenge:string;
sig:[ubyte];
}
table UserInput {
input_container:[ubyte];
signature:[ubyte];
protocol:uint8;
}
table UserInputGroup {
pubkey:[ubyte];
messages:[UserInput];
}
union Message {
Peer_Challenge_Response_Message,
Peer_Challenge_Message,
NonUnl_Proposal_Message,
Proposal_Message,
Npl_Message,
Hpfs_Request_Message,
Hpfs_Response_Message,
Peer_Requirement_Announcement_Message,
Peer_List_Request_Message,
Peer_List_Response_Message,
Available_Capacity_Announcement_Message
} //message content type
table Content {
message:Message;
}
table NonUnl_Proposal_Message {
user_inputs:[UserInputGroup];
}
table Proposal_Message { //Proposal type message schema
stage:uint8;
time:uint64;
roundtime:uint32;
nonce: [ubyte];
users:[ByteArray];
input_hashes:[ByteArray];
last_blob_shard_id: Sequence_Hash;
output_hash:[ubyte];
output_sig:[ubyte];
state_hash: [ubyte];
patch_hash: [ubyte];
}
table Sequence_Hash {
shard_seq_no: uint64;
shard_hash: [ubyte];
}
table Npl_Message { //NPL type message schema
data:[ubyte];
}
enum Ledger_Response_Error : ubyte
{
None = 0,
Invalid_Min_Ledger = 1,
Req_Ledger_Not_Found = 2
}
table Hpfs_Request_Message { //Hpfs request message schema
mount_id: uint32;
parent_path:string;
is_file:bool;
block_id:int32;
expected_hash:[ubyte];
}
union Hpfs_Response{ File_HashMap_Response, Block_Response, Fs_Entry_Response }
table Hpfs_Response_Message{
hpfs_response:Hpfs_Response;
hash:[ubyte];
path: string;
mount_id: uint32;
}
table Fs_Entry_Response{
entries: [Hpfs_FS_Hash_Entry];
}
table File_HashMap_Response{
file_length:uint64;
hash_map:[ubyte];
}
table Block_Response{
block_id:uint32;
data: [ubyte];
}
table Hpfs_FS_Hash_Entry{
name: string;
is_file: bool;
hash: [ubyte];
}
table Peer_Requirement_Announcement_Message{
need_consensus_msg_forwarding: bool;
}
table Available_Capacity_Announcement_Message{
available_capacity:int16;
timestamp:uint64;
}
table Peer_List_Request_Message{
}
table Peer_List_Response_Message{
peer_list: [Peer_Properties];
}
table Peer_Properties {
host_address:string;
port:uint16;
available_capacity:int16;
timestamp:uint64;
}
root_type Content; //root type for message content

2070
src/msg/fbuf/p2pmsg_content_generated.h
View File

File diff suppressed because it is too large Load Diff

554
src/msg/fbuf/p2pmsg_conversion.cpp Normal file
View File

@@ -0,0 +1,554 @@
#include "../../hpfs/hpfs_mount.hpp"
#include "../../unl.hpp"
#include "../../crypto.hpp"
#include "../../p2p/p2p.hpp"
#include "common_helpers.hpp"
#include "flatbuf_hasher.hpp"
#include "p2pmsg_conversion.hpp"
namespace msg::fbuf::p2pmsg
{
// Length of a peer connection challange.
constexpr size_t PEERCHALLENGE_LEN = 16;
// Max size of messages which are subjected to time (too old) check.
constexpr size_t MAX_SIZE_FOR_TIME_CHECK = 1 * 1024 * 1024; // 1 MB
/**
* This section contains Flatbuffer message reading/writing helpers.
* These helpers are mainly used by peer_session_handler and other components which sends outgoing p2p messages.
*
* A p2p flatbuffer message is a bucket with hp version and the message 'content'.
*/
//---Flatbuf to std---//
bool verify_peer_message(std::string_view message)
{
// Accessing message buffer
const uint8_t *buf = reinterpret_cast<const uint8_t *>(message.data());
const size_t buf_size = message.size();
// Verify container message using flatbuffer verifier
flatbuffers::Verifier verifier(buf, buf_size, 16, 100);
return VerifyP2PMsgBuffer(verifier);
}
const p2p::peer_message_info get_peer_message_info(std::string_view message)
{
const auto p2p_msg = p2pmsg::GetP2PMsg(message.data());
// Check message timestamp (ignore this for large messages).
if (message.size() <= MAX_SIZE_FOR_TIME_CHECK)
{
const uint64_t time_now = util::get_epoch_milliseconds();
if (p2p_msg->created_on() < (time_now - (conf::cfg.contract.roundtime * 4)))
{
LOG_DEBUG << "Peer message is too old.";
return p2p::peer_message_info{NULL, P2PMsgContent_NONE, 0};
}
}
return p2p::peer_message_info{p2p_msg, p2p_msg->content_type(), p2p_msg->created_on()};
}
bool verify_proposal_msg_signature(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
// Get hash of proposal data field values and verify the signature against the hash.
flatbuf_hasher hasher;
hasher.add(msg.stage());
hasher.add(msg.time());
hasher.add(msg.roundtime());
hasher.add(msg.nonce());
hasher.add(msg.users());
hasher.add(msg.input_hashes());
hasher.add(msg.output_hash());
hasher.add(msg.output_sig());
hasher.add(msg.state_hash());
hasher.add(msg.patch_hash());
hasher.add(msg.last_primary_shard_id());
hasher.add(msg.last_blob_shard_id());
return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), flatbuf_bytes_to_sv(msg.pubkey())) == 0;
}
bool verify_npl_msg_signature(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_NplMsg();
// Get hash of npl message field values and verify the signature against the hash.
flatbuf_hasher hasher;
hasher.add(msg.data());
hasher.add(msg.lcl_id());
return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), flatbuf_bytes_to_sv(msg.pubkey())) == 0;
}
const p2p::peer_challenge create_peer_challenge_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_PeerChallengeMsg();
return {
std::string(flatbuf_str_to_sv(msg.contract_id())),
msg.roundtime(),
std::string(flatbuf_bytes_to_sv(msg.challenge()))};
}
const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_PeerChallengeResponseMsg();
return {
std::string(flatbuf_bytes_to_sv(msg.challenge())),
std::string(flatbuf_bytes_to_sv(msg.sig())),
std::string(flatbuf_bytes_to_sv(msg.pubkey()))};
}
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
p2p::proposal p;
p.pubkey = flatbuf_bytes_to_sv(msg.pubkey());
p.sent_timestamp = mi.originated_on;
p.recv_timestamp = util::get_epoch_milliseconds();
p.time = msg.time();
p.roundtime = msg.roundtime();
p.nonce = flatbuf_bytes_to_sv(msg.nonce());
p.stage = msg.stage();
p.state_hash = flatbuf_bytes_to_sv(msg.state_hash());
p.patch_hash = flatbuf_bytes_to_sv(msg.patch_hash());
p.last_primary_shard_id = flatbuf_seqhash_to_seqhash(msg.last_primary_shard_id());
p.last_blob_shard_id = flatbuf_seqhash_to_seqhash(msg.last_blob_shard_id());
if (msg.users())
p.users = flatbuf_bytearrayvector_to_stringlist(msg.users());
if (msg.input_hashes())
p.input_hashes = flatbuf_bytearrayvector_to_stringlist(msg.input_hashes());
if (msg.output_hash())
p.output_hash = flatbuf_bytes_to_sv(msg.output_hash());
if (msg.output_sig())
p.output_sig = flatbuf_bytes_to_sv(msg.output_sig());
return p;
}
const p2p::npl_message create_npl_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_NplMsg();
return {
std::string(flatbuf_bytes_to_sv(msg.pubkey())),
flatbuf_seqhash_to_seqhash(msg.lcl_id()),
std::string(flatbuf_bytes_to_sv(msg.data()))};
}
const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_NonUnlProposalMsg();
p2p::nonunl_proposal nup;
if (msg.user_inputs())
nup.user_inputs = flatbuf_user_input_group_to_user_input_map(msg.user_inputs());
return nup;
}
const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_PeerListResponseMsg();
return flatbuf_peer_propertieslist_to_peer_propertiesvector(msg.peer_list());
}
const p2p::peer_capacity_announcement create_peer_capacity_announcement_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_PeerCapacityAnnouncementMsg();
return {
msg.available_capacity(),
msg.timestamp()};
}
const p2p::peer_requirement_announcement create_peer_requirement_announcement_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_PeerRequirementAnnouncementMsg();
return {
msg.need_consensus_msg_forwarding()};
}
const p2p::hpfs_request create_hpfs_request_from_msg(const p2p::peer_message_info &mi)
{
const auto &msg = *mi.p2p_msg->content_as_HpfsRequestMsg();
p2p::hpfs_request hr;
hr.mount_id = msg.mount_id();
hr.block_id = msg.block_id();
hr.is_file = msg.is_file();
hr.parent_path = flatbuf_str_to_sv(msg.parent_path());
hr.expected_hash = flatbuf_bytes_to_hash(msg.expected_hash());
return hr;
}
p2p::sequence_hash flatbuf_seqhash_to_seqhash(const SequenceHash *fbseqhash)
{
return {
fbseqhash->seq_no(),
flatbuf_bytes_to_hash(fbseqhash->hash())};
}
const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec)
{
std::set<std::string> set;
for (const auto el : *fbvec)
set.emplace(std::string(flatbuf_bytes_to_sv(el->array())));
return set;
}
const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec)
{
std::unordered_map<std::string, std::list<usr::submitted_user_input>> map;
map.reserve(fbvec->size());
for (const UserInputGroup *group : *fbvec)
{
std::list<usr::submitted_user_input> user_inputs_list;
for (const auto msg : *group->messages())
{
user_inputs_list.push_back(usr::submitted_user_input{
std::string(flatbuf_bytes_to_sv(msg->input_container())),
std::string(flatbuf_bytes_to_sv(msg->sig())),
static_cast<util::PROTOCOL>(msg->protocol())});
}
map.emplace(flatbuf_bytes_to_sv(group->pubkey()), std::move(user_inputs_list));
}
return map;
}
void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>> *fhashes)
{
for (const HpfsFSHashEntry *f_hash : *fhashes)
{
p2p::hpfs_fs_hash_entry entry;
entry.name = flatbuf_str_to_sv(f_hash->name());
entry.is_file = f_hash->is_file();
entry.hash = flatbuf_bytes_to_hash(f_hash->hash());
fs_entries.emplace(entry.name, std::move(entry));
}
}
const std::vector<conf::peer_properties>
flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<PeerProperties>> *fbvec)
{
std::vector<conf::peer_properties> peers;
for (const PeerProperties *peer : *fbvec)
{
conf::peer_properties properties;
properties.ip_port.host_address = flatbuf_str_to_sv(peer->host_address());
properties.ip_port.port = peer->port();
properties.timestamp = peer->timestamp();
properties.available_capacity = peer->available_capacity();
peers.push_back(properties);
}
return peers;
}
//---std to Flatbuf---//
const std::string generate_proposal_signature(const p2p::proposal &p)
{
flatbuf_hasher hasher;
hasher.add(p.stage);
hasher.add(p.time);
hasher.add(p.roundtime);
hasher.add(p.nonce);
hasher.add(p.users);
hasher.add(p.input_hashes);
hasher.add(p.output_hash);
hasher.add(p.output_sig);
hasher.add(p.state_hash);
hasher.add(p.patch_hash);
hasher.add(p.last_primary_shard_id);
hasher.add(p.last_blob_shard_id);
return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
}
const std::string generate_npl_signature(std::string_view data, const p2p::sequence_hash &lcl_id)
{
flatbuf_hasher hasher;
hasher.add(data);
hasher.add(lcl_id);
return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
}
void create_p2p_msg(flatbuffers::FlatBufferBuilder &builder, const msg::fbuf::p2pmsg::P2PMsgContent content_type, const flatbuffers::Offset<void> content)
{
const auto p2pmsg = CreateP2PMsg(builder,
sv_to_flatbuf_str(builder, conf::cfg.hp_version),
util::get_epoch_milliseconds(),
content_type,
content);
builder.Finish(p2pmsg);
}
void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &builder, std::string &challenge)
{
// We calculate the peer challenge to be a random string.
crypto::random_bytes(challenge, PEERCHALLENGE_LEN);
const auto msg = CreatePeerChallengeMsg(
builder,
sv_to_flatbuf_str(builder, conf::cfg.contract.id),
conf::cfg.contract.roundtime,
sv_to_flatbuf_bytes(builder, challenge));
create_p2p_msg(builder, P2PMsgContent_PeerChallengeMsg, msg.Union());
}
void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &builder, const std::string &challenge)
{
const std::string sig = crypto::sign(challenge, conf::cfg.node.private_key);
const auto msg = CreatePeerChallengeResponseMsg(
builder,
sv_to_flatbuf_bytes(builder, challenge),
sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
sv_to_flatbuf_bytes(builder, sig));
create_p2p_msg(builder, P2PMsgContent_PeerChallengeResponseMsg, msg.Union());
}
void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::nonunl_proposal &nup)
{
const auto msg = CreateNonUnlProposalMsg(
builder,
user_input_map_to_flatbuf_user_input_group(builder, nup.user_inputs));
create_p2p_msg(builder, P2PMsgContent_NonUnlProposalMsg, msg.Union());
}
void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p)
{
const auto msg = CreateProposalMsg(
builder,
sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
sv_to_flatbuf_bytes(builder, generate_proposal_signature(p)),
p.stage,
p.time,
p.roundtime,
sv_to_flatbuf_bytes(builder, p.nonce),
stringlist_to_flatbuf_bytearrayvector(builder, p.users),
stringlist_to_flatbuf_bytearrayvector(builder, p.input_hashes),
sv_to_flatbuf_bytes(builder, p.output_hash),
sv_to_flatbuf_bytes(builder, p.output_sig),
hash_to_flatbuf_bytes(builder, p.state_hash),
hash_to_flatbuf_bytes(builder, p.patch_hash),
seqhash_to_flatbuf_seqhash(builder, p.last_primary_shard_id),
seqhash_to_flatbuf_seqhash(builder, p.last_blob_shard_id));
create_p2p_msg(builder, P2PMsgContent_ProposalMsg, msg.Union());
}
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &builder, std::string_view data, const p2p::sequence_hash &lcl_id)
{
const auto msg = CreateNplMsg(
builder,
sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
sv_to_flatbuf_bytes(builder, generate_npl_signature(data, lcl_id)),
sv_to_flatbuf_bytes(builder, data),
seqhash_to_flatbuf_seqhash(builder, lcl_id));
create_p2p_msg(builder, P2PMsgContent_NplMsg, msg.Union());
}
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &builder, const p2p::hpfs_request &hr)
{
const auto msg = CreateHpfsRequestMsg(
builder,
hr.mount_id,
sv_to_flatbuf_str(builder, hr.parent_path),
hr.is_file,
hr.block_id,
hash_to_flatbuf_bytes(builder, hr.expected_hash));
create_p2p_msg(builder, P2PMsgContent_HpfsRequestMsg, msg.Union());
}
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash)
{
const auto child_msg = CreateHpfsFsEntryResponse(
builder,
hpfsfshashentry_to_flatbuf_hpfsfshashentry(builder, hash_nodes));
const auto msg = CreateHpfsResponseMsg(
builder,
hash_to_flatbuf_bytes(builder, expected_hash),
sv_to_flatbuf_str(builder, path),
mount_id,
HpfsResponse_HpfsFsEntryResponse,
child_msg.Union());
create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
}
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash)
{
std::string_view hashmap_sv(reinterpret_cast<const char *>(hashmap.data()), hashmap.size() * sizeof(util::h32));
const auto child_msg = CreateHpfsFileHashMapResponse(
builder,
file_length,
sv_to_flatbuf_bytes(builder, hashmap_sv));
const auto msg = CreateHpfsResponseMsg(
builder,
hash_to_flatbuf_bytes(builder, expected_hash),
sv_to_flatbuf_str(builder, path),
mount_id,
HpfsResponse_HpfsFileHashMapResponse,
child_msg.Union());
create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
}
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &builder, p2p::block_response &block_resp, const uint32_t mount_id)
{
const auto child_msg = CreateHpfsBlockResponse(
builder,
block_resp.block_id,
sv_to_flatbuf_bytes(builder, block_resp.data));
const auto msg = CreateHpfsResponseMsg(
builder,
hash_to_flatbuf_bytes(builder, block_resp.hash),
sv_to_flatbuf_str(builder, block_resp.path),
mount_id,
HpfsResponse_HpfsBlockResponse,
child_msg.Union());
create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
}
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &builder, const bool need_consensus_msg_forwarding)
{
const auto msg = CreatePeerRequirementAnnouncementMsg(
builder,
need_consensus_msg_forwarding);
create_p2p_msg(builder, P2PMsgContent_PeerRequirementAnnouncementMsg, msg.Union());
}
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &builder, const int16_t &available_capacity, const uint64_t &timestamp)
{
const auto msg = CreatePeerCapacityAnnouncementMsg(
builder,
available_capacity,
timestamp);
create_p2p_msg(builder, P2PMsgContent_PeerCapacityAnnouncementMsg, msg.Union());
}
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &builder)
{
const auto msg = CreatePeerListRequestMsg(builder);
create_p2p_msg(builder, P2PMsgContent_PeerListRequestMsg, msg.Union());
}
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
{
const auto msg = CreatePeerListResponseMsg(
builder,
peer_propertiesvector_to_flatbuf_peer_propertieslist(builder, peers, skipping_ip_port));
create_p2p_msg(builder, P2PMsgContent_PeerListResponseMsg, msg.Union());
}
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map)
{
std::vector<flatbuffers::Offset<UserInputGroup>> fbvec;
fbvec.reserve(map.size());
for (const auto &[pubkey, msglist] : map)
{
std::vector<flatbuffers::Offset<UserInput>> fbmsgsvec;
for (const usr::submitted_user_input &msg : msglist)
{
fbmsgsvec.push_back(CreateUserInput(
builder,
sv_to_flatbuf_bytes(builder, msg.input_container),
sv_to_flatbuf_bytes(builder, msg.sig),
static_cast<uint8_t>(msg.protocol)));
}
fbvec.push_back(CreateUserInputGroup(
builder,
sv_to_flatbuf_bytes(builder, pubkey),
builder.CreateVector(fbmsgsvec)));
}
return builder.CreateVector(fbvec);
}
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>>>
hpfsfshashentry_to_flatbuf_hpfsfshashentry(
flatbuffers::FlatBufferBuilder &builder,
std::vector<hpfs::child_hash_node> &hash_nodes)
{
std::vector<flatbuffers::Offset<HpfsFSHashEntry>> fbvec;
fbvec.reserve(hash_nodes.size());
for (auto const &hash_node : hash_nodes)
{
flatbuffers::Offset<HpfsFSHashEntry> hpfs_fs_entry = CreateHpfsFSHashEntry(
builder,
sv_to_flatbuf_str(builder, hash_node.name),
hash_node.is_file,
hash_to_flatbuf_bytes(builder, hash_node.hash));
fbvec.push_back(hpfs_fs_entry);
}
return builder.CreateVector(fbvec);
}
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<PeerProperties>>>
peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
{
std::vector<flatbuffers::Offset<PeerProperties>> fbvec;
fbvec.reserve(peers.size());
for (auto peer : peers)
{
// Skipping the requestedc peer from the peer list response.
if (!skipping_ip_port.has_value() || peer.ip_port != skipping_ip_port.value())
fbvec.push_back(CreatePeerProperties(
builder,
sv_to_flatbuf_str(builder, peer.ip_port.host_address),
peer.ip_port.port,
peer.available_capacity,
peer.timestamp));
}
return builder.CreateVector(fbvec);
}
const flatbuffers::Offset<msg::fbuf::p2pmsg::SequenceHash>
seqhash_to_flatbuf_seqhash(flatbuffers::FlatBufferBuilder &builder, const p2p::sequence_hash &seqhash)
{
return CreateSequenceHash(builder, seqhash.seq_no, hash_to_flatbuf_bytes(builder, seqhash.hash));
}
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set)
{
std::vector<flatbuffers::Offset<ByteArray>> fbvec;
fbvec.reserve(set.size());
for (std::string_view str : set)
fbvec.push_back(CreateByteArray(builder, sv_to_flatbuf_bytes(builder, str)));
return builder.CreateVector(fbvec);
}
}

108
src/msg/fbuf/p2pmsg_conversion.hpp Normal file
View File

@@ -0,0 +1,108 @@
#ifndef _HP_MSG_FBUF_P2PMSG_CONVERSION_
#define _HP_MSG_FBUF_P2PMSG_CONVERSION_
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "../../hpfs/hpfs_mount.hpp"
#include "p2pmsg_generated.h"
namespace msg::fbuf::p2pmsg
{
//---Flatbuf to std---//
bool verify_peer_message(std::string_view message);
const p2p::peer_message_info get_peer_message_info(std::string_view message);
bool verify_proposal_msg_signature(const p2p::peer_message_info &mi);
bool verify_npl_msg_signature(const p2p::peer_message_info &mi);
const p2p::peer_challenge create_peer_challenge_from_msg(const p2p::peer_message_info &mi);
const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const p2p::peer_message_info &mi);
const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi);
const p2p::npl_message create_npl_from_msg(const p2p::peer_message_info &mi);
const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const p2p::peer_message_info &mi);
const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const p2p::peer_message_info &mi);
const p2p::peer_capacity_announcement create_peer_capacity_announcement_from_msg(const p2p::peer_message_info &mi);
const p2p::peer_requirement_announcement create_peer_requirement_announcement_from_msg(const p2p::peer_message_info &mi);
const p2p::hpfs_request create_hpfs_request_from_msg(const p2p::peer_message_info &mi);
p2p::sequence_hash flatbuf_seqhash_to_seqhash(const msg::fbuf::p2pmsg::SequenceHash *fbseqhash);
const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec);
void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>> *fhashes);
const std::vector<conf::peer_properties>
flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<PeerProperties>> *fbvec);
//---std to Flatbuf---//
const std::string generate_proposal_signature(const p2p::proposal &p);
const std::string generate_npl_signature(std::string_view data, const p2p::sequence_hash &lcl_id);
void create_p2p_msg(flatbuffers::FlatBufferBuilder &builder, const msg::fbuf::p2pmsg::P2PMsgContent content_type, const flatbuffers::Offset<void> content);
void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &builder, std::string &challenge);
void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &builder, const std::string &challenge);
void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::nonunl_proposal &nup);
void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p);
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &builder, std::string_view data, const p2p::sequence_hash &lcl_id);
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &builder, const p2p::hpfs_request &hr);
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash);
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash);
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &builder, p2p::block_response &block_resp, const uint32_t mount_id);
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &builder, const bool need_consensus_msg_forwarding);
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &builder, const int16_t &available_capacity, const uint64_t &timestamp);
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &builder);
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>>>
hpfsfshashentry_to_flatbuf_hpfsfshashentry(
flatbuffers::FlatBufferBuilder &builder,
std::vector<hpfs::child_hash_node> &hash_nodes);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<PeerProperties>>>
peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
const flatbuffers::Offset<msg::fbuf::p2pmsg::SequenceHash>
seqhash_to_flatbuf_seqhash(flatbuffers::FlatBufferBuilder &builder, const p2p::sequence_hash &seqhash);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
}
#endif

2272
src/msg/fbuf/p2pmsg_generated.h Normal file
View File

File diff suppressed because it is too large Load Diff

800
src/msg/fbuf/p2pmsg_helpers.cpp
View File

@@ -1,800 +0,0 @@
#include "../../pchheader.hpp"
#include "../../conf.hpp"
#include "../../crypto.hpp"
#include "../../util/util.hpp"
#include "../../hplog.hpp"
#include "../../util/h32.hpp"
#include "../../unl.hpp"
#include "p2pmsg_container_generated.h"
#include "p2pmsg_content_generated.h"
#include "common_helpers.hpp"
#include "p2pmsg_helpers.hpp"
namespace msg::fbuf::p2pmsg
{
// Length of a peer connection challange.
constexpr size_t PEERCHALLENGE_LEN = 16;
// Max size of messages which are subjected to time (too old) check.
constexpr size_t MAX_SIZE_FOR_TIME_CHECK = 1 * 1024 * 1024; // 1 MB
/**
* This section contains Flatbuffer message reading/writing helpers.
* These helpers are mainly used by peer_session_handler.
*
* All Flatbuffer peer messages are 'Container' messages. 'Container' message is a bucket
* which some common headers (version, singature etc..) and the message 'Content' (Proposal, NPL etc..).
*
* Therefore, when constructing peer messages, we have to first construct 'Content' message and then
* place the 'Content' inside a 'Conatiner. 'Content' and 'Container' messages are constructed using
* Flatbuffer builders.
*
* Reading is also 2 steps because of this. We have first interprit the 'Container' message from the
* received data and then interprit the 'Content' portion of it separately to read the actual content.
*/
//---Message validation helpers---/
/**
* Verifies Conatiner message structure and outputs faltbuffer Container pointer to access the given buffer.
*
* @param container_ref A pointer reference to assign the pointer to the Container object.
* @param container_buf The buffer containing the data that should be validated and interpreted
* via the container pointer.
* @return 0 on successful verification. -1 for failure.
*/
int validate_and_extract_container(const Container **container_ref, std::string_view container_buf)
{
//Accessing message buffer
const uint8_t *container_buf_ptr = reinterpret_cast<const uint8_t *>(container_buf.data());
const size_t container_buf_size = container_buf.length();
//Defining Flatbuffer verifier (default max depth = 64, max_tables = 1000000,)
flatbuffers::Verifier container_verifier(container_buf_ptr, container_buf_size);
//Verify container message using flatbuffer verifier
if (!VerifyContainerBuffer(container_verifier))
{
LOG_DEBUG << "Flatbuffer verify: Bad peer message container.";
return -1;
}
//Get message container
const Container *container = GetContainer(container_buf_ptr);
//check protocol version of message whether it is greater than minimum supported protocol version.
const uint16_t version = container->version();
if (version < util::MIN_PEERMSG_VERSION)
{
LOG_DEBUG << "Peer message is from unsupported protocol version (" << version << ").";
return -1;
}
//check message timestamp (ignore this for large messages).
if (container_buf_size <= MAX_SIZE_FOR_TIME_CHECK)
{
const uint64_t time_now = util::get_epoch_milliseconds();
if (container->timestamp() < (time_now - conf::cfg.contract.roundtime * 4))
{
LOG_DEBUG << "Peer message is too old.";
return -1;
}
}
//Assign container and content out params.
*container_ref = container;
return 0;
}
/**
* Validates the container message signing keys to see if the message is from a trusted source (UNL).
* @return 0 on successful verification. -1 for failure.
*/
int validate_container_trust(const Container *container)
{
std::string_view msg_pubkey = flatbuff_bytes_to_sv(container->pubkey());
std::string_view msg_sig = flatbuff_bytes_to_sv(container->signature());
if (msg_pubkey.empty() || msg_sig.empty())
{
LOG_DEBUG << "Peer message key pair incomplete. Trust verification failed.";
return -1;
}
//validate if the message is not from a unl node.
if (!unl::exists(std::string(msg_pubkey)))
{
LOG_DEBUG << "Peer message pubkey verification failed. Not a UNL node.";
return -1;
}
//verify message signature.
//this is performed towards end since this is bit expensive
std::string_view msg_content = flatbuff_bytes_to_sv(container->content());
if (crypto::verify(msg_content, msg_sig, msg_pubkey) != 0)
{
LOG_DEBUG << "Peer message signature verification failed.";
return -1;
}
return 0;
}
/**
* Verifies the Content message structure and outputs faltbuffer Content pointer to access the given buffer.
*
* @param content_ref A pointer reference to assign the pointer to the Content object.
* @param content_ptr Pointer to the buffer containing the data that should validated and interpreted
* via the container pointer.
* @param content_size Data buffer size.
* @return 0 on successful verification. -1 for failure.
*/
int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size)
{
//Defining Flatbuffer verifier for message content verification.
//Since content is also serialised by using Flatbuffer we can verify it using Flatbuffer.
flatbuffers::Verifier content_verifier(content_ptr, content_size);
//verify content message using flatbuffer verifier.
if (!VerifyContainerBuffer(content_verifier))
{
LOG_DEBUG << "Flatbuffer verify: Bad content.";
return -1;
}
*content_ref = GetContent(content_ptr);
return 0;
}
//---Message reading helpers---/
/**
* Returns challenge from the peer challenge message.
* @param The Flatbuffer peer challenge message received from the peer.
* @return Peer challenge struct.
*/
const p2p::peer_challenge get_peer_challenge_from_msg(const Peer_Challenge_Message &msg)
{
return {
std::string(flatbuff_str_to_sv(msg.contract_id())),
msg.roundtime(),
std::string(flatbuff_str_to_sv(msg.challenge()))};
}
/**
* Creates a peer challenge response struct from the given peer challenge response message.
* @param The Flatbuffer peer challenge response message received from the peer.
* @return A peer challenge response struct representing the message.
*/
const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey)
{
p2p::peer_challenge_response pchalresp;
pchalresp.challenge = flatbuff_str_to_sv(msg.challenge());
pchalresp.signature = flatbuff_bytes_to_sv(msg.sig());
pchalresp.pubkey = flatbuff_bytes_to_sv(pubkey);
return pchalresp;
}
/**
* Creates a non-unl proposal stuct from the given non-unl proposal message.
* @param The Flatbuffer non-unl poporal received from the peer.
* @return A non-unl proposal struct representing the message.
*/
const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp)
{
p2p::nonunl_proposal nup;
if (msg.user_inputs())
nup.user_inputs = flatbuf_user_input_group_to_user_input_map(msg.user_inputs());
return nup;
}
/**
* Creates a proposal stuct from the given proposal message.
* @param msg The Flatbuffer poposal received from the peer.
* @return A proposal struct representing the message.
*/
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const Sequence_Hash &last_primary_shard_id_msg)
{
p2p::proposal p;
p.pubkey = flatbuff_bytes_to_sv(pubkey);
p.sent_timestamp = timestamp;
p.recv_timestamp = util::get_epoch_milliseconds();
p.time = msg.time();
p.roundtime = msg.roundtime();
p.nonce = flatbuff_bytes_to_sv(msg.nonce());
p.stage = msg.stage();
p.state_hash = flatbuff_bytes_to_sv(msg.state_hash());
p.patch_hash = flatbuff_bytes_to_sv(msg.patch_hash());
p2p::sequence_hash last_primary_shard_id;
last_primary_shard_id.seq_no = last_primary_shard_id_msg.shard_seq_no();
last_primary_shard_id.hash = flatbuff_bytes_to_hash(last_primary_shard_id_msg.shard_hash());
p.last_primary_shard_id = last_primary_shard_id;
p2p::sequence_hash last_blob_shard_id;
const Sequence_Hash &last_blob_shard_id_msg = *msg.last_blob_shard_id();
last_blob_shard_id.seq_no = last_blob_shard_id_msg.shard_seq_no();
last_blob_shard_id.hash = flatbuff_bytes_to_hash(last_blob_shard_id_msg.shard_hash());
p.last_blob_shard_id = last_blob_shard_id;
if (msg.users())
p.users = flatbuf_bytearrayvector_to_stringlist(msg.users());
if (msg.input_hashes())
p.input_hashes = flatbuf_bytearrayvector_to_stringlist(msg.input_hashes());
if (msg.output_hash())
p.output_hash = flatbuff_bytes_to_sv(msg.output_hash());
if (msg.output_sig())
p.output_sig = flatbuff_bytes_to_sv(msg.output_sig());
return p;
}
/**
* Creates a hpfs request struct from the given hpfs request message.
* @param msg Flatbuffer State request message received from the peer.
* @return A hpfs request struct representing the message.
*/
const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg)
{
p2p::hpfs_request hr;
hr.mount_id = msg.mount_id();
hr.block_id = msg.block_id();
hr.is_file = msg.is_file();
hr.parent_path = flatbuff_str_to_sv(msg.parent_path());
hr.expected_hash = flatbuff_bytes_to_hash(msg.expected_hash());
return hr;
}
/**
* Creates a peer property list from the given peer list response message.
* @param msg Flatbuffer Peer List response message received from the peer.
* @return A Peer list representing the message.
*/
const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const Peer_List_Response_Message &msg)
{
return flatbuf_peer_propertieslist_to_peer_propertiesvector(msg.peer_list());
}
//---Message creation helpers---//
/**
* Create peer challenge message from the given challenge.
* @param container_builder Flatbuffer builder for the container message.
* @param challenge Challenge message needed to convert to flatbuffer message.
*/
void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge)
{
flatbuffers::FlatBufferBuilder builder(1024);
// We calculate the peer challenge to be a random string.
// Use libsodium to generate the random challenge bytes.
challenge.resize(PEERCHALLENGE_LEN);
randombytes_buf(challenge.data(), PEERCHALLENGE_LEN);
const flatbuffers::Offset<Peer_Challenge_Message> peer_challenge_msg =
CreatePeer_Challenge_Message(
builder,
sv_to_flatbuff_str(builder, conf::cfg.contract.id),
conf::cfg.contract.roundtime,
sv_to_flatbuff_str(builder, challenge));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Challenge_Message, peer_challenge_msg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message
create_containermsg_from_content(container_builder, builder, {}, false);
}
/**
* Create peer challenge response message from the given challenge.
* @param container_builder Flatbuffer builder for the container message.
* @param challenge Message which need to be signed and placed in the container message.
*/
void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Peer_Challenge_Response_Message> challenge_resp_msg =
CreatePeer_Challenge_Response_Message(
builder,
sv_to_flatbuff_str(builder, challenge),
sv_to_flatbuff_bytes(builder, crypto::sign(challenge, conf::cfg.node.private_key)));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Challenge_Response_Message, challenge_resp_msg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, {}, true);
}
void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<NonUnl_Proposal_Message> nupmsg =
CreateNonUnl_Proposal_Message(
builder,
user_input_map_to_flatbuf_user_input_group(builder, nup.user_inputs));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_NonUnl_Proposal_Message, nupmsg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, {}, false);
}
/**
* Create proposal peer message from the given proposal struct.
* @param container_builder Flatbuffer builder for the container message.
* @param p The proposal struct to be placed in the container message.
*/
void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p)
{
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Sequence_Hash> last_blob_shard_id_msg = CreateSequence_Hash(
builder,
p.last_blob_shard_id.seq_no,
hash_to_flatbuff_bytes(builder, p.last_blob_shard_id.hash));
const flatbuffers::Offset<Proposal_Message> proposal =
CreateProposal_Message(
builder,
p.stage,
p.time,
p.roundtime,
sv_to_flatbuff_bytes(builder, p.nonce),
stringlist_to_flatbuf_bytearrayvector(builder, p.users),
stringlist_to_flatbuf_bytearrayvector(builder, p.input_hashes),
last_blob_shard_id_msg,
sv_to_flatbuff_bytes(builder, p.output_hash),
sv_to_flatbuff_bytes(builder, p.output_sig),
hash_to_flatbuff_bytes(builder, p.state_hash),
hash_to_flatbuff_bytes(builder, p.patch_hash));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Proposal_Message, proposal.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, p.last_primary_shard_id, true);
}
/**
* Ctreat npl message from the given npl output srtuct.
* @param container_builder Flatbuffer builder for the container message.
* @param msg The message to be sent as NPL message.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Npl_Message> npl =
CreateNpl_Message(
builder,
sv_to_flatbuff_bytes(builder, msg));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Npl_Message, npl.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
}
/**
* Create hpfs request message from the given hpfs request struct.
* @param container_builder Flatbuffer builder for the container message.
* @param hr The hpfs request struct to be placed in the container message.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
flatbuffers::Offset<Hpfs_Request_Message> srmsg =
CreateHpfs_Request_Message(
builder,
hr.mount_id,
sv_to_flatbuff_str(builder, hr.parent_path),
hr.is_file,
hr.block_id,
hash_to_flatbuff_bytes(builder, hr.expected_hash));
flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Request_Message, srmsg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
}
/**
* Create content response message from the given content response.
* @param container_builder Flatbuffer builder for the container message.
* @param path The path of the directory.
* @param mount_id The mount id of the relavent hpfs mount.
* @param hash_nodes File or directory entries with hashes in the given parent path.
* @param expected_hash The exptected hash of the requested path.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Fs_Entry_Response> resp =
CreateFs_Entry_Response(
builder,
hpfsfshashentry_to_flatbuff_hpfsfshashentry(builder, hash_nodes));
const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
builder, Hpfs_Response_Fs_Entry_Response,
resp.Union(),
hash_to_flatbuff_bytes(builder, expected_hash),
sv_to_flatbuff_str(builder, path), mount_id);
flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
}
/**
* Create content response message from the given content response.
* @param container_builder Flatbuffer builder for the container message.
* @param path The path of the directory.
* @param mount_id The mount id of the relavent hpfs mount.
* @param hashmap Hashmap of the file
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id)
{
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
std::string_view hashmap_sv(reinterpret_cast<const char *>(hashmap.data()), hashmap.size() * sizeof(util::h32));
const flatbuffers::Offset<File_HashMap_Response> resp =
CreateFile_HashMap_Response(
builder,
file_length,
sv_to_flatbuff_bytes(builder, hashmap_sv));
const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
builder,
Hpfs_Response_File_HashMap_Response,
resp.Union(),
hash_to_flatbuff_bytes(builder, expected_hash),
sv_to_flatbuff_str(builder, path), mount_id);
flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
}
/**
* Create content response message from the given content response.
* @param container_builder Flatbuffer builder for the container message.
* @param block_resp Block response struct to place in the message.
* @param mount_id The mount id of the relavent hpfs mount.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
const p2p::sequence_hash &last_primary_shard_id)
{
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Block_Response> resp =
CreateBlock_Response(
builder,
block_resp.block_id,
sv_to_flatbuff_bytes(builder, block_resp.data));
const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
builder,
Hpfs_Response_Block_Response,
resp.Union(),
hash_to_flatbuff_bytes(builder, block_resp.hash),
sv_to_flatbuff_str(builder, block_resp.path), mount_id);
flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
}
/**
* Create connected status announcement message.
* @param container_builder Flatbuffer builder for the container message.
* @param need_consensus_msg_forwarding True if number of connections are below threshold and false otherwise.
*/
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Peer_Requirement_Announcement_Message> announcement =
CreatePeer_Requirement_Announcement_Message(
builder,
need_consensus_msg_forwarding);
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Requirement_Announcement_Message, announcement.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
}
/**
* Create available capacity announcement message.
* @param container_builder Flatbuffer builder for the container message.
* @param available_capacity Number of incoming connection slots available, -1 means there's no limitation for connections.
* @param timestamp Announced timestamp.
*/
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Available_Capacity_Announcement_Message> announcement =
CreateAvailable_Capacity_Announcement_Message(
builder,
available_capacity,
timestamp);
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Available_Capacity_Announcement_Message, announcement.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
}
/**
* Create peer list request message.
* @param container_builder Flatbuffer builder for the container message.
*/
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Peer_List_Request_Message> request =
CreatePeer_List_Request_Message(
builder);
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_List_Request_Message, request.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
}
/**
* Create peer list response message.
* @param container_builder Flatbuffer builder for the container message.
* @param peers Peer list to be sent to another peer.
* @param skipping_peer Peer that does not need to be sent.
*/
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Peer_List_Response_Message> response =
CreatePeer_List_Response_Message(
builder,
peer_propertiesvector_to_flatbuf_peer_propertieslist(builder, peers, skipping_ip_port));
const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_List_Response_Message, response.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
}
/**
* Creates a Flatbuffer container message from the given Content message.
* @param container_builder The Flatbuffer builder to which the final container message should be written to.
* @param content_builder The Flatbuffer builder containing the content message that should be placed
* inside the container message.
* @param sign Whether to sign the message content.
*/
void create_containermsg_from_content(
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, const p2p::sequence_hash &last_primary_shard_id, const bool sign)
{
const uint8_t *content_buf = content_builder.GetBufferPointer();
const flatbuffers::uoffset_t content_size = content_builder.GetSize();
// Create container message content from serialised content from previous step.
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content = container_builder.CreateVector(content_buf, content_size);
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey_offset = 0;
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> sig_offset = 0;
if (sign)
{
// Sign message content with this node's private key.
std::string_view content_to_sign(reinterpret_cast<const char *>(content_buf), content_size);
sig_offset = sv_to_flatbuff_bytes(container_builder, crypto::sign(content_to_sign, conf::cfg.node.private_key));
pubkey_offset = sv_to_flatbuff_bytes(container_builder, conf::cfg.node.public_key);
}
const flatbuffers::Offset<Sequence_Hash> last_primary_shard_id_msg = CreateSequence_Hash(
container_builder,
last_primary_shard_id.seq_no,
hash_to_flatbuff_bytes(container_builder, last_primary_shard_id.hash));
const flatbuffers::Offset<Container> container_message = CreateContainer(
container_builder,
util::PEERMSG_VERSION,
util::get_epoch_milliseconds(),
pubkey_offset,
0,
last_primary_shard_id_msg,
sig_offset,
content);
// Finish building message container to get serialised message.
container_builder.Finish(container_message);
}
//---Conversion helpers from flatbuffers data types to std data types---//
const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec)
{
std::unordered_map<std::string, std::list<usr::submitted_user_input>> map;
map.reserve(fbvec->size());
for (const UserInputGroup *group : *fbvec)
{
std::list<usr::submitted_user_input> user_inputs_list;
for (const auto msg : *group->messages())
{
user_inputs_list.push_back(usr::submitted_user_input{
std::string(flatbuff_bytes_to_sv(msg->input_container())),
std::string(flatbuff_bytes_to_sv(msg->signature())),
static_cast<util::PROTOCOL>(msg->protocol())});
}
map.emplace(flatbuff_bytes_to_sv(group->pubkey()), std::move(user_inputs_list));
}
return map;
}
//---Conversion helpers from std data types to flatbuffers data types---//
//---These are used in constructing Flatbuffer messages using builders---//
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map)
{
std::vector<flatbuffers::Offset<UserInputGroup>> fbvec;
fbvec.reserve(map.size());
for (const auto &[pubkey, msglist] : map)
{
std::vector<flatbuffers::Offset<UserInput>> fbmsgsvec;
for (const usr::submitted_user_input &msg : msglist)
{
fbmsgsvec.push_back(CreateUserInput(
builder,
sv_to_flatbuff_bytes(builder, msg.input_container),
sv_to_flatbuff_bytes(builder, msg.sig),
static_cast<uint8_t>(msg.protocol)));
}
fbvec.push_back(CreateUserInputGroup(
builder,
sv_to_flatbuff_bytes(builder, pubkey),
builder.CreateVector(fbmsgsvec)));
}
return builder.CreateVector(fbvec);
}
void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> *fhashes)
{
for (const Hpfs_FS_Hash_Entry *f_hash : *fhashes)
{
p2p::hpfs_fs_hash_entry entry;
entry.name = flatbuff_str_to_sv(f_hash->name());
entry.is_file = f_hash->is_file();
entry.hash = flatbuff_bytes_to_hash(f_hash->hash());
fs_entries.emplace(entry.name, std::move(entry));
}
}
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>>>
hpfsfshashentry_to_flatbuff_hpfsfshashentry(
flatbuffers::FlatBufferBuilder &builder,
std::vector<hpfs::child_hash_node> &hash_nodes)
{
std::vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> fbvec;
fbvec.reserve(hash_nodes.size());
for (auto const &hash_node : hash_nodes)
{
flatbuffers::Offset<Hpfs_FS_Hash_Entry> hpfs_fs_entry = CreateHpfs_FS_Hash_Entry(
builder,
sv_to_flatbuff_str(builder, hash_node.name),
hash_node.is_file,
hash_to_flatbuff_bytes(builder, hash_node.hash));
fbvec.push_back(hpfs_fs_entry);
}
return builder.CreateVector(fbvec);
}
/**
* Create peer list message from the given vector of peer properties structs.
* @param container_builder Flatbuffer builder for the container message.
* @param peers The Vector of peer properties to be placed in the container message.
* @param skipping_peer Peer that does not need to be sent.
*/
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>>>
peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
{
std::vector<flatbuffers::Offset<Peer_Properties>> fbvec;
fbvec.reserve(peers.size());
for (auto peer : peers)
{
// Skipping the requestedc peer from the peer list response.
if (!skipping_ip_port.has_value() || peer.ip_port != skipping_ip_port.value())
fbvec.push_back(CreatePeer_Properties(
builder,
sv_to_flatbuff_str(builder, peer.ip_port.host_address),
peer.ip_port.port,
peer.available_capacity,
peer.timestamp));
}
return builder.CreateVector(fbvec);
}
/**
* Create vector of peer properties structs from the given peer list message.
* @param fbvec The peer list message to be convert to a list of peer properties structs.
*/
const std::vector<conf::peer_properties>
flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>> *fbvec)
{
std::vector<conf::peer_properties> peers;
for (const Peer_Properties *peer : *fbvec)
{
conf::peer_properties properties;
properties.ip_port.host_address = flatbuff_str_to_sv(peer->host_address());
properties.ip_port.port = peer->port();
properties.timestamp = peer->timestamp();
properties.available_capacity = peer->available_capacity();
peers.push_back(properties);
}
return peers;
}
} // namespace msg::fbuf::p2pmsg

106
src/msg/fbuf/p2pmsg_helpers.hpp
View File

@@ -1,106 +0,0 @@
#ifndef _HP_MSG_FBUF_P2PMSG_HELPERS_
#define _HP_MSG_FBUF_P2PMSG_HELPERS_
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "../../util/h32.hpp"
#include "../../hpfs/hpfs_mount.hpp"
#include "p2pmsg_container_generated.h"
#include "p2pmsg_content_generated.h"
namespace msg::fbuf::p2pmsg
{
/**
* This section contains Flatbuffer p2p message reading/writing helpers.
*/
//---Message validation helpers---/
int validate_and_extract_container(const Container **container_ref, std::string_view container_buf);
int validate_container_trust(const Container *container);
int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size);
//---Message reading helpers---/
const p2p::peer_challenge get_peer_challenge_from_msg(const Peer_Challenge_Message &msg);
const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey);
const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp);
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const Sequence_Hash &last_primary_shard_id_msg);
const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg);
const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const Peer_List_Response_Message &msg);
//---Message creation helpers---//
void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge);
void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge);
void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup);
void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p);
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
const p2p::sequence_hash &last_primary_shard_id);
void create_containermsg_from_content(
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, const p2p::sequence_hash &last_primary_shard_id, const bool sign);
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
const p2p::sequence_hash &last_primary_shard_id);
//---Conversion helpers from flatbuffers data types to std data types---//
const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec);
//---Conversion helpers from std data types to flatbuffers data types---//
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map);
const std::vector<conf::peer_properties>
flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>> *fbvec);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>>>
peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries,
const flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> *fhashes);
void hpfsfilehash_to_flatbuf_hpfsfilehash(flatbuffers::FlatBufferBuilder &builder, std::vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> &list,
std::string_view full_path, bool is_file, std::string_view hash);
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>>>
hpfsfshashentry_to_flatbuff_hpfsfshashentry(
flatbuffers::FlatBufferBuilder &builder,
std::vector<hpfs::child_hash_node> &hash_nodes);
} // namespace msg::fbuf::p2pmsg
#endif