mirror of
https://github.com/Xahau/xahaud.git
synced 2025-12-06 17:27:52 +00:00
STVar: optimized storage for STObject (RIPD-825):
This introduces the STVar container, capable of holding any STBase-derived class and implementing a "small string" optimization. STObject is changed to store std::vector<STVar> instead of boost::ptr_vector<STBase>. This eliminates a significant number of needless dynamic memory allocations and deallocations during transaction processing when ledger entries are deserialized. It comes at the expense of larger overall storage requirements for STObject.
This commit is contained in:
Vinnie Falco
@@ -31,222 +31,150 @@
|
||||
|
||||
namespace ripple {
|
||||
|
||||
std::unique_ptr<STBase>
|
||||
STObject::makeDefaultObject (SerializedTypeID id, SField::ref name)
|
||||
STObject::~STObject()
|
||||
{
|
||||
assert ((id == STI_NOTPRESENT) || (id == name.fieldType));
|
||||
|
||||
switch (id)
|
||||
{
|
||||
case STI_NOTPRESENT:
|
||||
return std::make_unique <STBase> (name);
|
||||
|
||||
case STI_UINT8:
|
||||
return std::make_unique <STUInt8> (name);
|
||||
|
||||
case STI_UINT16:
|
||||
return std::make_unique <STUInt16> (name);
|
||||
|
||||
case STI_UINT32:
|
||||
return std::make_unique <STUInt32> (name);
|
||||
|
||||
case STI_UINT64:
|
||||
return std::make_unique <STUInt64> (name);
|
||||
|
||||
case STI_AMOUNT:
|
||||
return std::make_unique <STAmount> (name);
|
||||
|
||||
case STI_HASH128:
|
||||
return std::make_unique <STHash128> (name);
|
||||
|
||||
case STI_HASH160:
|
||||
return std::make_unique <STHash160> (name);
|
||||
|
||||
case STI_HASH256:
|
||||
return std::make_unique <STHash256> (name);
|
||||
|
||||
case STI_VECTOR256:
|
||||
return std::make_unique <STVector256> (name);
|
||||
|
||||
case STI_VL:
|
||||
return std::make_unique <STBlob> (name);
|
||||
|
||||
case STI_ACCOUNT:
|
||||
return std::make_unique <STAccount> (name);
|
||||
|
||||
case STI_PATHSET:
|
||||
return std::make_unique <STPathSet> (name);
|
||||
|
||||
case STI_OBJECT:
|
||||
return std::make_unique <STObject> (name);
|
||||
|
||||
case STI_ARRAY:
|
||||
return std::make_unique <STArray> (name);
|
||||
|
||||
default:
|
||||
WriteLog (lsFATAL, STObject) <<
|
||||
"Object type: " << beast::lexicalCast <std::string> (id);
|
||||
assert (false);
|
||||
throw std::runtime_error ("Unknown object type");
|
||||
}
|
||||
#if 0
|
||||
// Turn this on to get a histogram on exit
|
||||
static beast::static_initializer<Log> log;
|
||||
(*log)(v_.size());
|
||||
#endif
|
||||
}
|
||||
|
||||
// VFALCO TODO Remove the 'depth' parameter
|
||||
std::unique_ptr<STBase>
|
||||
STObject::makeDeserializedObject (SerializedTypeID id, SField::ref name,
|
||||
SerialIter& sit, int depth)
|
||||
STObject::STObject(STObject&& other)
|
||||
: STBase(other.getFName())
|
||||
, v_(std::move(other.v_))
|
||||
, mType(other.mType)
|
||||
{
|
||||
switch (id)
|
||||
{
|
||||
case STI_NOTPRESENT:
|
||||
return STBase::deserialize (name);
|
||||
}
|
||||
|
||||
case STI_UINT8:
|
||||
return STUInt8::deserialize (sit, name);
|
||||
STObject::STObject (SField::ref name)
|
||||
: STBase (name)
|
||||
, mType (nullptr)
|
||||
{
|
||||
// VFALCO TODO See if this is the right thing to do
|
||||
//v_.reserve(reserveSize);
|
||||
}
|
||||
|
||||
case STI_UINT16:
|
||||
return STUInt16::deserialize (sit, name);
|
||||
STObject::STObject (SOTemplate const& type,
|
||||
SField::ref name)
|
||||
: STBase (name)
|
||||
{
|
||||
set (type);
|
||||
}
|
||||
|
||||
case STI_UINT32:
|
||||
return STUInt32::deserialize (sit, name);
|
||||
STObject::STObject (SOTemplate const& type,
|
||||
SerialIter & sit, SField::ref name)
|
||||
: STBase (name)
|
||||
{
|
||||
v_.reserve(type.peek().size());
|
||||
set (sit);
|
||||
setType (type);
|
||||
}
|
||||
|
||||
case STI_UINT64:
|
||||
return STUInt64::deserialize (sit, name);
|
||||
STObject::STObject (SField::ref name,
|
||||
boost::ptr_vector<STBase>& data)
|
||||
: STBase (name)
|
||||
, mType (nullptr)
|
||||
{
|
||||
v_.reserve(data.size());
|
||||
for (auto const& b : data)
|
||||
v_.emplace_back(b);
|
||||
}
|
||||
|
||||
case STI_AMOUNT:
|
||||
return STAmount::deserialize (sit, name);
|
||||
STObject::STObject (SerialIter& sit, SField::ref name)
|
||||
: STBase(name)
|
||||
, mType(nullptr)
|
||||
{
|
||||
set(sit, 0);
|
||||
}
|
||||
|
||||
case STI_HASH128:
|
||||
return STHash128::deserialize (sit, name);
|
||||
|
||||
case STI_HASH160:
|
||||
return STHash160::deserialize (sit, name);
|
||||
|
||||
case STI_HASH256:
|
||||
return STHash256::deserialize (sit, name);
|
||||
|
||||
case STI_VECTOR256:
|
||||
return STVector256::deserialize (sit, name);
|
||||
|
||||
case STI_VL:
|
||||
return STBlob::deserialize (sit, name);
|
||||
|
||||
case STI_ACCOUNT:
|
||||
return STAccount::deserialize (sit, name);
|
||||
|
||||
case STI_PATHSET:
|
||||
return STPathSet::deserialize (sit, name);
|
||||
|
||||
case STI_ARRAY:
|
||||
return STArray::deserialize (sit, name);
|
||||
|
||||
case STI_OBJECT:
|
||||
return STObject::deserialize (sit, name);
|
||||
|
||||
default:
|
||||
throw std::runtime_error ("Unknown object type");
|
||||
}
|
||||
STObject&
|
||||
STObject::operator= (STObject&& other)
|
||||
{
|
||||
setFName(other.getFName());
|
||||
mType = other.mType;
|
||||
v_ = std::move(other.v_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void STObject::set (const SOTemplate& type)
|
||||
{
|
||||
mData.clear ();
|
||||
v_.clear();
|
||||
v_.reserve(type.peek().size());
|
||||
mType = &type;
|
||||
|
||||
for (SOTemplate::value_type const& elem : type.peek ())
|
||||
for (auto const& elem : type.peek())
|
||||
{
|
||||
if (elem->flags != SOE_REQUIRED)
|
||||
giveObject (makeNonPresentObject (elem->e_field));
|
||||
v_.emplace_back(detail::nonPresentObject, elem->e_field);
|
||||
else
|
||||
giveObject (makeDefaultObject (elem->e_field));
|
||||
v_.emplace_back(detail::defaultObject, elem->e_field);
|
||||
}
|
||||
}
|
||||
|
||||
bool STObject::setType (const SOTemplate& type)
|
||||
{
|
||||
boost::ptr_vector<STBase> newData (type.peek ().size ());
|
||||
bool valid = true;
|
||||
|
||||
mType = &type;
|
||||
|
||||
STBase** array = mData.c_array();
|
||||
std::size_t count = mData.size ();
|
||||
|
||||
for (auto const& elem : type.peek ())
|
||||
decltype(v_) v;
|
||||
v.reserve(type.peek().size());
|
||||
for (auto const& e : type.peek())
|
||||
{
|
||||
// Loop through all the fields in the template
|
||||
bool match = false;
|
||||
|
||||
for (std::size_t i = 0; i < count; ++i)
|
||||
if ((array[i] != nullptr) &&
|
||||
(array[i]->getFName () == elem->e_field))
|
||||
{
|
||||
// matching entry in the object, move to new vector
|
||||
match = true;
|
||||
|
||||
if ((elem->flags == SOE_DEFAULT) && array[i]->isDefault ())
|
||||
{
|
||||
WriteLog (lsWARNING, STObject) <<
|
||||
"setType( " << getFName ().getName () <<
|
||||
") invalid default " << elem->e_field.fieldName;
|
||||
valid = false;
|
||||
}
|
||||
|
||||
newData.push_back (array[i]);
|
||||
array[i] = nullptr;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!match)
|
||||
auto const iter = std::find_if(
|
||||
v_.begin(), v_.end(), [&](detail::STVar const& b)
|
||||
{ return b.get().getFName() == e->e_field; });
|
||||
if (iter != v_.end())
|
||||
{
|
||||
// no match found in the object for an entry in the template
|
||||
if (elem->flags == SOE_REQUIRED)
|
||||
if ((e->flags == SOE_DEFAULT) && iter->get().isDefault())
|
||||
{
|
||||
WriteLog (lsWARNING, STObject) <<
|
||||
"setType( " << getFName ().getName () <<
|
||||
") invalid missing " << elem->e_field.fieldName;
|
||||
") invalid default " << e->e_field.fieldName;
|
||||
valid = false;
|
||||
}
|
||||
|
||||
// Make a default object
|
||||
newData.push_back (makeNonPresentObject (elem->e_field).release ());
|
||||
v.emplace_back(std::move(*iter));
|
||||
v_.erase(iter);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (e->flags == SOE_REQUIRED)
|
||||
{
|
||||
WriteLog (lsWARNING, STObject) <<
|
||||
"setType( " << getFName ().getName () <<
|
||||
") invalid missing " << e->e_field.fieldName;
|
||||
valid = false;
|
||||
}
|
||||
v.emplace_back(detail::nonPresentObject, e->e_field);
|
||||
}
|
||||
}
|
||||
|
||||
for (std::size_t i = 0; i < count; ++i)
|
||||
for (auto const& e : v_)
|
||||
{
|
||||
// Anything left over in the object must be discardable
|
||||
if ((array[i] != nullptr) && !array[i]->getFName ().isDiscardable ())
|
||||
if (! e->getFName().isDiscardable())
|
||||
{
|
||||
WriteLog (lsWARNING, STObject) <<
|
||||
"setType( " << getFName ().getName () <<
|
||||
") invalid leftover " << array[i]->getFName ().getName ();
|
||||
") invalid leftover " << e->getFName ().getName ();
|
||||
valid = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Swap the template matching data in for the old data,
|
||||
// freeing any leftover junk
|
||||
mData.swap (newData);
|
||||
|
||||
v_.swap(v);
|
||||
return valid;
|
||||
}
|
||||
|
||||
bool STObject::isValidForType ()
|
||||
{
|
||||
boost::ptr_vector<STBase>::iterator it = mData.begin ();
|
||||
|
||||
for (SOTemplate::value_type const& elem : mType->peek ())
|
||||
auto it = v_.begin();
|
||||
for (SOTemplate::value_type const& elem : mType->peek())
|
||||
{
|
||||
if (it == mData.end ())
|
||||
if (it == v_.end())
|
||||
return false;
|
||||
|
||||
if (elem->e_field != it->getFName ())
|
||||
if (elem->e_field != it->get().getFName())
|
||||
return false;
|
||||
|
||||
++it;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -263,9 +191,7 @@ bool STObject::set (SerialIter& sit, int depth)
|
||||
{
|
||||
bool reachedEndOfObject = false;
|
||||
|
||||
// Empty the destination buffer
|
||||
//
|
||||
mData.clear ();
|
||||
v_.clear();
|
||||
|
||||
// Consume data in the pipe until we run out or reach the end
|
||||
//
|
||||
@@ -302,24 +228,13 @@ bool STObject::set (SerialIter& sit, int depth)
|
||||
}
|
||||
|
||||
// Unflatten the field
|
||||
//
|
||||
giveObject (
|
||||
makeDeserializedObject (fn.fieldType, fn, sit, depth + 1));
|
||||
v_.emplace_back(sit, fn);
|
||||
}
|
||||
}
|
||||
|
||||
return reachedEndOfObject;
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<STBase>
|
||||
STObject::deserialize (SerialIter& sit, SField::ref name)
|
||||
{
|
||||
std::unique_ptr <STObject> object (std::make_unique <STObject> (name));
|
||||
object->set (sit, 1);
|
||||
return std::move (object);
|
||||
}
|
||||
|
||||
bool STObject::hasMatchingEntry (const STBase& t)
|
||||
{
|
||||
const STBase* o = peekAtPField (t.getFName ());
|
||||
@@ -342,16 +257,16 @@ std::string STObject::getFullText () const
|
||||
}
|
||||
else ret = "{";
|
||||
|
||||
for (STBase const& elem : mData)
|
||||
for (auto const& elem : v_)
|
||||
{
|
||||
if (elem.getSType () != STI_NOTPRESENT)
|
||||
if (elem->getSType () != STI_NOTPRESENT)
|
||||
{
|
||||
if (!first)
|
||||
ret += ", ";
|
||||
else
|
||||
first = false;
|
||||
|
||||
ret += elem.getFullText ();
|
||||
ret += elem->getFullText ();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,32 +276,30 @@ std::string STObject::getFullText () const
|
||||
|
||||
void STObject::add (Serializer& s, bool withSigningFields) const
|
||||
{
|
||||
std::map<int, const STBase*> fields;
|
||||
|
||||
for (STBase const& elem : mData)
|
||||
std::map<int, STBase const*> fields;
|
||||
for (auto const& e : v_)
|
||||
{
|
||||
// pick out the fields and sort them
|
||||
if ((elem.getSType () != STI_NOTPRESENT) &&
|
||||
elem.getFName ().shouldInclude (withSigningFields))
|
||||
if ((e->getSType() != STI_NOTPRESENT) &&
|
||||
e->getFName().shouldInclude (withSigningFields))
|
||||
{
|
||||
fields.insert (std::make_pair (elem.getFName ().fieldCode, &elem));
|
||||
fields.insert (std::make_pair (
|
||||
e->getFName().fieldCode, &e.get()));
|
||||
}
|
||||
}
|
||||
|
||||
for (auto const& mapEntry : fields)
|
||||
// insert sorted
|
||||
for (auto const& e : fields)
|
||||
{
|
||||
// insert them in sorted order
|
||||
const STBase* field = mapEntry.second;
|
||||
auto const field = e.second;
|
||||
|
||||
// When we serialize an object inside another object,
|
||||
// the type associated by rule with this field name
|
||||
// must be OBJECT, or the object cannot be deserialized
|
||||
assert ((field->getSType() != STI_OBJECT) ||
|
||||
(field->getFName().fieldType == STI_OBJECT));
|
||||
|
||||
field->addFieldID (s);
|
||||
field->add (s);
|
||||
|
||||
if (dynamic_cast<const STArray*> (field) != nullptr)
|
||||
s.addFieldID (STI_ARRAY, 1);
|
||||
else if (dynamic_cast<const STObject*> (field) != nullptr)
|
||||
@@ -398,15 +311,15 @@ std::string STObject::getText () const
|
||||
{
|
||||
std::string ret = "{";
|
||||
bool first = false;
|
||||
for (STBase const& elem : mData)
|
||||
for (auto const& elem : v_)
|
||||
{
|
||||
if (!first)
|
||||
if (! first)
|
||||
{
|
||||
ret += ", ";
|
||||
first = false;
|
||||
}
|
||||
|
||||
ret += elem.getText ();
|
||||
ret += elem->getText ();
|
||||
}
|
||||
ret += "}";
|
||||
return ret;
|
||||
@@ -423,23 +336,23 @@ bool STObject::isEquivalent (const STBase& t) const
|
||||
return false;
|
||||
}
|
||||
|
||||
typedef boost::ptr_vector<STBase>::const_iterator const_iter;
|
||||
const_iter it1 = mData.begin (), end1 = mData.end ();
|
||||
const_iter it2 = v->mData.begin (), end2 = v->mData.end ();
|
||||
auto it1 = v_.begin (), end1 = v_.end ();
|
||||
auto it2 = v->v_.begin (), end2 = v->v_.end ();
|
||||
|
||||
while ((it1 != end1) && (it2 != end2))
|
||||
{
|
||||
if ((it1->getSType () != it2->getSType ()) || !it1->isEquivalent (*it2))
|
||||
if ((it1->get().getSType () != it2->get().getSType ()) ||
|
||||
!it1->get().isEquivalent (it2->get()))
|
||||
{
|
||||
if (it1->getSType () != it2->getSType ())
|
||||
if (it1->get().getSType () != it2->get().getSType ())
|
||||
{
|
||||
WriteLog (lsDEBUG, STObject) << "notEquiv type " <<
|
||||
it1->getFullText() << " != " << it2->getFullText();
|
||||
it1->get().getFullText() << " != " << it2->get().getFullText();
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteLog (lsDEBUG, STObject) << "notEquiv " <<
|
||||
it1->getFullText() << " != " << it2->getFullText();
|
||||
it1->get().getFullText() << " != " << it2->get().getFullText();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -473,11 +386,10 @@ int STObject::getFieldIndex (SField::ref field) const
|
||||
return mType->getIndex (field);
|
||||
|
||||
int i = 0;
|
||||
for (STBase const& elem : mData)
|
||||
for (auto const& elem : v_)
|
||||
{
|
||||
if (elem.getFName () == field)
|
||||
if (elem->getFName () == field)
|
||||
return i;
|
||||
|
||||
++i;
|
||||
}
|
||||
return -1;
|
||||
@@ -505,7 +417,7 @@ STBase& STObject::getField (SField::ref field)
|
||||
|
||||
SField::ref STObject::getFieldSType (int index) const
|
||||
{
|
||||
return mData[index].getFName ();
|
||||
return v_[index]->getFName ();
|
||||
}
|
||||
|
||||
const STBase* STObject::peekAtPField (SField::ref field) const
|
||||
@@ -525,7 +437,7 @@ STBase* STObject::getPField (SField::ref field, bool createOkay)
|
||||
if (index == -1)
|
||||
{
|
||||
if (createOkay && isFree ())
|
||||
return getPIndex (giveObject (makeDefaultObject (field)));
|
||||
return getPIndex(emplace_back(detail::defaultObject, field));
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
@@ -607,7 +519,7 @@ STBase* STObject::makeFieldPresent (SField::ref field)
|
||||
if (!isFree ())
|
||||
throw std::runtime_error ("Field not found");
|
||||
|
||||
return getPIndex (giveObject (makeNonPresentObject (field)));
|
||||
return getPIndex (emplace_back(detail::nonPresentObject, field));
|
||||
}
|
||||
|
||||
STBase* f = getPIndex (index);
|
||||
@@ -615,7 +527,8 @@ STBase* STObject::makeFieldPresent (SField::ref field)
|
||||
if (f->getSType () != STI_NOTPRESENT)
|
||||
return f;
|
||||
|
||||
mData.replace (index, makeDefaultObject (f->getFName ()).release ());
|
||||
v_[index] = detail::STVar(
|
||||
detail::defaultObject, f->getFName());
|
||||
return getPIndex (index);
|
||||
}
|
||||
|
||||
@@ -630,8 +543,8 @@ void STObject::makeFieldAbsent (SField::ref field)
|
||||
|
||||
if (f.getSType () == STI_NOTPRESENT)
|
||||
return;
|
||||
|
||||
mData.replace (index, makeNonPresentObject (f.getFName ()).release ());
|
||||
v_[index] = detail::STVar(
|
||||
detail::nonPresentObject, f.getFName());
|
||||
}
|
||||
|
||||
bool STObject::delField (SField::ref field)
|
||||
@@ -647,7 +560,7 @@ bool STObject::delField (SField::ref field)
|
||||
|
||||
void STObject::delField (int index)
|
||||
{
|
||||
mData.erase (mData.begin () + index);
|
||||
v_.erase (v_.begin () + index);
|
||||
}
|
||||
|
||||
std::string STObject::getFieldString (SField::ref field) const
|
||||
@@ -771,14 +684,14 @@ STObject::set (std::unique_ptr<STBase> v)
|
||||
getFieldIndex(v->getFName());
|
||||
if (i != -1)
|
||||
{
|
||||
mData.replace(i, v.release());
|
||||
v_[i] = std::move(*v);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (! isFree())
|
||||
throw std::runtime_error(
|
||||
"missing field in templated STObject");
|
||||
mData.push_back(v.release());
|
||||
v_.emplace_back(std::move(*v));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -862,14 +775,14 @@ Json::Value STObject::getJson (int options) const
|
||||
|
||||
// TODO(tom): this variable is never changed...?
|
||||
int index = 1;
|
||||
for (auto const& it: mData)
|
||||
for (auto const& elem : v_)
|
||||
{
|
||||
if (it.getSType () != STI_NOTPRESENT)
|
||||
if (elem->getSType () != STI_NOTPRESENT)
|
||||
{
|
||||
auto const& n = it.getFName ();
|
||||
auto const& n = elem->getFName ();
|
||||
auto key = n.hasName () ? std::string(n.getJsonName ()) :
|
||||
std::to_string (index);
|
||||
ret[key] = it.getJson (options);
|
||||
ret[key] = elem->getJson (options);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@@ -880,15 +793,15 @@ bool STObject::operator== (const STObject& obj) const
|
||||
// This is not particularly efficient, and only compares data elements
|
||||
// with binary representations
|
||||
int matches = 0;
|
||||
for (STBase const& t1 : mData)
|
||||
for (auto const& t1 : v_)
|
||||
{
|
||||
if ((t1.getSType () != STI_NOTPRESENT) && t1.getFName ().isBinary ())
|
||||
if ((t1->getSType () != STI_NOTPRESENT) && t1->getFName ().isBinary ())
|
||||
{
|
||||
// each present field must have a matching field
|
||||
bool match = false;
|
||||
for (STBase const& t2 : obj.mData)
|
||||
for (auto const& t2 : obj.v_)
|
||||
{
|
||||
if (t1.getFName () == t2.getFName ())
|
||||
if (t1->getFName () == t2->getFName ())
|
||||
{
|
||||
if (t2 != t1)
|
||||
return false;
|
||||
@@ -903,16 +816,16 @@ bool STObject::operator== (const STObject& obj) const
|
||||
{
|
||||
WriteLog (lsTRACE, STObject) <<
|
||||
"STObject::operator==: no match for " <<
|
||||
t1.getFName ().getName ();
|
||||
t1->getFName ().getName ();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int fields = 0;
|
||||
for (STBase const& t2 : obj.mData)
|
||||
for (auto const& t2 : obj.v_)
|
||||
{
|
||||
if ((t2.getSType () != STI_NOTPRESENT) && t2.getFName ().isBinary ())
|
||||
if ((t2->getSType () != STI_NOTPRESENT) && t2->getFName ().isBinary ())
|
||||
++fields;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user