From dbd75169e57a69780e80a5f580da8f6c941e4484 Mon Sep 17 00:00:00 2001 From: Tom Ritchford Date: Wed, 1 Oct 2014 18:16:36 -0400 Subject: [PATCH] New JsonWriter for improved client performance (RIPD-439): When JSON-RPC and Websocket responses are calculated, the result is stored in intermediate Json::Value objects and later composed in a single linear memory buffer before being sent to the socket. These classes support a new model for building responses that supports incremental construction of JSON replies in constant time and removes the requirement that all data returned be located in continuguous memory. * New JsonWriter incrementally writes JSON with O(1) granularity and memory. * Array, Object are RAII wrappers for the O(1) JsonWriter. --- Builds/VisualStudio2013/RippleD.vcxproj | 20 ++ .../VisualStudio2013/RippleD.vcxproj.filters | 24 ++ src/ripple/rpc/Output.h | 34 ++ src/ripple/rpc/impl/JsonObject.cpp | 147 +++++++++ src/ripple/rpc/impl/JsonObject.h | 287 +++++++++++++++++ src/ripple/rpc/impl/JsonObject_test.cpp | 243 ++++++++++++++ src/ripple/rpc/impl/JsonWriter.cpp | 303 ++++++++++++++++++ src/ripple/rpc/impl/JsonWriter.h | 215 +++++++++++++ src/ripple/rpc/impl/JsonWriter_test.cpp | 182 +++++++++++ src/ripple/rpc/impl/TestOutputSuite.h | 67 ++++ src/ripple/unity/rpcx.cpp | 5 + 11 files changed, 1527 insertions(+) create mode 100644 src/ripple/rpc/Output.h create mode 100644 src/ripple/rpc/impl/JsonObject.cpp create mode 100644 src/ripple/rpc/impl/JsonObject.h create mode 100644 src/ripple/rpc/impl/JsonObject_test.cpp create mode 100644 src/ripple/rpc/impl/JsonWriter.cpp create mode 100644 src/ripple/rpc/impl/JsonWriter.h create mode 100644 src/ripple/rpc/impl/JsonWriter_test.cpp create mode 100644 src/ripple/rpc/impl/TestOutputSuite.h diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index 4e9d0bc01a..d2e57e9712 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -3166,6 +3166,22 @@ + + True + + + + + True + + + True + + + + + True + True @@ -3196,6 +3212,8 @@ True + + True @@ -3207,6 +3225,8 @@ + + diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 6870950610..3d318f39f9 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -4350,6 +4350,24 @@ ripple\rpc\impl + + ripple\rpc\impl + + + ripple\rpc\impl + + + ripple\rpc\impl + + + ripple\rpc\impl + + + ripple\rpc\impl + + + ripple\rpc\impl + ripple\rpc\impl @@ -4383,6 +4401,9 @@ ripple\rpc\impl + + ripple\rpc\impl + ripple\rpc\impl @@ -4398,6 +4419,9 @@ ripple\rpc + + ripple\rpc + ripple\rpc diff --git a/src/ripple/rpc/Output.h b/src/ripple/rpc/Output.h new file mode 100644 index 0000000000..e39dd108b5 --- /dev/null +++ b/src/ripple/rpc/Output.h @@ -0,0 +1,34 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_RIPPLE_BASICS_TYPES_OUTPUT_H +#define RIPPLED_RIPPLE_BASICS_TYPES_OUTPUT_H + +namespace ripple { + +class Output +{ +public: + virtual void output (char const* data, size_t length) = 0; + virtual ~Output() = default; +}; + +} // ripple + +#endif diff --git a/src/ripple/rpc/impl/JsonObject.cpp b/src/ripple/rpc/impl/JsonObject.cpp new file mode 100644 index 0000000000..44ca9bf220 --- /dev/null +++ b/src/ripple/rpc/impl/JsonObject.cpp @@ -0,0 +1,147 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +Collection::Collection (Collection* parent, Writer* writer) + : parent_ (parent), writer_ (writer), enabled_ (true) +{ + checkWritable ("Collection::Collection()"); + if (parent_) + { + check (parent_->enabled_, "Parent not enabled in constructor"); + parent_->enabled_ = false; + } +} + +Collection::~Collection () +{ + if (writer_) + writer_->finish (); + if (parent_) + parent_->enabled_ = true; +} + +Collection& Collection::operator= (Collection&& that) +{ + parent_ = that.parent_; + writer_ = that.writer_; + enabled_ = that.enabled_; + + that.parent_ = nullptr; + that.writer_ = nullptr; + that.enabled_ = false; + + return *this; +} + +Collection::Collection (Collection&& that) +{ + *this = std::move (that); +} + +void Collection::checkWritable (std::string const& label) +{ + if (!enabled_) + throw JsonException (label + ": not enabled"); + if (!writer_) + throw JsonException (label + ": not writable"); +} + +//------------------------------------------------------------------------------ + +template +Array& Array::append (Scalar value) +{ + checkWritable ("append"); + if (writer_) + writer_->append (value); + return *this; +} + +//------------------------------------------------------------------------------ + +Object::Root::Root (Writer& w) : Object (nullptr, &w) +{ + writer_->startRoot (Writer::object); // writer_ can't be null. +} + +//------------------------------------------------------------------------------ + +template +Object& Object::set (std::string const& key, Scalar value) +{ + checkWritable ("set"); + if (writer_) + writer_->set (key, value); + return *this; +} + +Object Object::makeObject (std::string const& key) +{ + checkWritable ("Object::makeObject"); + if (writer_) + writer_->startSet (Writer::object, key); + return Object (this, writer_); +} + +Array Object::makeArray (std::string const& key) { + checkWritable ("Object::makeArray"); + if (writer_) + writer_->startSet (Writer::array, key); + return Array (this, writer_); +} + +Object Array::makeObject () +{ + checkWritable ("Array::makeObject"); + if (writer_) + writer_->startAppend (Writer::object); + return Object (this, writer_); +} + +Array Array::makeArray () +{ + checkWritable ("Array::makeArray"); + if (writer_) + writer_->startAppend (Writer::array); + return Array (this, writer_); +} + +//------------------------------------------------------------------------------ + +Object::Proxy::Proxy (Object& object, std::string const& key) + : object_ (object) + , key_ (key) +{ +} + +Object::Proxy Object::operator[] (std::string const& key) +{ + return {*this, key}; +} + +} // New +} // RPC +} // ripple diff --git a/src/ripple/rpc/impl/JsonObject.h b/src/ripple/rpc/impl/JsonObject.h new file mode 100644 index 0000000000..380c999733 --- /dev/null +++ b/src/ripple/rpc/impl/JsonObject.h @@ -0,0 +1,287 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_RIPPLE_RPC_IMPL_JSONCOLLECTIONS_H +#define RIPPLED_RIPPLE_RPC_IMPL_JSONCOLLECTIONS_H + +namespace ripple { +namespace RPC { +namespace New { + +class Writer; + +/** + Collection is a base class for Array and Object, classes which provide the + facade of JSON collections for the O(1) JSON writer, while still using no + heap memory and only a very small amount of stack. + + From http://json.org, JSON has two types of collection: array, and object. + Everything else is a *scalar* - a number, a string, a boolean or the special + value null. + + Collections must write JSON "as-it-goes" in order to get the strong + performance guarantees. This puts restrictions upon API users: + + 1. Only one collection can be open for change at any one time. + + This condition is enforced automatically and a JsonException thrown if it + is violated. + + 2. A tag may only be used once in an Object. + + Some objects have many tags, so this condition might be a little + expensive. Enforcement of this condition is turned on in debug builds and + a JsonException is thrown when the tag is added for a second time. + + Code samples: + + Writer writer; + + // An empty object. + { + Object::Root (writer); + } + // Outputs {} + + // An object with one scalar value. + { + Object::Root root (writer); + write["hello"] = "world"; + } + // Outputs {"hello":"world"} + + // Same, using chaining. + { + Object::Root (writer)["hello"] = "world"; + } + // Output is the same. + + // Add several scalars, with chaining. + { + Object::Root (writer) + .set ("hello", "world") + .set ("flag", false) + .set ("x", 42); + } + // Outputs {"hello":"world","flag":false,"x":42} + + // Add an array. + { + Object::Root root (writer); + { + auto array = root.makeArray ("hands"); + array.append ("left"); + array.append ("right"); + } + } + // Outputs {"hands":["left", "right"]} + + // Same, using chaining. + { + Object::Root (writer) + .makeArray ("hands") + .append ("left") + .append ("right"); + } + // Output is the same. + + // Add an object. + { + Object::Root root (writer); + { + auto object = root.makeObject ("hands"); + object["left"] = false; + object["right"] = true; + } + } + // Outputs {"hands":{"left":false,"right":true}} + + // Same, using chaining. + { + Object::Root (writer) + .makeObject ("hands") + .set ("left", false) + .set ("right", true); + } + } + // Outputs {"hands":{"left":false,"right":true}} + + + Typical ways to make mistakes and get a JsonException: + + Writer writer; + Object::Root root (writer); + + // Repeat a tag. + { + root ["hello"] = "world"; + root ["hello"] = "there"; // THROWS! in a debug build. + } + + // Open a subcollection, then set something else. + { + auto object = root.makeObject ("foo"); + root ["hello"] = "world"; // THROWS! + } + + // Open two subcollections at a time. + { + auto object = root.makeObject ("foo"); + auto array = root.makeArray ("bar"); // THROWS!! + } + + For more examples, check the unit tests. + */ + +class Collection +{ +public: + Collection (Collection&& c); + Collection& operator= (Collection&& c); + Collection() = delete; + + ~Collection(); + +protected: + // A null parent means "no parent at all". + // Writers cannot be null. + Collection (Collection* parent, Writer*); + void checkWritable (std::string const& label); + + Collection* parent_; + Writer* writer_; + bool enabled_; +}; + +class Array; + +//------------------------------------------------------------------------------ + +/** Represents a JSON object being written to a Writer. */ +class Object : protected Collection +{ +public: + /** Object::Root is the only Collection that has a public constructor. */ + class Root; + + /** Set a scalar value in the Object for a key. + + A JSON scalar is a single value - a number, string, boolean or null. + + `set()` throws an exception if this object is disabled (which means that + one of its children is enabled). + + In a debug build, `set()` also throws an exception if the key has + already been set() before. + + An operator[] is provided to allow writing `object["key"] = scalar;`. + */ + template + Object& set (std::string const& key, Scalar); + + // Detail class and method used to implement operator[]. + class Proxy; + Proxy operator[] (std::string const& key); + + /** Make a new Object at a key and return it. + + This Object is disabled until that sub-object is destroyed. + Throws an exception if this Object was already disabled. + */ + Object makeObject (std::string const& key); + + /** Make a new Array at a key and return it. + + This Object is disabled until that sub-array is destroyed. + Throws an exception if this Object was already disabled. + */ + Array makeArray (std::string const& key); + +protected: + friend class Array; + Object (Collection* parent, Writer* w) : Collection (parent, w) {} +}; + +//------------------------------------------------------------------------------ + +class Object::Root : public Object +{ + public: + /** Each Object::Root must be constructed with its own unique Writer. */ + Root (Writer&); +}; + +//------------------------------------------------------------------------------ + +/** Represents a JSON array being written to a Writer. */ +class Array : private Collection +{ +public: + /** Append a scalar to the Arrary. + + Throws an exception if this array is disabled (which means that one of + its sub-collections is enabled). + */ + template + Array& append (Scalar); + + /** Append a new Object and return it. + + This Array is disabled until that sub-object is destroyed. + Throws an exception if this Array was already disabled. + */ + Object makeObject (); + + /** Append a new Array and return it. + + This Array is disabled until that sub-array is destroyed. + Throws an exception if this Array was already disabled. + */ + Array makeArray (); + + protected: + friend class Object; + Array (Collection* parent, Writer* w) : Collection (parent, w) {} +}; + +//------------------------------------------------------------------------------ + +// Detail class for Object::operator[]. +class Object::Proxy +{ +private: + Object& object_; + std::string const& key_; + +public: + Proxy (Object& object, std::string const& key); + + template + Object& operator= (T const& t) + { + object_.set (key_, t); + return object_; + } +}; + +} // New +} // RPC +} // ripple + +#endif diff --git a/src/ripple/rpc/impl/JsonObject_test.cpp b/src/ripple/rpc/impl/JsonObject_test.cpp new file mode 100644 index 0000000000..d915d1a566 --- /dev/null +++ b/src/ripple/rpc/impl/JsonObject_test.cpp @@ -0,0 +1,243 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +class JsonObject_test : public TestOutputSuite +{ +public: + void testTrivial () + { + setup ("trivial"); + + { + Object::Root root (*writer_); + (void) root; + } + expectResult ("{}"); + } + + void testSimple () + { + setup ("simple"); + { + Object::Root root (*writer_); + root["hello"] = "world"; + root["skidoo"] = 23; + root["awake"] = false; + root["temperature"] = 98.6; + } + + expectResult ( + "{\"hello\":\"world\"," + "\"skidoo\":23," + "\"awake\":false," + "\"temperature\":98.6}"); + } + + void testSimpleShort () + { + setup ("simpleShort"); + Object::Root (*writer_) + .set ("hello", "world") + .set ("skidoo", 23) + .set ("awake", false) + .set ("temperature", 98.6); + + expectResult ( + "{\"hello\":\"world\"," + "\"skidoo\":23," + "\"awake\":false," + "\"temperature\":98.6}"); + } + + void testOneSub () + { + setup ("oneSub"); + { + Object::Root root (*writer_); + root.makeArray ("ar"); + } + expectResult ("{\"ar\":[]}"); + } + + void testSubs () + { + setup ("subs"); + { + Object::Root root (*writer_); + + { + // Add an array with three entries. + auto array = root.makeArray ("ar"); + array.append (23); + array.append (false); + array.append (23.5); + } + + { + // Add an object with one entry. + auto obj = root.makeObject ("obj"); + obj["hello"] = "world"; + } + + { + // Add another object with two entries. + auto obj = root.makeObject ("obj2"); + obj["h"] = "w"; + obj["f"] = false; + } + } + + expectResult ( + "{\"ar\":[23,false,23.5]," + "\"obj\":{\"hello\":\"world\"}," + "\"obj2\":{\"h\":\"w\",\"f\":false}}"); + } + + void testSubsShort () + { + setup ("subsShort"); + + { + Object::Root root (*writer_); + + // Add an array with three entries. + root.makeArray ("ar") + .append (23) + .append (false) + .append (23.5); + + // Add an object with one entry. + root.makeObject ("obj")["hello"] = "world"; + + // Add another object with two entries. + root.makeObject ("obj2") + .set("h", "w") + .set("f", false); + } + + expectResult ( + "{\"ar\":[23,false,23.5]," + "\"obj\":{\"hello\":\"world\"}," + "\"obj2\":{\"h\":\"w\",\"f\":false}}"); + } + + template + void expectException (Functor f) + { + bool success = true; + try + { + f(); + success = false; + } catch (std::exception) + { + } + expect (success, "no exception thrown"); + } + + void testFailureObject() + { + { + setup ("object failure assign"); + Object::Root root (*writer_); + auto obj = root.makeObject ("o1"); + expectException ([&]() { root["fail"] = "complete"; }); + } + { + setup ("object failure object"); + Object::Root root (*writer_); + auto obj = root.makeObject ("o1"); + expectException ([&] () { root.makeObject ("o2"); }); + } + { + setup ("object failure Array"); + Object::Root root (*writer_); + auto obj = root.makeArray ("o1"); + expectException ([&] () { root.makeArray ("o2"); }); + } + } + + void testFailureArray() + { + { + setup ("array failure append"); + Object::Root root (*writer_); + auto array = root.makeArray ("array"); + auto subarray = array.makeArray (); + auto fail = [&]() { array.append ("fail"); }; + expectException (fail); + } + { + setup ("array failure makeArray"); + Object::Root root (*writer_); + auto array = root.makeArray ("array"); + auto subarray = array.makeArray (); + auto fail = [&]() { array.makeArray (); }; + expectException (fail); + } + { + setup ("array failure makeObject"); + Object::Root root (*writer_); + auto array = root.makeArray ("array"); + auto subarray = array.makeArray (); + auto fail = [&]() { array.makeObject (); }; + expectException (fail); + } + } + + void testKeyFailure () + { +#ifdef DEBUG + setup ("repeating keys"); + Object::Root root(*writer_); + root.set ("foo", "bar") + .set ("baz", 0); + auto fail = [&]() { root.set ("foo", "bar"); }; + expectException (fail); +#endif + } + + void run () override + { + testSimple (); + testSimpleShort (); + + testOneSub (); + testSubs (); + testSubsShort (); + + testFailureObject (); + testFailureArray (); + testKeyFailure (); + } +}; + +BEAST_DEFINE_TESTSUITE(JsonObject, ripple_basics, ripple); + +} // New +} // RPC +} // ripple diff --git a/src/ripple/rpc/impl/JsonWriter.cpp b/src/ripple/rpc/impl/JsonWriter.cpp new file mode 100644 index 0000000000..7b0fa7731e --- /dev/null +++ b/src/ripple/rpc/impl/JsonWriter.cpp @@ -0,0 +1,303 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +namespace { + +std::map jsonSpecialCharacterEscape = { + {'"', "\\\""}, + {'\\', "\\\\"}, + {'/', "\\/"}, + {'\b', "\\b"}, + {'\f', "\\f"}, + {'\n', "\\n"}, + {'\r', "\\r"}, + {'\t', "\\t"} +}; + +static int const jsonEscapeLength = 2; + +// All other JSON punctuation. +const char closeBrace = '}'; +const char closeBracket = ']'; +const char colon = ':'; +const char comma = ','; +const char openBrace = '{'; +const char openBracket = '['; +const char quote = '"'; + +const std::string none; + +size_t lengthWithoutTrailingZeros (std::string const& s) +{ + if (s.find ('.') == std::string::npos) + return s.size(); + + return s.find_last_not_of ('0') + 1; +} + +} // namespace + +class Writer::Impl +{ +public: + Impl (Output& output) : output_(output) {} + + Impl(Impl&&) = delete; + Impl& operator=(Impl&&) = delete; + + + bool empty() const { return stack_.empty (); } + + void start (CollectionType ct) + { + char ch = (ct == array) ? openBracket : openBrace; + output (&ch, 1); + stack_.push (Collection()); + stack_.top().type = ct; + } + + void output (char const* data, size_t size) + { + markStarted (); + output_.output (data, size); + } + + void stringOutput (char const* data, size_t size) + { + markStarted (); + size_t position = 0, writtenUntil = 0; + + output_.output ("e, 1); + for (; position < size; ++position) + { + auto i = jsonSpecialCharacterEscape.find (data[position]); + if (i != jsonSpecialCharacterEscape.end ()) + { + if (writtenUntil < position) + { + output_.output ( + data + writtenUntil, position - writtenUntil); + } + output_.output (i->second, jsonEscapeLength); + writtenUntil = position + 1; + }; + } + if (writtenUntil < position) + output_.output (data + writtenUntil, position - writtenUntil); + output_.output ("e, 1); + } + + void markStarted () + { + check (!isFinished(), "isFinished() in output."); + isStarted_ = true; + } + + void nextCollectionEntry (CollectionType type, std::string const& message) + { + check (!empty() , "empty () in " + message); + + auto t = stack_.top ().type; + if (t != type) + { + check (false, "Not an " + + ((type == array ? "array: " : "object: ") + message)); + } + if (stack_.top ().isFirst) + stack_.top ().isFirst = false; + else + output (&comma, 1); + } + + void writeObjectTag (std::string const& tag) + { +#ifdef DEBUG + // Make sure we haven't already seen this tag. + auto& tags = stack_.top ().tags; + check (tags.find (tag) == tags.end (), "Already seen tag " + tag); + tags.insert (tag); +#endif + + stringOutput (tag.data(), tag.size()); + output (&colon, 1); + } + + bool isFinished() const + { + return isStarted_ && empty(); + } + + void finish () + { + check (!empty(), "Empty stack in finish()"); + + auto isArray = stack_.top().type == array; + auto ch = isArray ? closeBracket : closeBrace; + output (&ch, 1); + stack_.pop(); + } + + void finishAll () + { + if (isStarted_) + { + while (!isFinished()) + finish(); + } + } + +private: + // JSON collections are either arrrays, or objects. + struct Collection + { + /** What type of collection are we in? */ + Writer::CollectionType type; + + /** Is this the first entry in a collection? + * If false, we have to emit a , before we write the next entry. */ + bool isFirst = true; + +#ifdef DEBUG + /** What tags have we already seen in this collection? */ + std::set tags; +#endif + }; + + using Stack = std::stack >; + + Output& output_; + Stack stack_; + + bool isStarted_ = false; +}; + +Writer::Writer (Output& output) : impl_(std::make_unique (output)) +{ +} + +Writer::~Writer() +{ + impl_->finishAll (); +} + +Writer::Writer(Writer&& w) +{ + impl_ = std::move (w.impl_); +} + +Writer& Writer::operator=(Writer&& w) +{ + impl_ = std::move (w.impl_); + return *this; +} + +void Writer::output (char const* s) +{ + impl_->stringOutput (s, strlen (s)); +} + +void Writer::output (std::string const& s) +{ + impl_->stringOutput (s.data(), s.size ()); +} + +template <> +void Writer::output (float f) +{ + auto s = to_string (f); + impl_->output (s.data (), lengthWithoutTrailingZeros (s)); +} + +template <> +void Writer::output (double f) +{ + auto s = to_string (f); + impl_->output (s.data (), lengthWithoutTrailingZeros (s)); +} + +template <> +void Writer::output (std::nullptr_t) +{ + impl_->output ("null", strlen("null")); +} + +template +void Writer::output (Type t) +{ + auto s = to_string (t); + impl_->output (s.data(), s.size()); +} + +void Writer::finishAll () +{ + impl_->finishAll (); +} + +template +void Writer::append (Type t) +{ + impl_->nextCollectionEntry (array, "append"); + output (t); +} + +template +void Writer::set (std::string const& tag, Type t) +{ + check (!tag.empty(), "Tag can't be empty"); + + impl_->nextCollectionEntry (object, "set"); + impl_->writeObjectTag (tag); + output (t); +} + +void Writer::startRoot (CollectionType type) +{ + check (impl_->empty(), "stack_ not empty() in start"); + impl_->start (type); +} + +void Writer::startAppend (CollectionType type) +{ + impl_->nextCollectionEntry (array, "startAppend"); + impl_->start (type); +} + +void Writer::startSet (CollectionType type, std::string const& key) +{ + impl_->nextCollectionEntry (object, "startSet"); + impl_->writeObjectTag (key); + impl_->start (type); +} + +void Writer::finish () +{ + impl_->finish (); +} + +} // New +} // RPC +} // ripple diff --git a/src/ripple/rpc/impl/JsonWriter.h b/src/ripple/rpc/impl/JsonWriter.h new file mode 100644 index 0000000000..55f56ec5e4 --- /dev/null +++ b/src/ripple/rpc/impl/JsonWriter.h @@ -0,0 +1,215 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_RIPPLE_BASICS_TYPES_JSONWRITER_H +#define RIPPLED_RIPPLE_BASICS_TYPES_JSONWRITER_H + +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +/** + * Writer implements an O(1)-space, O(1)-granular output JSON writer. + * + * O(1)-space means that it uses a fixed amount of memory, and that there are + * no heap allocations at each step of the way. + * + * O(1)-granular output means the writer only outputs in small segments of a + * bounded size, using a bounded number of CPU cycles in doing so. This is + * very helpful in scheduling long jobs. + * + * The tradeoff is that you have to fill items in the JSON tree as you go, + * and you can never go backward. + * + * Writer can write single JSON tokens, but the typical use is to write out an + * entire JSON object. For example: + * + * { + * Writer w (out); + * + * w.startObject (); // Start the root object. + * w.set ("hello", "world"); + * w.set ("goodbye", 23); + * w.finishObject (); // Finish the root object. + * } + * + * which outputs the string + * + * {"hello":"world","goodbye":23} + * + * There can be an object inside an object: + * + * { + * Writer w (out); + * + * w.startObject (); // Start the root object. + * w.set ("hello", "world"); + * + * w.startObjectSet ("subobject"); // Start a sub-object. + * w.set ("goodbye", 23); // Add a key, value assignment. + * w.finishObject (); // Finish the sub-object. + * + * w.finishObject (); // Finish the root-object. + * } + * + * which outputs the string + * + * {"hello":"world","subobject":{"goodbye":23}}. + * + * Arrays work similarly + * + * { + * Writer w (out); + * w.startObject (); // Start the root object. + * + * w.startArraySet ("hello"); // Start an array. + * w.append (23) // Append some items. + * w.append ("skidoo") + * w.finishArray (); // Finish the array. + * + * w.finishObject (); // Finish the root object. + * } + * + * which outputs the string + * + * {"hello":[23,"skidoo"]}. + * + * + * If you've reached the end of a long object, you can just use finishAll() + * which finishes all arrays and objects that you have started. + * + * { + * Writer w (out); + * w.startObject (); // Start the root object. + * + * w.startArraySet ("hello"); // Start an array. + * w.append (23) // Append an item. + * + * w.startArrayAppend () // Start a sub-array. + * w.append ("one"); + * w.append ("two"); + * + * w.startObjectAppend (); // Append a sub-object. + * w.finishAll (); // Finish everything. + * } + * + * which outputs the string + * + * {"hello":[23,["one","two",{}]]}. + * + * For convenience, the destructor of Writer calls w.finishAll() which makes + * sure that all arrays and objects are closed. This means that you can throw + * an exception, or have a coroutine simply clean up the stack, and be sure + * that you do in fact generate a complete JSON object. + */ + +class Writer +{ +public: + enum CollectionType {array, object}; + + explicit Writer (Output& output); + Writer(Writer&&); + Writer& operator=(Writer&&); + + ~Writer(); + + /** Start a new collection at the root level. May only be called once. */ + void startRoot (CollectionType); + + /** Start a new collection inside an array. */ + void startAppend (CollectionType); + + /** Start a new collection inside an object. */ + void startSet (CollectionType, std::string const& key); + + /** Finish the collection most recently started. */ + void finish (); + + /** Finish all objects and arrays. After finishArray() has been called, no + * more operations can be performed. */ + void finishAll (); + + /** Append a value to an array. + * + * Scalar must be a scalar - that is, a number, boolean, string, string + * literal, or nullptr. + */ + template + void append (Scalar); + + /** Add a key, value assignment to an object. + * + * Scalar must be a scalar - that is, a number, boolean, string, string + * literal, or nullptr. + * + * While the JSON spec doesn't explicitly disallow this, you should avoid + * calling this method twice with the same tag for the same object. + * + * If CHECK_JSON_WRITER is defined, this function throws an exception if if + * the tag you use has already been used in this object. + */ + template + void set (std::string const& key, Scalar value); + + // You won't need to call anything below here until you are writing single + // items (numbers, strings, bools, null) to a JSON stream. + + /*** Output a string. */ + void output (std::string const&); + + /*** Output a literal constant or C string. */ + void output (char const*); + + /** Output numbers, booleans, or nullptr. */ + template + void output (Scalar t); + +private: + class Impl; + std::unique_ptr impl_; +}; + +class JsonException : public std::exception +{ +public: + explicit JsonException (std::string const& name) : name_(name) {} + const char* what() const throw() override + { + return name_.c_str(); + } + +private: + std::string const name_; +}; + +inline void check (bool condition, std::string const& message) +{ + if (!condition) + throw JsonException (message); +} + +} // New +} // RPC +} // ripple + +#endif diff --git a/src/ripple/rpc/impl/JsonWriter_test.cpp b/src/ripple/rpc/impl/JsonWriter_test.cpp new file mode 100644 index 0000000000..e8de1344b0 --- /dev/null +++ b/src/ripple/rpc/impl/JsonWriter_test.cpp @@ -0,0 +1,182 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +class JsonWriter_test : public TestOutputSuite +{ +public: + void testTrivial () + { + setup ("trivial"); + expect (output_.data.empty ()); + writer_->output (0); + expectResult("0"); + } + + void testPrimitives () + { + setup ("true"); + writer_->output (true); + expectResult ("true"); + + setup ("false"); + writer_->output (false); + expectResult ("false"); + + setup ("23"); + writer_->output (23); + expectResult ("23"); + + setup ("23.5"); + writer_->output (23.5); + expectResult ("23.5"); + + setup ("a string"); + writer_->output ("a string"); + expectResult ("\"a string\""); + + setup ("nullptr"); + writer_->output (nullptr); + expectResult ("null"); + } + + void testEmpty () + { + setup ("empty array"); + writer_->startRoot (Writer::array); + writer_->finish (); + expectResult ("[]"); + + setup ("empty object"); + writer_->startRoot (Writer::object); + writer_->finish (); + expectResult ("{}"); + } + + void testEscaping () + { + setup ("backslash"); + writer_->output ("\\"); + expectResult ("\"\\\\\""); + + setup ("quote"); + writer_->output ("\""); + expectResult ("\"\\\"\""); + + setup ("backslash and quote"); + writer_->output ("\\\""); + expectResult ("\"\\\\\\\"\""); + + setup ("escape embedded"); + writer_->output ("this contains a \\ in the middle of it."); + expectResult ("\"this contains a \\\\ in the middle of it.\""); + + setup ("remaining escapes"); + writer_->output ("\b\f\n\r\t"); + expectResult ("\"\\b\\f\\n\\r\\t\""); + } + + void testArray () + { + setup ("empty array"); + writer_->startRoot (Writer::array); + writer_->append (12); + writer_->finish (); + expectResult ("[12]"); + } + + void testLongArray () + { + setup ("long array"); + writer_->startRoot (Writer::array); + writer_->append (12); + writer_->append (true); + writer_->append ("hello"); + writer_->finish (); + expectResult ("[12,true,\"hello\"]"); + } + + void testEmbeddedArraySimple () + { + setup ("embedded array simple"); + writer_->startRoot (Writer::array); + writer_->startAppend (Writer::array); + writer_->finish (); + writer_->finish (); + expectResult ("[[]]"); + } + + void testObject () + { + setup ("object"); + writer_->startRoot (Writer::object); + writer_->set ("hello", "world"); + writer_->finish (); + + expectResult ("{\"hello\":\"world\"}"); + } + + void testComplexObject () + { + setup ("complex object"); + writer_->startRoot (Writer::object); + + writer_->set ("hello", "world"); + writer_->startSet (Writer::array, "array"); + + writer_->append (true); + writer_->append (12); + writer_->startAppend (Writer::array); + writer_->startAppend (Writer::object); + writer_->set ("goodbye", "cruel world."); + writer_->startSet (Writer::array, "subarray"); + writer_->append (23.5); + writer_->finishAll (); + + expectResult ("{\"hello\":\"world\",\"array\":[true,12," + "[{\"goodbye\":\"cruel world.\"," + "\"subarray\":[23.5]}]]}"); + } + + void run () override + { + testTrivial (); + testPrimitives (); + testEmpty (); + testEscaping (); + testArray (); + testLongArray (); + testEmbeddedArraySimple (); + testObject (); + testComplexObject (); + } +}; + +BEAST_DEFINE_TESTSUITE(JsonWriter, ripple_basics, ripple); + +} // New +} // RPC +} // ripple diff --git a/src/ripple/rpc/impl/TestOutputSuite.h b/src/ripple/rpc/impl/TestOutputSuite.h new file mode 100644 index 0000000000..604b633b8c --- /dev/null +++ b/src/ripple/rpc/impl/TestOutputSuite.h @@ -0,0 +1,67 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_RIPPLE_RPC_IMPL_TESTOUTPUT_H +#define RIPPLED_RIPPLE_RPC_IMPL_TESTOUTPUT_H + +#include +#include + +namespace ripple { +namespace RPC { +namespace New { + +struct TestOutput : public Output +{ + void output (char const* s, size_t length) override + { + data.append (s, length); + } + + std::string data; +}; + + +class TestOutputSuite : public beast::unit_test::suite +{ +protected: + TestOutput output_; + std::unique_ptr writer_; + + void setup (std::string const& testName) + { + testcase (testName); + output_.data.clear (); + writer_ = std::make_unique (output_); + } + + // Test the result and report values. + void expectResult (std::string const& expected) + { + expect (output_.data == expected, + "\nresult: " + output_.data + + "\nexpected: " + expected); + } +}; + +} // New +} // RPC +} // ripple + +#endif diff --git a/src/ripple/unity/rpcx.cpp b/src/ripple/unity/rpcx.cpp index 52b9dc43d3..d8a37586fa 100644 --- a/src/ripple/unity/rpcx.cpp +++ b/src/ripple/unity/rpcx.cpp @@ -32,6 +32,8 @@ #include #include +#include +#include #include #include #include @@ -107,3 +109,6 @@ #include #include #include + +#include +#include