Merge commit '827e6fe617808139556e4eb1513f468941f20912' into sync-2.0.1-conan

Add a new RPC / WS call for `server_definitions`, which returns an
SDK-compatible `definitions.json` (binary enum definitions) generated by
the server. This enables clients/libraries to dynamically work with new
fields and features, such as ones that may become available on side
chains. Clients query `server_definitions` on a node from the network
they want to work with, and immediately know how to speak that node's
binary "language", even if new features are added to it in the future
(as long as there are no new serialized types that the software doesn't
know how to serialize/deserialize).

Example:

```js
> {"command": "server_definitions"}
< {
    "result": {
        "FIELDS": [
            [
                "Generic",
                {
                    "isSerialized": false,
                    "isSigningField": false,
                    "isVLEncoded": false,
                    "nth": 0,
                    "type": "Unknown"
                }
            ],
            [
                "Invalid",
                {
                    "isSerialized": false,
                    "isSigningField": false,
                    "isVLEncoded": false,
                    "nth": -1,
                    "type": "Unknown"
                }
            ],
            [
                "ObjectEndMarker",
                {
                    "isSerialized": false,
                    "isSigningField": true,
                    "isVLEncoded": false,
                    "nth": 1,
                    "type": "STObject"
                }
            ],
        ...
```

Close #3657

---------

Co-authored-by: Richard Holland <richard.holland@starstone.co.nz>

src/ripple/app/main/Main.cpp

@@ -168,6 +168,7 @@ printHelp(const po::options_description& desc)
            "     peer_reservations_list\n"
            "     ripple ...\n"
            "     ripple_path_find <json> [<ledger>]\n"
+           "     server_definitions [<hash>]\n"
            "     server_info [counters]\n"
            "     server_state [counters]\n"
            "     sign <private_key> <tx_json> [offline]\n"

src/ripple/net/impl/RPCCall.cpp

@@ -1340,6 +1340,20 @@ private:
         return jvRequest;
     }
 
+    // server_definitions [hash]
+    Json::Value
+    parseServerDefinitions(Json::Value const& jvParams)
+    {
+        Json::Value jvRequest{Json::objectValue};
+
+        if (jvParams.size() == 1)
+        {
+            jvRequest[jss::hash] = jvParams[0u].asString();
+        }
+
+        return jvRequest;
+    }
+
     // server_info [counters]
     Json::Value
     parseServerInfo(Json::Value const& jvParams)
@@ -1406,6 +1420,7 @@ public:
             {"channel_verify", &RPCParser::parseChannelVerify, 4, 4},
             {"connect", &RPCParser::parseConnect, 1, 2},
             {"consensus_info", &RPCParser::parseAsIs, 0, 0},
+            {"crawl_shards", &RPCParser::parseAsIs, 0, 2},
             {"deposit_authorized", &RPCParser::parseDepositAuthorized, 2, 3},
             {"download_shard", &RPCParser::parseDownloadShard, 2, -1},
             {"feature", &RPCParser::parseFeature, 0, 2},
@@ -1444,14 +1459,14 @@ public:
              1},
             {"peer_reservations_list", &RPCParser::parseAsIs, 0, 0},
             {"ripple_path_find", &RPCParser::parseRipplePathFind, 1, 2},
-            {"sign", &RPCParser::parseSignSubmit, 2, 3},
-            {"sign_for", &RPCParser::parseSignFor, 3, 4},
-            {"submit", &RPCParser::parseSignSubmit, 1, 3},
-            {"submit_multisigned", &RPCParser::parseSubmitMultiSigned, 1, 1},
+            {"server_definitions", &RPCParser::parseServerDefinitions, 0, 1},
             {"server_info", &RPCParser::parseServerInfo, 0, 1},
             {"server_state", &RPCParser::parseServerInfo, 0, 1},
-            {"crawl_shards", &RPCParser::parseAsIs, 0, 2},
+            {"sign", &RPCParser::parseSignSubmit, 2, 3},
+            {"sign_for", &RPCParser::parseSignFor, 3, 4},
             {"stop", &RPCParser::parseAsIs, 0, 0},
+            {"submit", &RPCParser::parseSignSubmit, 1, 3},
+            {"submit_multisigned", &RPCParser::parseSubmitMultiSigned, 1, 1},
             {"transaction_entry", &RPCParser::parseTransactionEntry, 2, 2},
             {"tx", &RPCParser::parseTx, 1, 4},
             {"tx_history", &RPCParser::parseTxHistory, 1, 1},

src/ripple/protocol/SField.h

@@ -53,43 +53,65 @@ class STXChainBridge;
 class STVector256;
 class Definitions;
 
-enum SerializedTypeID {
-    // special types
-    STI_UNKNOWN = -2,
-    STI_NOTPRESENT = 0,
+#pragma push_macro("XMACRO")
+#undef XMACRO
 
-    // // types (common)
-    STI_UINT16 = 1,
-    STI_UINT32 = 2,
-    STI_UINT64 = 3,
-    STI_UINT128 = 4,
-    STI_UINT256 = 5,
-    STI_AMOUNT = 6,
-    STI_VL = 7,
-    STI_ACCOUNT = 8,
-    // 9-13 are reserved
-    STI_OBJECT = 14,
-    STI_ARRAY = 15,
+#define XMACRO(STYPE)                             \
+    /* special types */                           \
+    STYPE(STI_UNKNOWN, -2)                        \
+    STYPE(STI_NOTPRESENT, 0)                      \
+    STYPE(STI_UINT16, 1)                          \
+                                                  \
+    /* types (common) */                          \
+    STYPE(STI_UINT32, 2)                          \
+    STYPE(STI_UINT64, 3)                          \
+    STYPE(STI_UINT128, 4)                         \
+    STYPE(STI_UINT256, 5)                         \
+    STYPE(STI_AMOUNT, 6)                          \
+    STYPE(STI_VL, 7)                              \
+    STYPE(STI_ACCOUNT, 8)                         \
+                                                  \
+    /* 9-13 are reserved */                       \
+    STYPE(STI_OBJECT, 14)                         \
+    STYPE(STI_ARRAY, 15)                          \
+                                                  \
+    /* types (uncommon) */                        \
+    STYPE(STI_UINT8, 16)                          \
+    STYPE(STI_UINT160, 17)                        \
+    STYPE(STI_PATHSET, 18)                        \
+    STYPE(STI_VECTOR256, 19)                      \
+    STYPE(STI_UINT96, 20)                         \
+    STYPE(STI_UINT192, 21)                        \
+    STYPE(STI_UINT384, 22)                        \
+    STYPE(STI_UINT512, 23)                        \
+    STYPE(STI_ISSUE, 24)                          \
+    STYPE(STI_XCHAIN_BRIDGE, 25)                  \
+                                                  \
+    /* high-level types */                        \
+    /* cannot be serialized inside other types */ \
+    STYPE(STI_TRANSACTION, 10001)                 \
+    STYPE(STI_LEDGERENTRY, 10002)                 \
+    STYPE(STI_VALIDATION, 10003)                  \
+    STYPE(STI_METADATA, 10004)
 
-    // types (uncommon)
-    STI_UINT8 = 16,
-    STI_UINT160 = 17,
-    STI_PATHSET = 18,
-    STI_VECTOR256 = 19,
-    STI_UINT96 = 20,
-    STI_UINT192 = 21,
-    STI_UINT384 = 22,
-    STI_UINT512 = 23,
-    STI_ISSUE = 24,
-    STI_XCHAIN_BRIDGE = 25,
+#pragma push_macro("TO_ENUM")
+#undef TO_ENUM
+#pragma push_macro("TO_MAP")
+#undef TO_MAP
 
-    // high level types
-    // cannot be serialized inside other types
-    STI_TRANSACTION = 10001,
-    STI_LEDGERENTRY = 10002,
-    STI_VALIDATION = 10003,
-    STI_METADATA = 10004,
-};
+#define TO_ENUM(name, value) name = value,
+#define TO_MAP(name, value) {#name, value},
+
+enum SerializedTypeID { XMACRO(TO_ENUM) };
+
+static std::map<std::string, int> const sTypeMap = {XMACRO(TO_MAP)};
+
+#undef XMACRO
+#undef TO_ENUM
+
+#pragma pop_macro("XMACRO")
+#pragma pop_macro("TO_ENUM")
+#pragma pop_macro("TO_MAP")
 
 // constexpr
 inline int

src/ripple/protocol/TER.h

@@ -26,6 +26,7 @@
 #include <optional>
 #include <ostream>
 #include <string>
+#include <unordered_map>
 
 namespace ripple {
 
@@ -668,6 +669,11 @@ isTecClaim(TER x)
     return ((x) >= tecCLAIM);
 }
 
+std::unordered_map<
+    TERUnderlyingType,
+    std::pair<char const* const, char const* const>> const&
+transResults();
+
 bool
 transResultInfo(TER code, std::string& token, std::string& text);
 

src/ripple/protocol/impl/TER.cpp

@@ -20,13 +20,10 @@
 #include <ripple/protocol/TER.h>
 #include <boost/range/adaptor/transformed.hpp>
 #include <type_traits>
-#include <unordered_map>
 
 namespace ripple {
 
-namespace detail {
-
-static std::unordered_map<
+std::unordered_map<
     TERUnderlyingType,
     std::pair<char const* const, char const* const>> const&
 transResults()
@@ -239,12 +236,10 @@ transResults()
     return results;
 }
 
-}  // namespace detail
-
 bool
 transResultInfo(TER code, std::string& token, std::string& text)
 {
-    auto& results = detail::transResults();
+    auto& results = transResults();
 
     auto const r = results.find(TERtoInt(code));
 
@@ -278,7 +273,7 @@ std::optional<TER>
 transCode(std::string const& token)
 {
     static auto const results = [] {
-        auto& byTer = detail::transResults();
+        auto& byTer = transResults();
         auto range = boost::make_iterator_range(byTer.begin(), byTer.end());
         auto tRange = boost::adaptors::transform(range, [](auto const& r) {
             return std::make_pair(r.second.first, r.first);

src/ripple/protocol/jss.h

@@ -110,8 +110,11 @@ JSS(HookParameterValue);  // field
 JSS(HookParameter);       // field
 JSS(HookGrant);           // field
 JSS(isSerialized);        // out: RPC server_definitions
+                          // matches definitions.json format
 JSS(isSigningField);      // out: RPC server_definitions
+                          // matches definitions.json format
 JSS(isVLEncoded);         // out: RPC server_definitions
+                          // matches definitions.json format
 JSS(Import);
 JSS(ImportVLSequence);
 JSS(Invalid);                  //
@@ -788,8 +791,11 @@ JSS(type);                  // in: AccountObjects
                             // out: NetworkOPs RPC server_definitions
                             //      OverlayImpl, Logic
 JSS(TRANSACTION_RESULTS);   // out: RPC server_definitions
+                            // matches definitions.json format
 JSS(TRANSACTION_TYPES);     // out: RPC server_definitions
+                            // matches definitions.json format
 JSS(TYPES);                 // out: RPC server_definitions
+                            // matches definitions.json format
 JSS(TRANSACTION_FLAGS);     // out: RPC server_definitions
 JSS(TRANSACTION_FLAGS_INDICES);  // out: RPC server_definitions
 JSS(type_hex);                   // out: STPathSet

src/ripple/rpc/handlers/ServerInfo.cpp

@@ -20,10 +20,21 @@
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/reporting/P2pProxy.h>
+#include <ripple/json/json_value.h>
+#include <ripple/net/RPCErr.h>
+#include <ripple/protocol/LedgerFormats.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/TER.h>
+#include <ripple/protocol/TxFormats.h>
+#include <ripple/protocol/digest.h>
 #include <ripple/protocol/jss.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/Role.h>
 
+#include <boost/algorithm/string.hpp>
+
+#include <unordered_map>
+
 namespace ripple {
 
 Json::Value

src/ripple/rpc/impl/Handler.cpp

@@ -100,6 +100,7 @@ Handler const handlerArray[]{
     {"channel_verify", byRef(&doChannelVerify), Role::USER, NO_CONDITION},
     {"connect", byRef(&doConnect), Role::ADMIN, NO_CONDITION},
     {"consensus_info", byRef(&doConsensusInfo), Role::ADMIN, NO_CONDITION},
+    {"crawl_shards", byRef(&doCrawlShards), Role::ADMIN, NO_CONDITION},
     {"deposit_authorized",
      byRef(&doDepositAuthorized),
      Role::USER,
@@ -114,6 +115,7 @@ Handler const handlerArray[]{
     {"feature", byRef(&doFeature), Role::ADMIN, NO_CONDITION},
     {"fee", byRef(&doFee), Role::USER, NEEDS_CURRENT_LEDGER},
     {"fetch_info", byRef(&doFetchInfo), Role::ADMIN, NO_CONDITION},
+    {"inject", byRef(&doInject), Role::ADMIN, NEEDS_CURRENT_LEDGER},
     {"ledger_accept",
      byRef(&doLedgerAccept),
      Role::ADMIN,
@@ -159,22 +161,20 @@ Handler const handlerArray[]{
      Role::ADMIN,
      NO_CONDITION},
     {"ripple_path_find", byRef(&doRipplePathFind), Role::USER, NO_CONDITION},
-    {"sign", byRef(&doSign), Role::USER, NO_CONDITION},
-    {"sign_for", byRef(&doSignFor), Role::USER, NO_CONDITION},
-    {"inject", byRef(&doInject), Role::ADMIN, NEEDS_CURRENT_LEDGER},
-    {"submit", byRef(&doSubmit), Role::USER, NEEDS_CURRENT_LEDGER},
-    {"submit_multisigned",
-     byRef(&doSubmitMultiSigned),
-     Role::USER,
-     NEEDS_CURRENT_LEDGER},
     {"server_definitions",
      byRef(&doServerDefinitions),
      Role::USER,
      NO_CONDITION},
     {"server_info", byRef(&doServerInfo), Role::USER, NO_CONDITION},
     {"server_state", byRef(&doServerState), Role::USER, NO_CONDITION},
-    {"crawl_shards", byRef(&doCrawlShards), Role::ADMIN, NO_CONDITION},
+    {"sign", byRef(&doSign), Role::USER, NO_CONDITION},
+    {"sign_for", byRef(&doSignFor), Role::USER, NO_CONDITION},
     {"stop", byRef(&doStop), Role::ADMIN, NO_CONDITION},
+    {"submit", byRef(&doSubmit), Role::USER, NEEDS_CURRENT_LEDGER},
+    {"submit_multisigned",
+     byRef(&doSubmitMultiSigned),
+     Role::USER,
+     NEEDS_CURRENT_LEDGER},
     {"transaction_entry", byRef(&doTransactionEntry), Role::USER, NO_CONDITION},
     {"tx", byRef(&doTxJson), Role::USER, NEEDS_NETWORK_CONNECTION},
     {"tx_history", byRef(&doTxHistory), Role::USER, NO_CONDITION, 1, 1},

src/test/rpc/ServerInfo_test.cpp

@@ -79,6 +79,8 @@ admin = 127.0.0.1
     void
     testServerInfo()
     {
+        testcase("server_info");
+
         using namespace test::jtx;
 
         {
@@ -151,6 +153,8 @@ admin = 127.0.0.1
     void
     testServerDefinitions()
     {
+        testcase("server_definitions");
+
         using namespace test::jtx;
 
         {
@@ -158,6 +162,84 @@ admin = 127.0.0.1
             auto const result = env.rpc("server_definitions");
             BEAST_EXPECT(!result[jss::result].isMember(jss::error));
             BEAST_EXPECT(result[jss::result][jss::status] == "success");
+            BEAST_EXPECT(result[jss::result].isMember(jss::FIELDS));
+            BEAST_EXPECT(result[jss::result].isMember(jss::LEDGER_ENTRY_TYPES));
+            BEAST_EXPECT(
+                result[jss::result].isMember(jss::TRANSACTION_RESULTS));
+            BEAST_EXPECT(result[jss::result].isMember(jss::TRANSACTION_TYPES));
+            BEAST_EXPECT(result[jss::result].isMember(jss::TYPES));
+            BEAST_EXPECT(result[jss::result].isMember(jss::hash));
+
+            // test a random element of each result
+            // (testing the whole output would be difficult to maintain)
+
+            {
+                auto const firstField = result[jss::result][jss::FIELDS][0u];
+                BEAST_EXPECT(firstField[0u].asString() == "Generic");
+                BEAST_EXPECT(
+                    firstField[1][jss::isSerialized].asBool() == false);
+                BEAST_EXPECT(
+                    firstField[1][jss::isSigningField].asBool() == false);
+                BEAST_EXPECT(firstField[1][jss::isVLEncoded].asBool() == false);
+                BEAST_EXPECT(firstField[1][jss::nth].asUInt() == 0);
+                BEAST_EXPECT(firstField[1][jss::type].asString() == "Unknown");
+            }
+
+            BEAST_EXPECT(
+                result[jss::result][jss::LEDGER_ENTRY_TYPES]["AccountRoot"]
+                    .asUInt() == 97);
+            BEAST_EXPECT(
+                result[jss::result][jss::TRANSACTION_RESULTS]["tecDIR_FULL"]
+                    .asUInt() == 121);
+            BEAST_EXPECT(
+                result[jss::result][jss::TRANSACTION_TYPES]["Payment"]
+                    .asUInt() == 0);
+            BEAST_EXPECT(
+                result[jss::result][jss::TYPES]["AccountID"].asUInt() == 8);
+        }
+
+        // test providing the same hash
+        {
+            Env env(*this);
+            auto const firstResult = env.rpc("server_definitions");
+            auto const hash = firstResult[jss::result][jss::hash].asString();
+            auto const hashParam =
+                std::string("{ ") + "\"hash\": \"" + hash + "\"}";
+
+            auto const result =
+                env.rpc("json", "server_definitions", hashParam);
+            BEAST_EXPECT(!result[jss::result].isMember(jss::error));
+            BEAST_EXPECT(result[jss::result][jss::status] == "success");
+            BEAST_EXPECT(!result[jss::result].isMember(jss::FIELDS));
+            BEAST_EXPECT(
+                !result[jss::result].isMember(jss::LEDGER_ENTRY_TYPES));
+            BEAST_EXPECT(
+                !result[jss::result].isMember(jss::TRANSACTION_RESULTS));
+            BEAST_EXPECT(!result[jss::result].isMember(jss::TRANSACTION_TYPES));
+            BEAST_EXPECT(!result[jss::result].isMember(jss::TYPES));
+            BEAST_EXPECT(result[jss::result].isMember(jss::hash));
+        }
+
+        // test providing a different hash
+        {
+            Env env(*this);
+            std::string const hash =
+                "54296160385A27154BFA70A239DD8E8FD4CC2DB7BA32D970BA3A5B132CF749"
+                "D1";
+            auto const hashParam =
+                std::string("{ ") + "\"hash\": \"" + hash + "\"}";
+
+            auto const result =
+                env.rpc("json", "server_definitions", hashParam);
+            BEAST_EXPECT(!result[jss::result].isMember(jss::error));
+            BEAST_EXPECT(result[jss::result][jss::status] == "success");
+            BEAST_EXPECT(result[jss::result].isMember(jss::FIELDS));
+            BEAST_EXPECT(result[jss::result].isMember(jss::LEDGER_ENTRY_TYPES));
+            BEAST_EXPECT(
+                result[jss::result].isMember(jss::TRANSACTION_RESULTS));
+            BEAST_EXPECT(result[jss::result].isMember(jss::TRANSACTION_TYPES));
+            BEAST_EXPECT(result[jss::result].isMember(jss::TYPES));
+            BEAST_EXPECT(result[jss::result].isMember(jss::hash));
         }
     }