force build

* Merge fbbea9e6e2

* Merge 7741483894

* clang-format

* Oops

---------

Co-authored-by: Wietse Wind <wrw@Wietses-MacBook-Pro.local>

README.md

@@ -1,4 +1,4 @@
-# Xahau
+# Xahau 
 
 **Note:** Throughout this README, references to "we" or "our" pertain to the community and contributors involved in the Xahau network. It does not imply a legal entity or a specific collection of individuals.
 
@@ -68,4 +68,4 @@ git-subtree. See those directories' README files for more details.
 - **Testnet & Faucet**: Test applications and obtain test XAH at [xahau-test.net](https://xahau-test.net) and use the testnet explorer at [explorer.xahau.network](https://explorer.xahau.network).
 - **Supporting Wallets**: A list of wallets that support XAH and Xahau-based assets.
   - [Xumm](https://xumm.app)
-  - [Crossmark](https://crossmark.io)
+  - [Crossmark](https://crossmark.io)

src/ripple/app/consensus/RCLConsensus.cpp

@@ -134,8 +134,12 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& hash)
             acquiringLedger_ = hash;
 
             app_.getJobQueue().addJob(
-                jtADVANCE, "getConsensusLedger", [id = hash, &app = app_]() {
-                    app.getInboundLedgers().acquire(
+                jtADVANCE,
+                "getConsensusLedger1",
+                [id = hash, &app = app_, this]() {
+                    JLOG(j_.debug())
+                        << "JOB advanceLedger getConsensusLedger1 started";
+                    app.getInboundLedgers().acquireAsync(
                         id, 0, InboundLedger::Reason::CONSENSUS);
                 });
         }

src/ripple/app/consensus/RCLValidations.cpp

@@ -135,8 +135,10 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash)
         Application* pApp = &app_;
 
         app_.getJobQueue().addJob(
-            jtADVANCE, "getConsensusLedger", [pApp, hash]() {
-                pApp->getInboundLedgers().acquire(
+            jtADVANCE, "getConsensusLedger2", [pApp, hash, this]() {
+                JLOG(j_.debug())
+                    << "JOB advanceLedger getConsensusLedger2 started";
+                pApp->getInboundLedgers().acquireAsync(
                     hash, 0, InboundLedger::Reason::CONSENSUS);
             });
         return std::nullopt;
@@ -152,7 +154,9 @@ void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,
-    std::string const& source)
+    std::string const& source,
+    BypassAccept const bypassAccept,
+    std::optional<beast::Journal> j)
 {
     auto const& signingKey = val->getSignerPublic();
     auto const& hash = val->getLedgerHash();
@@ -177,7 +181,23 @@ handleNewValidation(
     if (outcome == ValStatus::current)
     {
         if (val->isTrusted())
-            app.getLedgerMaster().checkAccept(hash, seq);
+        {
+            // Was: app.getLedgerMaster().checkAccept(hash, seq);
+            // https://github.com/XRPLF/rippled/commit/fbbea9e6e25795a8a6bd1bf64b780771933a9579
+            if (bypassAccept == BypassAccept::yes)
+            {
+                assert(j.has_value());
+                if (j.has_value())
+                {
+                    JLOG(j->trace()) << "Bypassing checkAccept for validation "
+                                     << val->getLedgerHash();
+                }
+            }
+            else
+            {
+                app.getLedgerMaster().checkAccept(hash, seq);
+            }
+        }
         return;
     }
 

src/ripple/app/consensus/RCLValidations.h

@@ -25,12 +25,16 @@
 #include <ripple/protocol/Protocol.h>
 #include <ripple/protocol/RippleLedgerHash.h>
 #include <ripple/protocol/STValidation.h>
+#include <optional>
+#include <set>
 #include <vector>
 
 namespace ripple {
 
 class Application;
 
+enum class BypassAccept : bool { no = false, yes };
+
 /** Wrapper over STValidation for generic Validation code
 
     Wraps an STValidation for compatibility with the generic validation code.
@@ -248,7 +252,9 @@ void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,
-    std::string const& source);
+    std::string const& source,
+    BypassAccept const bypassAccept = BypassAccept::no,
+    std::optional<beast::Journal> j = std::nullopt);
 
 }  // namespace ripple
 

src/ripple/app/ledger/InboundLedgers.h

@@ -38,10 +38,21 @@ public:
     virtual ~InboundLedgers() = default;
 
     // VFALCO TODO Should this be called findOrAdd ?
+    // Callers should use this if they possibly need an authoritative
+    // response immediately.
     //
     virtual std::shared_ptr<Ledger const>
     acquire(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason) = 0;
 
+    // Callers should use this if they are known to be executing on the Job
+    // Queue. TODO review whether all callers of acquire() can use this
+    // instead. Inbound ledger acquisition is asynchronous anyway.
+    virtual void
+    acquireAsync(
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) = 0;
+
     virtual std::shared_ptr<InboundLedger>
     find(LedgerHash const& hash) = 0;
 

src/ripple/app/ledger/impl/InboundLedger.cpp

@@ -560,7 +560,7 @@ InboundLedger::trigger(std::shared_ptr<Peer> const& peer, TriggerReason reason)
         return;
     }
 
-    if (auto stream = journal_.trace())
+    if (auto stream = journal_.debug())
     {
         if (peer)
             stream << "Trigger acquiring ledger " << hash_ << " from " << peer;

src/ripple/app/ledger/impl/InboundLedgers.cpp

@@ -28,6 +28,7 @@
 #include <ripple/core/JobQueue.h>
 #include <ripple/nodestore/DatabaseShard.h>
 #include <ripple/protocol/jss.h>
+#include <exception>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -141,6 +142,37 @@ public:
         return inbound->getLedger();
     }
 
+    void
+    acquireAsync(
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) override
+    {
+        std::unique_lock lock(acquiresMutex_);
+        try
+        {
+            if (pendingAcquires_.contains(hash))
+                return;
+            pendingAcquires_.insert(hash);
+            lock.unlock();
+            acquire(hash, seq, reason);
+        }
+        catch (std::exception const& e)
+        {
+            JLOG(j_.warn())
+                << "Exception thrown for acquiring new inbound ledger " << hash
+                << ": " << e.what();
+        }
+        catch (...)
+        {
+            JLOG(j_.warn())
+                << "Unknown exception thrown for acquiring new inbound ledger "
+                << hash;
+        }
+        lock.lock();
+        pendingAcquires_.erase(hash);
+    }
+
     std::shared_ptr<InboundLedger>
     find(uint256 const& hash) override
     {
@@ -426,6 +458,9 @@ private:
     beast::insight::Counter mCounter;
 
     std::unique_ptr<PeerSetBuilder> mPeerSetBuilder;
+
+    std::set<uint256> pendingAcquires_;
+    std::mutex acquiresMutex_;
 };
 
 //------------------------------------------------------------------------------

src/ripple/app/misc/NetworkOPs.cpp

@@ -70,7 +70,9 @@
 #include <boost/asio/ip/host_name.hpp>
 #include <boost/asio/steady_timer.hpp>
 
+#include <exception>
 #include <mutex>
+#include <set>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -776,6 +778,9 @@ private:
 
     StateAccounting accounting_{};
 
+    std::set<uint256> pendingValidations_;
+    std::mutex validationsMutex_;
+
 private:
     struct Stats
     {
@@ -1791,7 +1796,8 @@ NetworkOPsImp::checkLastClosedLedger(
     }
 
     JLOG(m_journal.warn()) << "We are not running on the consensus ledger";
-    JLOG(m_journal.info()) << "Our LCL: " << getJson({*ourClosed, {}});
+    JLOG(m_journal.info()) << "Our LCL: " << ourClosed->info().hash
+                           << getJson({*ourClosed, {}});
     JLOG(m_journal.info()) << "Net LCL " << closedLedger;
 
     if ((mMode == OperatingMode::TRACKING) || (mMode == OperatingMode::FULL))
@@ -2345,7 +2351,37 @@ NetworkOPsImp::recvValidation(
     JLOG(m_journal.trace())
         << "recvValidation " << val->getLedgerHash() << " from " << source;
 
-    handleNewValidation(app_, val, source);
+    // handleNewValidation(app_, val, source);
+    // https://github.com/XRPLF/rippled/commit/fbbea9e6e25795a8a6bd1bf64b780771933a9579
+    std::unique_lock lock(validationsMutex_);
+    BypassAccept bypassAccept = BypassAccept::no;
+    try
+    {
+        if (pendingValidations_.contains(val->getLedgerHash()))
+            bypassAccept = BypassAccept::yes;
+        else
+            pendingValidations_.insert(val->getLedgerHash());
+        lock.unlock();
+        handleNewValidation(app_, val, source, bypassAccept, m_journal);
+    }
+    catch (std::exception const& e)
+    {
+        JLOG(m_journal.warn())
+            << "Exception thrown for handling new validation "
+            << val->getLedgerHash() << ": " << e.what();
+    }
+    catch (...)
+    {
+        JLOG(m_journal.warn())
+            << "Unknown exception thrown for handling new validation "
+            << val->getLedgerHash();
+    }
+    if (bypassAccept == BypassAccept::no)
+    {
+        lock.lock();
+        pendingValidations_.erase(val->getLedgerHash());
+        lock.unlock();
+    }
 
     pubValidation(val);
 

src/ripple/ledger/impl/ApplyView.cpp

@@ -91,8 +91,10 @@ ApplyView::dirAdd(
         return page;
     }
 
+    bool const capped = !rules().enabled(fixPageCap);
+
     // Check whether we're out of pages.
-    if (++page >= dirNodeMaxPages)
+    if (++page >= dirNodeMaxPages && capped)
         return std::nullopt;
 
     // We are about to create a new node; we'll link it to

src/ripple/protocol/Feature.h

@@ -74,7 +74,7 @@ namespace detail {
 // Feature.cpp. Because it's only used to reserve storage, and determine how
 // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than
 // the actual number of amendments. A LogicError on startup will verify this.
-static constexpr std::size_t numFeatures = 71;
+static constexpr std::size_t numFeatures = 72;
 
 /** Amendments that this server supports and the default voting behavior.
    Whether they are enabled depends on the Rules defined in the validated
@@ -359,6 +359,7 @@ extern uint256 const featureRemit;
 extern uint256 const featureZeroB2M;
 extern uint256 const fixNSDelete;
 extern uint256 const fix240819;
+extern uint256 const fixPageCap;
 
 }  // namespace ripple
 

src/ripple/protocol/impl/Feature.cpp

@@ -465,6 +465,7 @@ REGISTER_FEATURE(Remit,                         Supported::yes, VoteBehavior::De
 REGISTER_FEATURE(ZeroB2M,                       Supported::yes, VoteBehavior::DefaultNo);
 REGISTER_FIX    (fixNSDelete,                   Supported::yes, VoteBehavior::DefaultNo);
 REGISTER_FIX    (fix240819,                     Supported::yes, VoteBehavior::DefaultYes);
+REGISTER_FIX    (fixPageCap,                    Supported::yes, VoteBehavior::DefaultYes);
 
 // The following amendments are obsolete, but must remain supported
 // because they could potentially get enabled.

src/test/app/LedgerReplay_test.cpp

@@ -106,6 +106,14 @@ public:
         return {};
     }
 
+    virtual void
+    acquireAsync(
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) override
+    {
+    }
+
     virtual std::shared_ptr<InboundLedger>
     find(LedgerHash const& hash) override
     {

src/test/app/SetHook_test.cpp

@@ -19,6 +19,8 @@
 #include <ripple/app/hook/Enum.h>
 #include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/tx/impl/SetHook.h>
+#include <ripple/json/json_reader.h>
+#include <ripple/json/json_writer.h>
 #include <ripple/protocol/TxFlags.h>
 #include <ripple/protocol/jss.h>
 #include <test/app/SetHook_wasm.h>
@@ -87,6 +89,147 @@ public:
 // fee unit tests, the rest of the time we want to ignore it.
 #define HSFEE fee(100'000'000)
 #define M(m) memo(m, "", "")
+
+    std::unique_ptr<Config>
+    makePageCapConfig(
+        FeatureBitset features,
+        uint32_t networkID,
+        std::string fee,
+        std::string a_res,
+        std::string o_res,
+        uint32_t ledgerID)
+    {
+        using namespace jtx;
+
+        Json::Value jsonValue;
+        Json::Reader reader;
+        std::string base_genesis = R"json({
+        "ledger": {
+            "accepted": true,
+            "accountState": [
+                {
+                    "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+                    "Balance": "100000000000000000",
+                    "Flags": 0,
+                    "LedgerEntryType": "AccountRoot",
+                    "OwnerCount": 0,
+                    "PreviousTxnID": "A92EF82C3C68F771927E3892A2F708F12CBD492EF68A860F042E4053C8EC6C8D",
+                    "PreviousTxnLgrSeq": 0,
+                    "Sequence": 1,
+                    "index": "2B6AC232AA4C4BE41BF49D2459FA4A0347E1B543A4C92FCEE0821C0201E2E9A8"
+                },
+                {
+                    "Amendments": [],
+                    "Flags": 0,
+                    "LedgerEntryType": "Amendments",
+                    "index": "7DB0788C020F02780A673DC74757F23823FA3014C1866E72CC4CD8B226CD6EF4"
+                },
+                {
+                    "BaseFee": "A",
+                    "Flags": 0,
+                    "LedgerEntryType": "FeeSettings",
+                    "ReferenceFeeUnits": 10,
+                    "ReserveBase": 1000000,
+                    "ReserveIncrement": 200000,
+                    "XahauActivationLgrSeq": 0,
+                    "index": "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A651"
+                },
+                {
+                    "Flags": 0,
+                    "IndexNext": "40000",
+                    "IndexPrevious": "3fffe",
+                    "Indexes": [],
+                    "LedgerEntryType": "DirectoryNode",
+                    "Owner": "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn",
+                    "RootIndex": "EC1A88838E9ADA27E8ED583917C1530D2F48C3A3C93F50EDAD662D662E9BCC76",
+                    "index": "EC1A88838E9ADA27E8ED583917C1530D2F48C3A3C93F50EDAD662D662E9BCC76"
+                },
+                {
+                    "Flags": 0,
+                    "IndexNext": "3fffe",
+                    "IndexPrevious": "3fffd",
+                    "Indexes": [],
+                    "LedgerEntryType": "DirectoryNode",
+                    "Owner": "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn",
+                    "RootIndex": "EC1A88838E9ADA27E8ED583917C1530D2F48C3A3C93F50EDAD662D662E9BCC76",
+                    "index": "4A5F3F9E6762A4F89FFCD385FF2309E1F7D1309321BFEEA61D5C9ACB768DB61B"
+                },
+                {
+                    "Flags": 0,
+                    "Indexes": [],
+                    "LedgerEntryType": "DirectoryNode",
+                    "Owner": "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn",
+                    "RootIndex": "A33EC6BB85FB5674074C4A3A43373BB17645308F3EAE1933E3E35252162B217D",
+                    "index": "A33EC6BB85FB5674074C4A3A43373BB17645308F3EAE1933E3E35252162B217D"
+                },
+                {
+                    "Account": "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn",
+                    "Balance": "99999899000000",
+                    "Flags": 8388608,
+                    "LedgerEntryType": "AccountRoot",
+                    "HookNamespaces": [ "0000000000000000000000000000000000000000000000000000000000000000" ],
+                    "HookStateCount": 8388576,
+                    "OwnerCount": 8388577,
+                    "PreviousTxnID": "A92EF82C3C68F771927E3892A2F708F12CBD492EF68A860F042E4053C8EC6C8D",
+                    "PreviousTxnLgrSeq": 0,
+                    "Sequence": 3,
+                    "index": "92FA6A9FC8EA6018D5D16532D7795C91BFB0831355BDFDA177E86C8BF997985F"
+                }
+            ],
+            "account_hash": "5DF3A98772FB73E782B8740E87885C6BAD9BA486422E3626DEF968AD2CB2C514",
+            "close_flags": 0,
+            "close_time": 0,
+            "close_time_human": "2000-Jan-01 00:00:00.000000",
+            "close_time_resolution": 10,
+            "closed": true,
+            "hash": "56DA0940767AC2F17F0E384F04816002403D0756432B9D503DDA20128A2AAF11",
+            "ledger_hash": "56DA0940767AC2F17F0E384F04816002403D0756432B9D503DDA20128A2AAF11",
+            "ledger_index": "0",
+            "parent_close_time": 0,
+            "parent_hash": "56DA0940767AC2F17F0E384F04816002403D0756432B9D503DDA20128A2AAF11",
+            "seqNum": "5",
+            "totalCoins": "100000000000000000",
+            "total_coins": "100000000000000000",
+            "transaction_hash": "9A77D1D1A4B36DA77B9C4DC63FDEB8F821741D157802F9C42A6ED86003D8B4A0",
+            "transactions": []
+        },
+        "ledger_current_index": 0,
+        "status": "success",
+        "validated": true
+        })json";
+        reader.parse(base_genesis, jsonValue);
+
+        foreachFeature(features, [&](uint256 const& feature) {
+            std::string featureName = featureToName(feature);
+            std::optional<uint256> featureHash =
+                getRegisteredFeature(featureName);
+            if (featureHash.has_value())
+            {
+                std::string hashString = to_string(featureHash.value());
+                jsonValue["ledger"]["accountState"][1]["Amendments"].append(
+                    hashString);
+            }
+        });
+
+        jsonValue["ledger_current_index"] = ledgerID;
+        jsonValue["ledger"]["ledger_index"] = to_string(ledgerID);
+        jsonValue["ledger"]["seqNum"] = to_string(ledgerID);
+
+        return envconfig([&](std::unique_ptr<Config> cfg) {
+            cfg->NETWORK_ID = networkID;
+            cfg->START_LEDGER = jsonValue.toStyledString();
+            cfg->START_UP = Config::LOAD_JSON;
+            Section config;
+            config.append(
+                {"reference_fee = " + fee,
+                 "account_reserve = " + a_res,
+                 "owner_reserve = " + o_res});
+            auto setup = setup_FeeVote(config);
+            cfg->FEES = setup;
+            return cfg;
+        });
+    }
+
     void
     testHooksOwnerDir(FeatureBitset features)
     {
@@ -928,6 +1071,73 @@ public:
         }
     }
 
+    void
+    testPageCap(FeatureBitset features)
+    {
+        testcase("Test page cap");
+
+        using namespace jtx;
+
+        test::jtx::Env env{
+            *this,
+            makePageCapConfig(features, 21337, "10", "1000000", "200000", 0),
+            features};
+
+        bool const hasFix = env.current()->rules().enabled(fixPageCap);
+
+        auto const alice = Account{"alice"};
+        env.memoize(alice);
+
+        auto const bob = Account{"bob"};
+        env.fund(XRP(10000000), bob);
+
+        auto const preHookCount = (*env.le(alice))[sfHookStateCount];
+        auto const preOwnerCount = (*env.le(alice))[sfOwnerCount];
+
+        std::string hook =
+            "0061736D01000000012A0660057F7F7F7F7F017E60027F7F017E60027F7F017F60"
+            "047F7F7F7F017E60037F7F7E017E60017F017E02520603656E7605747261636500"
+            "0003656E760C686F6F6B5F6163636F756E74000103656E76025F67000203656E76"
+            "057374617465000303656E760973746174655F736574000303656E760661636365"
+            "70740004030201050503010002062B077F0141B088040B7F004180080B7F0041A2"
+            "080B7F004180080B7F0041B088040B7F0041000B7F0041010B07080104686F6F6B"
+            "00060ABF830001BB830002017F017E230041D0006B220124002001200036024C41"
+            "900841114180084110410010001A200141306A2200411410011A4101410110021A"
+            "200141286A41082000411410031A2001200131002F200131002842388620013100"
+            "294230867C200131002A4228867C200131002B4220867C200131002C4218867C20"
+            "0131002D4210867C200131002E4208867C7C3703202001410036021C0340419280"
+            "80807841C90110021A200128021C41C8014E4504402001200134021C2001290320"
+            "42C8017E7C370310200141106A220041082000410810041A2001200128021C4101"
+            "6A36021C0C010B0B2001200129032042017C3703202001200141286A220036020C"
+            "200128020C200129032042388842FF01833C0000200128020C2001290320423088"
+            "42FF01833C0001200128020C200129032042288842FF01833C0002200128020C20"
+            "0129032042208842FF01833C0003200128020C200129032042188842FF01833C00"
+            "04200128020C200129032042108842FF01833C0005200128020C20012903204208"
+            "8842FF01833C0006200128020C200129032042FF01833C00072000410820014130"
+            "6A411410041A4180084110421C1005200141D0006A24000B0B2801004180080B21"
+            "426173652E633A2043616C6C65642E0022426173652E633A2043616C6C65642E2"
+            "2";
+
+        // install the hook on alice
+        env(ripple::test::jtx::hook(alice, {{hso(hook, overrideFlag)}}, 0),
+            M("set fix_page_cap"),
+            HSFEE);
+        env.close();
+
+        env(invoke::invoke(alice),
+            M("test simple"),
+            fee(XRP(1)),
+            ter(tesSUCCESS));
+        env.close();
+
+        BEAST_EXPECT(
+            (*env.le(alice))[sfHookStateCount] == hasFix ? preHookCount + 200
+                                                         : preHookCount + 64);
+        BEAST_EXPECT(
+            (*env.le(alice))[sfOwnerCount] == hasFix ? preHookCount + 202
+                                                     : preHookCount + 66);
+    }
+
     void
     testCreate(FeatureBitset features)
     {
@@ -11761,6 +11971,7 @@ public:
 
         testNSDelete(features);
         testNSDeletePartial(features);
+        testPageCap(features);
 
         testWasm(features);
         test_accept(features);
@@ -11862,6 +12073,8 @@ public:
         testWithFeatures(sa - fixXahauV2);
         testWithFeatures(sa - fixXahauV1 - fixXahauV2);
         testWithFeatures(sa - fixXahauV1 - fixXahauV2 - fixNSDelete);
+        testWithFeatures(
+            sa - fixXahauV1 - fixXahauV2 - fixNSDelete - fixPageCap);
     }
 
 private: