feat(consensus): replace fake hashes with real SHAMap-backed commitSet/entropySet

Build real ephemeral (unbacked) SHAMaps for commitSet and entropySet using
ttCONSENSUS_ENTROPY entries with tfEntropyCommit/tfEntropyReveal flags.
Reuse InboundTransactions pipeline for peer fetch/diff/merge with no new
classes. Encode NodeID in sfAccount to avoid master-vs-signing key mismatch.
Add isPseudoTx guard in ConsensusTransSetSF to prevent pseudo-tx submission.
Route acquired RNG sets via isRngSet/gotRngSet in NetworkOPs mapComplete.

src/ripple/app/consensus/RCLConsensus.cpp

@@ -46,10 +46,12 @@
 #include <ripple/protocol/BuildInfo.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/TxFlags.h>
 #include <ripple/protocol/TxFormats.h>
 #include <ripple/protocol/digest.h>
 
 #include <algorithm>
+#include <cstring>
 #include <mutex>
 
 namespace ripple {
@@ -1150,51 +1152,93 @@ RCLConsensus::Adaptor::hasAnyReveals() const
 }
 
 uint256
-RCLConsensus::Adaptor::buildCommitSet()
+RCLConsensus::Adaptor::buildCommitSet(LedgerIndex seq)
 {
-    // Sort commits deterministically by public key
-    std::vector<std::pair<PublicKey, uint256>> sorted;
+    auto map =
+        std::make_shared<SHAMap>(SHAMapType::TRANSACTION, app_.getNodeFamily());
+    map->setUnbacked();
+
     for (auto const& [nodeId, commit] : pendingCommits_)
     {
-        auto it = nodeIdToKey_.find(nodeId);
-        if (it != nodeIdToKey_.end())
-            sorted.emplace_back(it->second, commit);
-    }
-    std::sort(sorted.begin(), sorted.end(), [](auto const& a, auto const& b) {
-        return a.first < b.first;
-    });
+        auto kit = nodeIdToKey_.find(nodeId);
+        if (kit == nodeIdToKey_.end())
+            continue;
 
-    Serializer s;
-    for (auto const& [key, commit] : sorted)
-    {
-        s.addVL(key.slice());
-        s.addBitString(commit);
+        // Encode the NodeID into sfAccount so handleAcquiredRngSet can
+        // recover it without recomputing (master vs signing key issue).
+        AccountID acctId;
+        std::memcpy(acctId.data(), nodeId.data(), acctId.size());
+
+        STTx tx(ttCONSENSUS_ENTROPY, [&](auto& obj) {
+            obj.setFieldU32(sfFlags, tfEntropyCommit);
+            obj.setFieldU32(sfLedgerSequence, seq);
+            obj.setAccountID(sfAccount, acctId);
+            obj.setFieldU32(sfSequence, 0);
+            obj.setFieldAmount(sfFee, STAmount{});
+            obj.setFieldH256(sfDigest, commit);
+            obj.setFieldVL(sfSigningPubKey, kit->second.slice());
+        });
+
+        Serializer s(2048);
+        tx.add(s);
+        map->addItem(
+            SHAMapNodeType::tnTRANSACTION_NM,
+            make_shamapitem(tx.getTransactionID(), s.slice()));
     }
-    return sha512Half(s.slice());
+
+    map = map->snapShot(false);
+    commitSetMap_ = map;
+
+    auto const hash = map->getHash().as_uint256();
+    inboundTransactions_.giveSet(hash, map, false);
+
+    JLOG(j_.debug()) << "RNG: built commitSet SHAMap hash=" << hash
+                     << " entries=" << pendingCommits_.size();
+    return hash;
 }
 
 uint256
-RCLConsensus::Adaptor::buildEntropySet()
+RCLConsensus::Adaptor::buildEntropySet(LedgerIndex seq)
 {
-    // Sort reveals deterministically by public key
-    std::vector<std::pair<PublicKey, uint256>> sorted;
+    auto map =
+        std::make_shared<SHAMap>(SHAMapType::TRANSACTION, app_.getNodeFamily());
+    map->setUnbacked();
+
     for (auto const& [nodeId, reveal] : pendingReveals_)
     {
-        auto it = nodeIdToKey_.find(nodeId);
-        if (it != nodeIdToKey_.end())
-            sorted.emplace_back(it->second, reveal);
-    }
-    std::sort(sorted.begin(), sorted.end(), [](auto const& a, auto const& b) {
-        return a.first < b.first;
-    });
+        auto kit = nodeIdToKey_.find(nodeId);
+        if (kit == nodeIdToKey_.end())
+            continue;
 
-    Serializer s;
-    for (auto const& [key, reveal] : sorted)
-    {
-        s.addVL(key.slice());
-        s.addBitString(reveal);
+        AccountID acctId;
+        std::memcpy(acctId.data(), nodeId.data(), acctId.size());
+
+        STTx tx(ttCONSENSUS_ENTROPY, [&](auto& obj) {
+            obj.setFieldU32(sfFlags, tfEntropyReveal);
+            obj.setFieldU32(sfLedgerSequence, seq);
+            obj.setAccountID(sfAccount, acctId);
+            obj.setFieldU32(sfSequence, 0);
+            obj.setFieldAmount(sfFee, STAmount{});
+            obj.setFieldH256(sfDigest, reveal);
+            obj.setFieldVL(sfSigningPubKey, kit->second.slice());
+        });
+
+        Serializer s(2048);
+        tx.add(s);
+        map->addItem(
+            SHAMapNodeType::tnTRANSACTION_NM,
+            make_shamapitem(tx.getTransactionID(), s.slice()));
     }
-    return sha512Half(s.slice());
+
+    map = map->snapShot(false);
+    entropySetMap_ = map;
+
+    auto const hash = map->getHash().as_uint256();
+    inboundTransactions_.giveSet(hash, map, false);
+
+    JLOG(j_.debug()) << "RNG: built entropySet SHAMap hash=" << hash
+                     << " entries=" << pendingReveals_.size();
+    return hash;
 }
 
 void
@@ -1231,6 +1275,169 @@ RCLConsensus::Adaptor::clearRngState()
     nodeIdToKey_.clear();
     myEntropySecret_ = uint256{};
     entropyFailed_ = false;
+    commitSetMap_.reset();
+    entropySetMap_.reset();
+    pendingRngFetches_.clear();
+}
+
+bool
+RCLConsensus::Adaptor::isRngSet(uint256 const& hash) const
+{
+    if (commitSetMap_ && commitSetMap_->getHash().as_uint256() == hash)
+        return true;
+    if (entropySetMap_ && entropySetMap_->getHash().as_uint256() == hash)
+        return true;
+    return pendingRngFetches_.count(hash) > 0;
+}
+
+void
+RCLConsensus::Adaptor::handleAcquiredRngSet(std::shared_ptr<SHAMap> const& map)
+{
+    auto const hash = map->getHash().as_uint256();
+    pendingRngFetches_.erase(hash);
+
+    JLOG(j_.debug()) << "RNG: handleAcquiredRngSet hash=" << hash;
+
+    // Determine if this is a commitSet or entropySet by inspecting entries
+    bool isCommitSet = false;
+    bool isEntropySet = false;
+
+    map->visitLeaves([&](boost::intrusive_ptr<SHAMapItem const> const& item) {
+        try
+        {
+            // Skip prefix (4 bytes) when deserializing
+            SerialIter sit(item->slice());
+            auto stx = std::make_shared<STTx const>(std::ref(sit));
+            auto flags = stx->getFieldU32(sfFlags);
+            if (flags & tfEntropyCommit)
+                isCommitSet = true;
+            else if (flags & tfEntropyReveal)
+                isEntropySet = true;
+        }
+        catch (std::exception const&)
+        {
+            // Skip malformed entries
+        }
+    });
+
+    if (!isCommitSet && !isEntropySet)
+    {
+        JLOG(j_.warn()) << "RNG: acquired set " << hash
+                        << " has no recognizable RNG entries";
+        return;
+    }
+
+    // Diff against our local set and merge missing entries
+    auto& localMap = isCommitSet ? commitSetMap_ : entropySetMap_;
+    auto& pendingData = isCommitSet ? pendingCommits_ : pendingReveals_;
+
+    std::size_t merged = 0;
+
+    if (localMap)
+    {
+        SHAMap::Delta delta;
+        localMap->compare(*map, delta, 65536);
+
+        for (auto const& [key, pair] : delta)
+        {
+            // pair.first = our entry, pair.second = their entry
+            // If we don't have it (pair.first is null), merge it
+            if (!pair.first && pair.second)
+            {
+                try
+                {
+                    SerialIter sit(pair.second->slice());
+                    auto stx = std::make_shared<STTx const>(std::ref(sit));
+
+                    auto pk = stx->getFieldVL(sfSigningPubKey);
+                    PublicKey pubKey(makeSlice(pk));
+                    auto digest = stx->getFieldH256(sfDigest);
+
+                    // Recover NodeID from sfAccount (encoded by
+                    // buildCommitSet/buildEntropySet) to avoid
+                    // master-vs-signing key mismatch.
+                    auto const acctId = stx->getAccountID(sfAccount);
+                    NodeID nodeId;
+                    std::memcpy(nodeId.data(), acctId.data(), nodeId.size());
+
+                    pendingData[nodeId] = digest;
+                    nodeIdToKey_[nodeId] = pubKey;
+                    ++merged;
+
+                    JLOG(j_.trace())
+                        << "RNG: merged " << (isCommitSet ? "commit" : "reveal")
+                        << " from " << nodeId;
+                }
+                catch (std::exception const& ex)
+                {
+                    JLOG(j_.warn())
+                        << "RNG: failed to parse entry from acquired set: "
+                        << ex.what();
+                }
+            }
+        }
+    }
+    else
+    {
+        // We don't have a local set yet — extract all entries
+        map->visitLeaves(
+            [&](boost::intrusive_ptr<SHAMapItem const> const& item) {
+                try
+                {
+                    SerialIter sit(item->slice());
+                    auto stx = std::make_shared<STTx const>(std::ref(sit));
+
+                    auto pk = stx->getFieldVL(sfSigningPubKey);
+                    PublicKey pubKey(makeSlice(pk));
+                    auto digest = stx->getFieldH256(sfDigest);
+
+                    auto const acctId = stx->getAccountID(sfAccount);
+                    NodeID nodeId;
+                    std::memcpy(nodeId.data(), acctId.data(), nodeId.size());
+
+                    pendingData[nodeId] = digest;
+                    nodeIdToKey_[nodeId] = pubKey;
+                    ++merged;
+                }
+                catch (std::exception const&)
+                {
+                    // Skip malformed entries
+                }
+            });
+    }
+
+    JLOG(j_.info()) << "RNG: merged " << merged << " entries from "
+                    << (isCommitSet ? "commitSet" : "entropySet")
+                    << " hash=" << hash;
+}
+
+void
+RCLConsensus::Adaptor::fetchRngSetIfNeeded(std::optional<uint256> const& hash)
+{
+    if (!hash || *hash == uint256{})
+        return;
+
+    // Check if we already have this set
+    if (commitSetMap_ && commitSetMap_->getHash().as_uint256() == *hash)
+        return;
+    if (entropySetMap_ && entropySetMap_->getHash().as_uint256() == *hash)
+        return;
+
+    // Check if already fetching
+    if (pendingRngFetches_.count(*hash))
+        return;
+
+    // Check if InboundTransactions already has it
+    if (auto existing = inboundTransactions_.getSet(*hash, false))
+    {
+        handleAcquiredRngSet(existing);
+        return;
+    }
+
+    // Trigger network fetch
+    JLOG(j_.debug()) << "RNG: triggering fetch for set " << *hash;
+    pendingRngFetches_.insert(*hash);
+    inboundTransactions_.getSet(*hash, true);
 }
 
 void

src/ripple/app/consensus/RCLConsensus.h

@@ -97,6 +97,13 @@ class RCLConsensus
         uint256 myEntropySecret_;
         bool entropyFailed_ = false;
 
+        // Real SHAMaps for the current round (unbacked, ephemeral)
+        std::shared_ptr<SHAMap> commitSetMap_;
+        std::shared_ptr<SHAMap> entropySetMap_;
+
+        // Track pending RNG set hashes we've triggered fetches for
+        hash_set<uint256> pendingRngFetches_;
+
     public:
         using Ledger_t = RCLCxLedger;
         using NodeID_t = NodeID;
@@ -207,13 +214,31 @@ class RCLConsensus
         bool
         hasAnyReveals() const;
 
-        /** Build deterministic hash of all collected commits */
+        /** Build real SHAMap from collected commits, register for fetch.
+            @param seq The ledger sequence being built
+            @return The SHAMap root hash (commitSetHash)
+        */
         uint256
-        buildCommitSet();
+        buildCommitSet(LedgerIndex seq);
 
-        /** Build deterministic hash of all collected reveals */
+        /** Build real SHAMap from collected reveals, register for fetch.
+            @param seq The ledger sequence being built
+            @return The SHAMap root hash (entropySetHash)
+        */
         uint256
-        buildEntropySet();
+        buildEntropySet(LedgerIndex seq);
+
+        /** Check if a hash is a known RNG set (commitSet or entropySet) */
+        bool
+        isRngSet(uint256 const& hash) const;
+
+        /** Handle an acquired RNG set — diff, merge missing entries */
+        void
+        handleAcquiredRngSet(std::shared_ptr<SHAMap> const& map);
+
+        /** Trigger fetch for a peer's unknown RNG set hash */
+        void
+        fetchRngSetIfNeeded(std::optional<uint256> const& hash);
 
         /** Generate new entropy secret for this round */
         void
@@ -562,6 +587,22 @@ public:
         return consensus_.inRngSubState();
     }
 
+    //! Check if a hash is a known RNG set (commitSet or entropySet)
+    bool
+    isRngSet(uint256 const& hash) const
+    {
+        std::lock_guard _{mutex_};
+        return adaptor_.isRngSet(hash);
+    }
+
+    //! Handle an acquired RNG set from InboundTransactions
+    void
+    gotRngSet(std::shared_ptr<SHAMap> const& map)
+    {
+        std::lock_guard _{mutex_};
+        adaptor_.handleAcquiredRngSet(map);
+    }
+
     //! @see Consensus::getJson
     Json::Value
     getJson(bool full) const;

src/ripple/app/ledger/ConsensusTransSetSF.cpp

@@ -26,6 +26,7 @@
 #include <ripple/core/JobQueue.h>
 #include <ripple/nodestore/Database.h>
 #include <ripple/protocol/HashPrefix.h>
+#include <ripple/protocol/STTx.h>
 #include <ripple/protocol/digest.h>
 
 namespace ripple {
@@ -61,6 +62,13 @@ ConsensusTransSetSF::gotNode(
             SerialIter sit(s.slice());
             auto stx = std::make_shared<STTx const>(std::ref(sit));
             assert(stx->getTransactionID() == nodeHash.as_uint256());
+
+            // Don't submit pseudo-transactions (consensus entropy, fees,
+            // amendments, etc.) — they exist as SHAMap entries for
+            // content-addressed identification but are not real user txns.
+            if (isPseudoTx(*stx))
+                return;
+
             auto const pap = &app_;
             app_.getJobQueue().addJob(jtTRANSACTION, "TXS->TXN", [pap, stx]() {
                 pap->getOPs().submitTransaction(stx);

src/ripple/app/misc/NetworkOPs.cpp

@@ -1882,7 +1882,17 @@ NetworkOPsImp::mapComplete(std::shared_ptr<SHAMap> const& map, bool fromAcquire)
 
     // We acquired it because consensus asked us to
     if (fromAcquire)
+    {
+        auto const hash = map->getHash().as_uint256();
+        if (mConsensus.isRngSet(hash))
+        {
+            // RNG set (commitSet or entropySet) — route to adaptor
+            // for diff/merge, not into txSet consensus machinery.
+            mConsensus.gotRngSet(map);
+            return;
+        }
         mConsensus.gotTxSet(app_.timeKeeper().closeTime(), RCLTxSet{map});
+    }
 }
 
 void

src/ripple/consensus/Consensus.h

@@ -832,6 +832,15 @@ Consensus<Adaptor>::peerProposalInternal(
                          << (newPeerProp.position().myReveal ? "yes" : "no");
         adaptor_.harvestRngData(
             peerID, newPeerPos.publicKey(), newPeerProp.position());
+
+        // Trigger fetch for unknown RNG set hashes
+        if constexpr (requires(Adaptor & a) {
+                          a.fetchRngSetIfNeeded(std::optional<uint256>{});
+                      })
+        {
+            adaptor_.fetchRngSetIfNeeded(newPeerProp.position().commitSetHash);
+            adaptor_.fetchRngSetIfNeeded(newPeerProp.position().entropySetHash);
+        }
     }
 
     if (newPeerProp.isInitial())
@@ -1344,10 +1353,12 @@ Consensus<Adaptor>::phaseEstablish()
     // --- RNG Sub-state Checkpoints (if adaptor supports RNG) ---
     if constexpr (requires(Adaptor & a) {
                       a.hasQuorumOfCommits();
-                      a.buildCommitSet();
+                      a.buildCommitSet(typename Ledger_t::Seq{});
                       a.generateEntropySecret();
                   })
     {
+        auto const buildSeq = previousLedger_.seq() + typename Ledger_t::Seq{1};
+
         JLOG(j_.debug()) << "RNG: phaseEstablish estState="
                          << static_cast<int>(estState_);
 
@@ -1355,7 +1366,7 @@ Consensus<Adaptor>::phaseEstablish()
         {
             if (adaptor_.hasQuorumOfCommits())
             {
-                auto commitSetHash = adaptor_.buildCommitSet();
+                auto commitSetHash = adaptor_.buildCommitSet(buildSeq);
 
                 // Keep the same entropy secret from onClose() — do NOT
                 // regenerate.  The commitment in the commitSet was built
@@ -1408,7 +1419,7 @@ Consensus<Adaptor>::phaseEstablish()
                 }
                 else
                 {
-                    auto entropySetHash = adaptor_.buildEntropySet();
+                    auto entropySetHash = adaptor_.buildEntropySet(buildSeq);
                     auto newPos = result_->position.position();
                     newPos.entropySetHash = entropySetHash;
 

src/ripple/protocol/TxFlags.h

@@ -206,6 +206,13 @@ enum CronSetFlags : uint32_t {
 };
 constexpr std::uint32_t const tfCronSetMask = ~(tfUniversal | tfCronUnset);
 
+// ConsensusEntropy flags (used on ttCONSENSUS_ENTROPY SHAMap entries):
+enum ConsensusEntropyFlags : uint32_t {
+    tfEntropyCommit = 0x00000001,  // entry is a commitment in commitSet
+    tfEntropyReveal = 0x00000002,  // entry is a reveal in entropySet
+};
+// flag=0 (no tfEntropyCommit/tfEntropyReveal) = final injected pseudo-tx
+
 // clang-format on
 
 }  // namespace ripple