Implement side chain federators:

Co-authored-by: seelabs <scott.determan@yahoo.com>
Co-authored-by: Peng Wang <pwang200@gmail.com>

Builds/CMake/RippledCore.cmake

@@ -407,6 +407,17 @@ target_sources (rippled PRIVATE
   src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp
   src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp
   src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp
+  src/ripple/app/sidechain/Federator.cpp
+  src/ripple/app/sidechain/FederatorEvents.cpp
+  src/ripple/app/sidechain/impl/ChainListener.cpp
+  src/ripple/app/sidechain/impl/DoorKeeper.cpp
+  src/ripple/app/sidechain/impl/InitialSync.cpp
+  src/ripple/app/sidechain/impl/MainchainListener.cpp
+  src/ripple/app/sidechain/impl/SidechainListener.cpp
+  src/ripple/app/sidechain/impl/SignatureCollector.cpp
+  src/ripple/app/sidechain/impl/SignerList.cpp
+  src/ripple/app/sidechain/impl/TicketHolder.cpp
+  src/ripple/app/sidechain/impl/WebsocketClient.cpp
   src/ripple/app/tx/impl/ApplyContext.cpp
   src/ripple/app/tx/impl/BookTip.cpp
   src/ripple/app/tx/impl/CancelCheck.cpp
@@ -576,6 +587,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/DepositAuthorized.cpp
   src/ripple/rpc/handlers/DownloadShard.cpp
   src/ripple/rpc/handlers/Feature1.cpp
+  src/ripple/rpc/handlers/FederatorInfo.cpp
   src/ripple/rpc/handlers/Fee1.cpp
   src/ripple/rpc/handlers/FetchInfo.cpp
   src/ripple/rpc/handlers/GatewayBalances.cpp

Builds/levelization/results/loops.txt

@@ -11,7 +11,7 @@ Loop: ripple.app ripple.nodestore
   ripple.app > ripple.nodestore
 
 Loop: ripple.app ripple.overlay
-  ripple.overlay ~= ripple.app
+  ripple.overlay == ripple.app
 
 Loop: ripple.app ripple.peerfinder
   ripple.peerfinder ~= ripple.app

Builds/levelization/results/ordering.txt

@@ -6,6 +6,7 @@ ripple.app > ripple.crypto
 ripple.app > ripple.json
 ripple.app > ripple.protocol
 ripple.app > ripple.resource
+ripple.app > ripple.server
 ripple.app > test.unit_test
 ripple.basics > ripple.beast
 ripple.conditions > ripple.basics

CMakeLists.txt

@@ -21,6 +21,12 @@ if(Git_FOUND)
     endif()
 endif() #git
 
+if (thread_safety_analysis)
+  add_compile_options(-Wthread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS -DRIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS)
+  add_compile_options("-stdlib=libc++")
+  add_link_options("-stdlib=libc++")
+endif()
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps")
 

src/ripple/app/main/Application.cpp

@@ -48,6 +48,7 @@
 #include <ripple/app/rdb/RelationalDBInterface_global.h>
 #include <ripple/app/rdb/backend/RelationalDBInterfacePostgres.h>
 #include <ripple/app/reporting/ReportingETL.h>
+#include <ripple/app/sidechain/Federator.h>
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/ByteUtilities.h>
 #include <ripple/basics/PerfLog.h>
@@ -215,6 +216,8 @@ public:
     RCLValidations mValidations;
     std::unique_ptr<LoadManager> m_loadManager;
     std::unique_ptr<TxQ> txQ_;
+
+    std::shared_ptr<sidechain::Federator> sidechainFederator_;
     ClosureCounter<void, boost::system::error_code const&> waitHandlerCounter_;
     boost::asio::steady_timer sweepTimer_;
     boost::asio::steady_timer entropyTimer_;
@@ -519,6 +522,8 @@ public:
     checkSigs(bool) override;
     bool
     isStopping() const override;
+    void
+    startFederator() override;
     int
     fdRequired() const override;
 
@@ -889,6 +894,12 @@ public:
         return *mWalletDB;
     }
 
+    std::shared_ptr<sidechain::Federator>
+    getSidechainFederator() override
+    {
+        return sidechainFederator_;
+    }
+
     ReportingETL&
     getReportingETL() override
     {
@@ -1057,6 +1068,8 @@ public:
         ledgerCleaner_->stop();
         if (reportingETL_)
             reportingETL_->stop();
+        if (sidechainFederator_)
+            sidechainFederator_->stop();
         if (auto pg = dynamic_cast<RelationalDBInterfacePostgres*>(
                 &*mRelationalDBInterface))
             pg->stop();
@@ -1162,7 +1175,9 @@ public:
         getInboundLedgers().sweep();
         getLedgerReplayer().sweep();
         m_acceptedLedgerCache.sweep();
-        cachedSLEs_.sweep();
+        cachedSLEs_.expire();
+        if (sidechainFederator_)
+            sidechainFederator_->sweep();
 
 #ifdef RIPPLED_REPORTING
         if (auto pg = dynamic_cast<RelationalDBInterfacePostgres*>(
@@ -1597,6 +1612,15 @@ ApplicationImp::setup()
     if (reportingETL_)
         reportingETL_->start();
 
+    sidechainFederator_ = sidechain::make_Federator(
+        *this,
+        get_io_service(),
+        *config_,
+        logs_->journal("SidechainFederator"));
+
+    if (sidechainFederator_)
+        sidechainFederator_->start();
+
     return true;
 }
 
@@ -1621,6 +1645,7 @@ ApplicationImp::start(bool withTimers)
     grpcServer_->start();
     ledgerCleaner_->start();
     perfLog_->start();
+    startFederator();
 }
 
 void
@@ -1678,6 +1703,13 @@ ApplicationImp::isStopping() const
     return isTimeToStop;
 }
 
+void
+ApplicationImp::startFederator()
+{
+    if (sidechainFederator_)
+        sidechainFederator_->unlockMainLoop();
+}
+
 int
 ApplicationImp::fdRequired() const
 {

src/ripple/app/main/Application.h

@@ -50,6 +50,10 @@ namespace RPC {
 class ShardArchiveHandler;
 }
 
+namespace sidechain {
+class Federator;
+}
+
 // VFALCO TODO Fix forward declares required for header dependency loops
 class AmendmentTable;
 
@@ -150,6 +154,9 @@ public:
     virtual bool
     isStopping() const = 0;
 
+    virtual void
+    startFederator() = 0;
+
     //
     // ---
     //
@@ -274,6 +281,9 @@ public:
     virtual DatabaseCon&
     getWalletDB() = 0;
 
+    virtual std::shared_ptr<sidechain::Federator>
+    getSidechainFederator() = 0;
+
     /** Ensure that a newly-started validator does not sign proposals older
      * than the last ledger it persisted. */
     virtual LedgerIndex

src/ripple/app/main/Main.cpp

@@ -20,6 +20,7 @@
 #include <ripple/app/main/Application.h>
 #include <ripple/app/main/DBInit.h>
 #include <ripple/app/rdb/RelationalDBInterface_global.h>
+#include <ripple/app/sidechain/Federator.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/basics/contract.h>

src/ripple/app/sidechain/Federator.h

@@ -0,0 +1,418 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_FEDERATOR_H_INCLUDED
+#define RIPPLE_SIDECHAIN_FEDERATOR_H_INCLUDED
+
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/app/sidechain/impl/DoorKeeper.h>
+#include <ripple/app/sidechain/impl/MainchainListener.h>
+#include <ripple/app/sidechain/impl/SidechainListener.h>
+#include <ripple/app/sidechain/impl/SignatureCollector.h>
+#include <ripple/app/sidechain/impl/SignerList.h>
+#include <ripple/app/sidechain/impl/TicketHolder.h>
+#include <ripple/basics/Buffer.h>
+#include <ripple/basics/ThreadSaftyAnalysis.h>
+#include <ripple/basics/UnorderedContainers.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/core/Config.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Quality.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/SecretKey.h>
+
+#include <boost/container/flat_map.hpp>
+
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <optional>
+#include <thread>
+#include <unordered_map>
+#include <variant>
+#include <vector>
+
+namespace ripple {
+
+class Application;
+class STTx;
+
+namespace sidechain {
+
+class Federator : public std::enable_shared_from_this<Federator>
+{
+public:
+    enum ChainType { sideChain, mainChain };
+
+    // These enums are encoded in the transaction. Changing the order will break
+    // backward compatibility. If a new type is added change txnTypeLast.
+    enum class TxnType { xChain, refund };
+    constexpr static std::uint8_t txnTypeLast = 2;
+
+    constexpr static size_t numChains = 2;
+
+    static constexpr std::uint32_t accountControlTxFee{1000};
+
+private:
+    // Tag so make_Federator can call `std::make_shared`
+    class PrivateTag
+    {
+    };
+
+    friend std::shared_ptr<Federator>
+    make_Federator(
+        Application& app,
+        boost::asio::io_service& ios,
+        BasicConfig const& config,
+        beast::Journal j);
+
+    std::thread thread_;
+    bool running_ = false;
+    std::atomic<bool> requestStop_ = false;
+
+    Application& app_;
+    std::array<AccountID, numChains> const account_;
+    std::array<std::atomic<std::uint32_t>, numChains> accountSeq_{1, 1};
+    std::array<std::atomic<std::uint32_t>, numChains> lastTxnSeqSent_{0, 0};
+    std::array<std::atomic<std::uint32_t>, numChains> lastTxnSeqConfirmed_{
+        0,
+        0};
+    std::shared_ptr<MainchainListener> mainchainListener_;
+    std::shared_ptr<SidechainListener> sidechainListener_;
+
+    mutable std::mutex eventsMutex_;
+    std::vector<FederatorEvent> GUARDED_BY(eventsMutex_) events_;
+
+    // When a user account sends an asset to the account controlled by the
+    // federator, the asset to be issued on the other chain is determined by the
+    // `assetProps` maps - one for each chain. The asset to be issued is
+    // `issue`, the amount of the asset to issue is determined by `quality`
+    // (ratio of output amount/input amount). When issuing refunds, the
+    // `refundPenalty` is subtracted from the sent amount before sending the
+    // refund.
+    struct OtherChainAssetProperties
+    {
+        Quality quality;
+        Issue issue;
+        STAmount refundPenalty;
+    };
+
+    std::array<
+        boost::container::flat_map<Issue, OtherChainAssetProperties>,
+        numChains> const assetProps_;
+
+    PublicKey signingPK_;
+    SecretKey signingSK_;
+
+    // federator signing public keys
+    mutable std::mutex federatorPKsMutex_;
+    hash_set<PublicKey> GUARDED_BY(federatorPKsMutex_) federatorPKs_;
+
+    SignerList mainSignerList_;
+    SignerList sideSignerList_;
+    SignatureCollector mainSigCollector_;
+    SignatureCollector sideSigCollector_;
+    TicketRunner ticketRunner_;
+    DoorKeeper mainDoorKeeper_;
+    DoorKeeper sideDoorKeeper_;
+
+    struct PeerTxnSignature
+    {
+        Buffer sig;
+        std::uint32_t seq;
+    };
+
+    struct SequenceInfo
+    {
+        // Number of signatures at this sequence number
+        std::uint32_t count{0};
+        // Serialization of the transaction for everything except the signature
+        // id (which varies for each signature). This can be used to verify one
+        // of the signatures in a multisig.
+        Blob partialTxnSerialization;
+    };
+
+    struct PendingTransaction
+    {
+        STAmount amount;
+        AccountID srcChainSrcAccount;
+        AccountID dstChainDstAccount;
+        // Key is the federator's public key
+        hash_map<PublicKey, PeerTxnSignature> sigs;
+        // Key is a sequence number
+        hash_map<std::uint32_t, SequenceInfo> sequenceInfo;
+        // True if the transaction was ever put into the toSendTxns_ queue
+        bool queuedToSend_{false};
+    };
+
+    // Key is the hash of the triggering transaction
+    mutable std::mutex pendingTxnsM_;
+    hash_map<uint256, PendingTransaction> GUARDED_BY(pendingTxnsM_)
+        pendingTxns_[numChains];
+
+    mutable std::mutex toSendTxnsM_;
+    // Signed transactions ready to send
+    // Key is the transaction's sequence number. The transactions must be sent
+    // in the correct order. If the next trasnaction the account needs to send
+    // has a sequence number of N, the transaction with sequence N+1 can't be
+    // sent just because it collected signatures first.
+    std::map<std::uint32_t, STTx> GUARDED_BY(toSendTxnsM_)
+        toSendTxns_[numChains];
+    std::set<std::uint32_t> GUARDED_BY(toSendTxnsM_) toSkipSeq_[numChains];
+
+    // Use a condition variable to prevent busy waiting when the queue is
+    // empty
+    mutable std::mutex m_;
+    std::condition_variable cv_;
+
+    // prevent the main loop from starting until explictly told to run.
+    // This is used to allow bootstrap code to run before any events are
+    // processed
+    mutable std::mutex mainLoopMutex_;
+    bool mainLoopLocked_{true};
+    std::condition_variable mainLoopCv_;
+    beast::Journal j_;
+
+    static std::array<
+        boost::container::flat_map<Issue, OtherChainAssetProperties>,
+        numChains>
+    makeAssetProps(BasicConfig const& config, beast::Journal j);
+
+public:
+    // Constructor should be private, but needs to be public so
+    // `make_shared` can use it
+    Federator(
+        PrivateTag,
+        Application& app,
+        SecretKey signingKey,
+        hash_set<PublicKey>&& federators,
+        boost::asio::ip::address mainChainIp,
+        std::uint16_t mainChainPort,
+        AccountID const& mainAccount,
+        AccountID const& sideAccount,
+        std::array<
+            boost::container::flat_map<Issue, OtherChainAssetProperties>,
+            numChains>&& assetProps,
+        beast::Journal j);
+
+    ~Federator();
+
+    void
+    start();
+
+    void
+    stop() EXCLUDES(m_);
+
+    void
+    push(FederatorEvent&& e) EXCLUDES(m_, eventsMutex_);
+
+    // Don't process any events until the bootstrap has a chance to run
+    void
+    unlockMainLoop() EXCLUDES(m_);
+
+    void
+    addPendingTxnSig(
+        TxnType txnType,
+        ChainType chaintype,
+        PublicKey const& federatorPk,
+        uint256 const& srcChainTxnHash,
+        std::optional<uint256> const& dstChainTxnHash,
+        STAmount const& amt,
+        AccountID const& srcChainSrcAccount,
+        AccountID const& dstChainDstAccount,
+        std::uint32_t seq,
+        Buffer&& sig) EXCLUDES(federatorPKsMutex_, pendingTxnsM_, toSendTxnsM_);
+
+    void
+    addPendingTxnSig(
+        ChainType chaintype,
+        PublicKey const& publicKey,
+        uint256 const& mId,
+        Buffer&& sig);
+
+    // Return true if a transaction with this sequence has already been sent
+    bool
+    alreadySent(ChainType chaintype, std::uint32_t seq) const;
+
+    void
+    setLastXChainTxnWithResult(
+        ChainType chaintype,
+        std::uint32_t seq,
+        std::uint32_t seqTook,
+        uint256 const& hash);
+    void
+    setNoLastXChainTxnWithResult(ChainType chaintype);
+    void
+    stopHistoricalTxns(ChainType chaintype);
+    void
+    initialSyncDone(ChainType chaintype);
+
+    // Get stats on the federator, including pending transactions and
+    // initialization state
+    Json::Value
+    getInfo() const EXCLUDES(pendingTxnsM_);
+
+    void
+    sweep();
+
+    SignatureCollector&
+    getSignatureCollector(ChainType chain);
+    DoorKeeper&
+    getDoorKeeper(ChainType chain);
+    TicketRunner&
+    getTicketRunner();
+    void
+    addSeqToSkip(ChainType chain, std::uint32_t seq) EXCLUDES(toSendTxnsM_);
+    // TODO multi-sig refactor?
+    void
+    addTxToSend(ChainType chain, std::uint32_t seq, STTx const& tx)
+        EXCLUDES(toSendTxnsM_);
+
+private:
+    // Two phase init needed for shared_from this.
+    // Only called from `make_Federator`
+    void
+    init(
+        boost::asio::io_service& ios,
+        boost::asio::ip::address& ip,
+        std::uint16_t port,
+        std::shared_ptr<MainchainListener>&& mainchainListener,
+        std::shared_ptr<SidechainListener>&& sidechainListener);
+
+    // Convert between the asset on the src chain to the asset on the other
+    // chain. The `assetProps_` array controls how this conversion is done.
+    // An empty option is returned if the from issue is not part of the map in
+    // the `assetProps_` array.
+    [[nodiscard]] std::optional<STAmount>
+    toOtherChainAmount(ChainType srcChain, STAmount const& from) const;
+
+    // Set the accountSeq to the max of the current value and the requested
+    // value. This is done with a lock free algorithm.
+    void
+    setAccountSeqMax(ChainType chaintype, std::uint32_t reqValue);
+
+    // Set the lastTxnSeqSent to the max of the current value and the requested
+    // value. This is done with a lock free algorithm.
+    void
+    setLastTxnSeqSentMax(ChainType chaintype, std::uint32_t reqValue);
+    void
+    setLastTxnSeqConfirmedMax(ChainType chaintype, std::uint32_t reqValue);
+
+    mutable std::mutex sendTxnsMutex_;
+    void
+    sendTxns() EXCLUDES(sendTxnsMutex_, toSendTxnsM_);
+
+    void
+    mainLoop() EXCLUDES(mainLoopMutex_);
+
+    void
+    payTxn(
+        TxnType txnType,
+        ChainType dstChain,
+        STAmount const& amt,
+        // srcChainSrcAccount is the origional sending account in a cross chain
+        // transaction. Note, for refunds, the srcChainSrcAccount and the dst
+        // will be the same.
+        AccountID const& srcChainSrcAccount,
+        AccountID const& dst,
+        uint256 const& srcChainTxnHash,
+        std::optional<uint256> const& dstChainTxnHash);
+
+    // Issue a refund to the destination account. Refunds may be issued when a
+    // cross chain transaction fails on the destination chain. In this case, the
+    // funds will already be locked on one chain, but can not be completed on
+    // the other chain. Note that refunds may not be for the full amount sent.
+    // In effect, not refunding the full amount charges a fee to discourage
+    // abusing refunds to try to overload the system.
+    void
+    sendRefund(
+        ChainType chaintype,
+        STAmount const& amt,
+        AccountID const& dst,
+        uint256 const& txnHash,
+        uint256 const& triggeringResultTxnHash);
+
+    void
+    onEvent(event::XChainTransferDetected const& e);
+    void
+    onEvent(event::XChainTransferResult const& e) EXCLUDES(pendingTxnsM_);
+    void
+    onEvent(event::RefundTransferResult const& e) EXCLUDES(pendingTxnsM_);
+    void
+    onEvent(event::HeartbeatTimer const& e);
+    void
+    onEvent(event::StartOfHistoricTransactions const& e);
+    void
+    onEvent(event::TicketCreateTrigger const& e);
+    void
+    onEvent(event::TicketCreateResult const& e);
+    void
+    onEvent(event::DepositAuthResult const& e);
+    void
+    onEvent(event::BootstrapTicket const& e);
+    void
+    onEvent(event::DisableMasterKeyResult const& e);
+    //    void
+    //    onEvent(event::SignerListSetResult const& e);
+
+    void
+    updateDoorKeeper(ChainType chainType) EXCLUDES(pendingTxnsM_);
+    void
+    onResult(ChainType chainType, std::uint32_t resultTxSeq);
+};
+
+[[nodiscard]] std::shared_ptr<Federator>
+make_Federator(
+    Application& app,
+    boost::asio::io_service& ios,
+    BasicConfig const& config,
+    beast::Journal j);
+
+// Id used for message suppression
+[[nodiscard]] uint256
+crossChainTxnSignatureId(
+    PublicKey signingPK,
+    uint256 const& srcChainTxnHash,
+    std::optional<uint256> const& dstChainTxnHash,
+    STAmount const& amt,
+    AccountID const& src,
+    AccountID const& dst,
+    std::uint32_t seq,
+    Slice const& signature);
+
+[[nodiscard]] Federator::ChainType
+srcChainType(event::Dir dir);
+
+[[nodiscard]] Federator::ChainType
+dstChainType(event::Dir dir);
+
+[[nodiscard]] Federator::ChainType
+otherChainType(Federator::ChainType ct);
+
+[[nodiscard]] Federator::ChainType
+getChainType(bool isMainchain);
+
+uint256
+computeMessageSuppression(uint256 const& mId, Slice const& signature);
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/FederatorEvents.cpp

@@ -0,0 +1,340 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/FederatorEvents.h>
+
+#include <string_view>
+#include <type_traits>
+
+namespace ripple {
+namespace sidechain {
+namespace event {
+
+namespace {
+
+std::string const&
+to_string(Dir dir)
+{
+    switch (dir)
+    {
+        case Dir::mainToSide: {
+            static std::string const r("main");
+            return r;
+        }
+        case Dir::sideToMain: {
+            static std::string const r("side");
+            return r;
+        }
+    }
+
+    // Some compilers will warn about not returning from all control paths
+    // without this, but this code will never execute.
+    assert(0);
+    static std::string const error("error");
+    return error;
+}
+
+std::string const&
+to_string(AccountFlagOp op)
+{
+    switch (op)
+    {
+        case AccountFlagOp::set: {
+            static std::string const r("set");
+            return r;
+        }
+        case AccountFlagOp::clear: {
+            static std::string const r("clear");
+            return r;
+        }
+    }
+
+    // Some compilers will warn about not returning from all control paths
+    // without this, but this code will never execute.
+    assert(0);
+    static std::string const error("error");
+    return error;
+}
+
+}  // namespace
+
+EventType
+XChainTransferDetected::eventType() const
+{
+    return EventType::trigger;
+}
+
+Json::Value
+XChainTransferDetected::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "XChainTransferDetected";
+    result["src"] = toBase58(src_);
+    result["dst"] = toBase58(dst_);
+    result["deliveredAmt"] = deliveredAmt_.getJson(JsonOptions::none);
+    result["txnSeq"] = txnSeq_;
+    result["txnHash"] = to_string(txnHash_);
+    result["rpcOrder"] = rpcOrder_;
+    return result;
+}
+
+EventType
+HeartbeatTimer::eventType() const
+{
+    return EventType::heartbeat;
+}
+
+Json::Value
+HeartbeatTimer::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "HeartbeatTimer";
+    return result;
+}
+
+EventType
+XChainTransferResult::eventType() const
+{
+    return EventType::result;
+}
+
+Json::Value
+XChainTransferResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "XChainTransferResult";
+    result["dir"] = to_string(dir_);
+    result["dst"] = toBase58(dst_);
+    if (deliveredAmt_)
+        result["deliveredAmt"] = deliveredAmt_->getJson(JsonOptions::none);
+    result["txnSeq"] = txnSeq_;
+    result["srcChainTxnHash"] = to_string(srcChainTxnHash_);
+    result["txnHash"] = to_string(txnHash_);
+    result["ter"] = transHuman(ter_);
+    result["rpcOrder"] = rpcOrder_;
+    return result;
+}
+
+EventType
+RefundTransferResult::eventType() const
+{
+    return EventType::result;
+}
+
+Json::Value
+RefundTransferResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "RefundTransferResult";
+    result["dir"] = to_string(dir_);
+    result["dst"] = toBase58(dst_);
+    if (deliveredAmt_)
+        result["deliveredAmt"] = deliveredAmt_->getJson(JsonOptions::none);
+    result["txnSeq"] = txnSeq_;
+    result["srcChainTxnHash"] = to_string(srcChainTxnHash_);
+    result["dstChainTxnHash"] = to_string(dstChainTxnHash_);
+    result["txnHash"] = to_string(txnHash_);
+    result["ter"] = transHuman(ter_);
+    result["rpcOrder"] = rpcOrder_;
+    return result;
+}
+
+EventType
+StartOfHistoricTransactions::eventType() const
+{
+    return EventType::startOfTransactions;
+}
+
+Json::Value
+StartOfHistoricTransactions::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "StartOfHistoricTransactions";
+    result["isMainchain"] = isMainchain_;
+    return result;
+}
+
+EventType
+TicketCreateTrigger::eventType() const
+{
+    return EventType::trigger;
+}
+
+Json::Value
+TicketCreateTrigger::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "TicketCreateTrigger";
+    result["dir"] = to_string(dir_);
+    result["success"] = success_;
+    result["txnSeq"] = txnSeq_;
+    result["ledgerIndex"] = ledgerIndex_;
+    result["txnHash"] = to_string(txnHash_);
+    result["rpcOrder"] = rpcOrder_;
+    result["sourceTag"] = sourceTag_;
+    result["memo"] = memoStr_;
+    return result;
+}
+
+EventType
+TicketCreateResult::eventType() const
+{
+    return EventType::resultAndTrigger;
+}
+
+Json::Value
+TicketCreateResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "TicketCreateResult";
+    result["dir"] = to_string(dir_);
+    result["success"] = success_;
+    result["txnSeq"] = txnSeq_;
+    result["ledgerIndex"] = ledgerIndex_;
+    result["srcChainTxnHash"] = to_string(srcChainTxnHash_);
+    result["txnHash"] = to_string(txnHash_);
+    result["rpcOrder"] = rpcOrder_;
+    result["sourceTag"] = sourceTag_;
+    result["memo"] = memoStr_;
+    return result;
+}
+
+void
+TicketCreateResult::removeTrigger()
+{
+    memoStr_.clear();
+}
+
+EventType
+DepositAuthResult::eventType() const
+{
+    return EventType::result;
+}
+
+Json::Value
+DepositAuthResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "DepositAuthResult";
+    result["dir"] = to_string(dir_);
+    result["success"] = success_;
+    result["txnSeq"] = txnSeq_;
+    result["ledgerIndex"] = ledgerIndex_;
+    result["srcChainTxnHash"] = to_string(srcChainTxnHash_);
+    result["rpcOrder"] = rpcOrder_;
+    result["op"] = to_string(op_);
+    return result;
+}
+
+EventType
+SignerListSetResult::eventType() const
+{
+    return EventType::result;
+}
+
+Json::Value
+SignerListSetResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "SignerListSetResult";
+    return result;
+}
+EventType
+BootstrapTicket::eventType() const
+{
+    return EventType::bootstrap;
+}
+
+Json::Value
+BootstrapTicket::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "BootstrapTicket";
+    result["isMainchain"] = isMainchain_;
+    result["txnSeq"] = txnSeq_;
+    result["rpcOrder"] = rpcOrder_;
+    return result;
+}
+EventType
+DisableMasterKeyResult::eventType() const
+{
+    return EventType::result;  // TODO change to bootstrap type too?
+}
+
+Json::Value
+DisableMasterKeyResult::toJson() const
+{
+    Json::Value result{Json::objectValue};
+    result["eventType"] = "DisableMasterKeyResult";
+    result["isMainchain"] = isMainchain_;
+    result["txnSeq"] = txnSeq_;
+    result["rpcOrder"] = rpcOrder_;
+    return result;
+}
+
+}  // namespace event
+
+namespace {
+template <typename T, typename = void>
+struct hasTxnHash : std::false_type
+{
+};
+template <typename T>
+struct hasTxnHash<T, std::void_t<decltype(std::declval<T>().txnHash_)>>
+    : std::true_type
+{
+};
+template <class T>
+inline constexpr bool hasTxnHash_v = hasTxnHash<T>::value;
+
+// Check that the traits work as expected
+static_assert(
+    hasTxnHash_v<event::XChainTransferResult> &&
+        !hasTxnHash_v<event::HeartbeatTimer>,
+    "");
+}  // namespace
+
+std::optional<uint256>
+txnHash(FederatorEvent const& event)
+{
+    return std::visit(
+        [](auto const& e) -> std::optional<uint256> {
+            if constexpr (hasTxnHash_v<std::decay_t<decltype(e)>>)
+            {
+                return e.txnHash_;
+            }
+            return std::nullopt;
+        },
+        event);
+}
+
+event::EventType
+eventType(FederatorEvent const& event)
+{
+    return std::visit([](auto const& e) { return e.eventType(); }, event);
+}
+
+Json::Value
+toJson(FederatorEvent const& event)
+{
+    return std::visit([](auto const& e) { return e.toJson(); }, event);
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/FederatorEvents.h

@@ -0,0 +1,277 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_FEDERATOR_EVENTS_H_INCLUDED
+#define RIPPLE_SIDECHAIN_FEDERATOR_EVENTS_H_INCLUDED
+
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Issue.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/TER.h>
+#include <beast/utility/Journal.h>
+
+#include <boost/format.hpp>
+
+#include <optional>
+#include <sstream>
+#include <string>
+#include <variant>
+
+namespace ripple {
+namespace sidechain {
+namespace event {
+
+enum class Dir { sideToMain, mainToSide };
+enum class AccountFlagOp { set, clear };
+static constexpr std::uint32_t MemoStringMax = 512;
+
+enum class EventType {
+    bootstrap,
+    trigger,
+    result,
+    resultAndTrigger,
+    heartbeat,
+    startOfTransactions
+};
+
+// A cross chain transfer was detected on this federator
+struct XChainTransferDetected
+{
+    // direction of the transfer
+    Dir dir_;
+    // Src account on the src chain
+    AccountID src_;
+    // Dst account on the dst chain
+    AccountID dst_;
+    STAmount deliveredAmt_;
+    std::uint32_t txnSeq_;
+    uint256 txnHash_;
+    std::int32_t rpcOrder_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct HeartbeatTimer
+{
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct XChainTransferResult
+{
+    // direction is the direction of the triggering transaction.
+    // I.e. A "mainToSide" transfer result is a transaction that
+    // happens on the sidechain (the triggering transaction happended on the
+    // mainchain)
+    Dir dir_;
+    AccountID dst_;
+    std::optional<STAmount> deliveredAmt_;
+    std::uint32_t txnSeq_;
+    // Txn hash of the initiating xchain transaction
+    uint256 srcChainTxnHash_;
+    // Txn has of the federator's transaction on the dst chain
+    uint256 txnHash_;
+    TER ter_;
+    std::int32_t rpcOrder_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct RefundTransferResult
+{
+    // direction is the direction of the triggering transaction.
+    // I.e. A "mainToSide" refund transfer result is a transaction that
+    // happens on the mainchain (the triggering transaction happended on the
+    // mainchain, the failed result happened on the side chain, and the refund
+    // result happened on the mainchain)
+    Dir dir_;
+    AccountID dst_;
+    std::optional<STAmount> deliveredAmt_;
+    std::uint32_t txnSeq_;
+    // Txn hash of the initiating xchain transaction
+    uint256 srcChainTxnHash_;
+    // Txn hash of the federator's transaction on the dst chain
+    uint256 dstChainTxnHash_;
+    // Txn hash of the refund result
+    uint256 txnHash_;
+    TER ter_;
+    std::int32_t rpcOrder_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+// The start of historic transactions has been reached
+struct StartOfHistoricTransactions
+{
+    bool isMainchain_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct TicketCreateTrigger
+{
+    Dir dir_;
+    bool success_;
+    std::uint32_t txnSeq_;
+    std::uint32_t ledgerIndex_;
+    uint256 txnHash_;
+    std::int32_t rpcOrder_;
+
+    std::uint32_t sourceTag_;
+    std::string memoStr_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct TicketCreateResult
+{
+    Dir dir_;
+    bool success_;
+    std::uint32_t txnSeq_;
+    std::uint32_t ledgerIndex_;
+    uint256 srcChainTxnHash_;
+    uint256 txnHash_;
+    std::int32_t rpcOrder_;
+
+    std::uint32_t sourceTag_;
+    std::string memoStr_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+
+    void
+    removeTrigger();
+};
+
+struct DepositAuthResult
+{
+    Dir dir_;
+    bool success_;
+    std::uint32_t txnSeq_;
+    std::uint32_t ledgerIndex_;
+    uint256 srcChainTxnHash_;
+    std::int32_t rpcOrder_;
+
+    AccountFlagOp op_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct SignerListSetResult
+{
+    // TODO
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct BootstrapTicket
+{
+    bool isMainchain_;
+    bool success_;
+    std::uint32_t txnSeq_;
+    std::uint32_t ledgerIndex_;
+    std::int32_t rpcOrder_;
+
+    std::uint32_t sourceTag_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+struct DisableMasterKeyResult
+{
+    bool isMainchain_;
+    std::uint32_t txnSeq_;
+    std::int32_t rpcOrder_;
+
+    EventType
+    eventType() const;
+
+    Json::Value
+    toJson() const;
+};
+
+}  // namespace event
+
+using FederatorEvent = std::variant<
+    event::XChainTransferDetected,
+    event::HeartbeatTimer,
+    event::XChainTransferResult,
+    event::RefundTransferResult,
+    event::StartOfHistoricTransactions,
+    event::TicketCreateTrigger,
+    event::TicketCreateResult,
+    event::DepositAuthResult,
+    event::BootstrapTicket,
+    event::DisableMasterKeyResult>;
+
+event::EventType
+eventType(FederatorEvent const& event);
+
+Json::Value
+toJson(FederatorEvent const& event);
+
+// If the event has a txnHash_ field (all the trigger events), return the hash,
+// otherwise return nullopt
+std::optional<uint256>
+txnHash(FederatorEvent const& event);
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/ChainListener.cpp

@@ -0,0 +1,912 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/ChainListener.h>
+
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/app/sidechain/impl/WebsocketClient.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/XRPAmount.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/json/Output.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/TxFlags.h>
+#include <ripple/protocol/jss.h>
+
+#include <type_traits>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+
+ChainListener::ChainListener(
+    IsMainchain isMainchain,
+    AccountID const& account,
+    std::weak_ptr<Federator>&& federator,
+    beast::Journal j)
+    : isMainchain_{isMainchain == IsMainchain::yes}
+    , doorAccount_{account}
+    , doorAccountStr_{toBase58(account)}
+    , federator_{std::move(federator)}
+    , initialSync_{std::make_unique<InitialSync>(federator_, isMainchain_, j)}
+    , j_{j}
+{
+}
+
+// destructor must be defined after WebsocketClient size is known (i.e. it can
+// not be defaulted in the header or the unique_ptr declration of
+// WebsocketClient won't work)
+ChainListener::~ChainListener() = default;
+
+std::string const&
+ChainListener::chainName() const
+{
+    // Note: If this function is ever changed to return a value instead of a
+    // ref, review the code to ensure the "jv" functions don't bind to temps
+    static const std::string m("Mainchain");
+    static const std::string s("Sidechain");
+    return isMainchain_ ? m : s;
+}
+
+namespace detail {
+// consider making this available as a general utility
+// Run a lambda on scope exit, unless the `reset` function is called.
+template <class F>
+[[nodiscard]] inline auto
+make_scope(F f)
+{
+    static int dummy = 0;
+    auto d = [f = std::move(f)](auto) { f(); };
+    return std::unique_ptr<int, decltype(d)>{&dummy, std::move(d)};
+}
+
+template <class T>
+std::optional<T>
+getMemoData(Json::Value const& v, std::uint32_t index) = delete;
+
+template <>
+std::optional<uint256>
+getMemoData<uint256>(Json::Value const& v, std::uint32_t index)
+{
+    try
+    {
+        uint256 result;
+        if (result.parseHex(
+                v[jss::Memos][index][jss::Memo][jss::MemoData].asString()))
+            return result;
+    }
+    catch (...)
+    {
+    }
+    return {};
+}
+
+template <>
+std::optional<uint8_t>
+getMemoData<uint8_t>(Json::Value const& v, std::uint32_t index)
+{
+    try
+    {
+        auto const hexData =
+            v[jss::Memos][index][jss::Memo][jss::MemoData].asString();
+        auto d = hexData.data();
+        if (hexData.size() != 2)
+            return {};
+        auto highNibble = charUnHex(d[0]);
+        auto lowNibble = charUnHex(d[1]);
+        if (highNibble < 0 || lowNibble < 0)
+            return {};
+        return (highNibble << 4) | lowNibble;
+    }
+    catch (...)
+    {
+    }
+    return {};
+}
+
+}  // namespace detail
+
+template <class E>
+void
+ChainListener::pushEvent(
+    E&& e,
+    int txHistoryIndex,
+    std::lock_guard<std::mutex> const&)
+{
+    static_assert(std::is_rvalue_reference_v<decltype(e)>, "");
+
+    if (initialSync_)
+    {
+        auto const hasReplayed = initialSync_->onEvent(std::move(e));
+        if (hasReplayed)
+            initialSync_.reset();
+    }
+    else if (auto f = federator_.lock(); f && txHistoryIndex >= 0)
+    {
+        f->push(std::move(e));
+    }
+}
+
+void
+ChainListener::processMessage(Json::Value const& msg)
+{
+    // Even though this lock has a large scope, this function does very little
+    // processing and should run relatively quickly
+    std::lock_guard l{m_};
+
+    JLOGV(
+        j_.trace(),
+        "chain listener message",
+        jv("msg", msg),
+        jv("isMainchain", isMainchain_));
+
+    if (!msg.isMember(jss::validated) || !msg[jss::validated].asBool())
+    {
+        JLOGV(
+            j_.trace(),
+            "ignoring listener message",
+            jv("reason", "not validated"),
+            jv("msg", msg),
+            jv("chain_name", chainName()));
+        return;
+    }
+
+    if (!msg.isMember(jss::engine_result_code))
+    {
+        JLOGV(
+            j_.trace(),
+            "ignoring listener message",
+            jv("reason", "no engine result code"),
+            jv("msg", msg),
+            jv("chain_name", chainName()));
+        return;
+    }
+
+    if (!msg.isMember(jss::account_history_tx_index))
+    {
+        JLOGV(
+            j_.trace(),
+            "ignoring listener message",
+            jv("reason", "no account history tx index"),
+            jv("msg", msg),
+            jv("chain_name", chainName()));
+        return;
+    }
+
+    if (!msg.isMember(jss::meta))
+    {
+        JLOGV(
+            j_.trace(),
+            "ignoring listener message",
+            jv("reason", "tx meta"),
+            jv("msg", msg),
+            jv("chain_name", chainName()));
+        return;
+    }
+
+    auto fieldMatchesStr =
+        [](Json::Value const& val, char const* field, char const* toMatch) {
+            if (!val.isMember(field))
+                return false;
+            auto const f = val[field];
+            if (!f.isString())
+                return false;
+            return f.asString() == toMatch;
+        };
+
+    bool const isLastInHistory = [&msg] {
+        if (msg.isMember(jss::account_history_tx_last))
+            return msg[jss::account_history_tx_last].asBool();
+        return false;
+    }();
+
+    // no matter how this function exits, run this code. It handles the
+    // "lastInHistory" condition. It is difficult to see how to property
+    // annotate `make_scope` for clang's thread safety analysis, so it is
+    // skipped.
+    auto onExit = detail::make_scope([&]() NO_THREAD_SAFETY_ANALYSIS {
+        if (!isLastInHistory)
+            return;
+
+        if (initialSync_ && isLastInHistory)
+        {
+            event::StartOfHistoricTransactions e{isMainchain_};
+            auto const hasReplayed = initialSync_->onEvent(std::move(e));
+            if (hasReplayed)
+                initialSync_.reset();
+
+            JLOGV(
+                j_.trace(),
+                "sent start of historic transactions event",
+                jv("isMainchain", isMainchain_),
+                jv("hasReplayed", hasReplayed));
+        }
+        else
+        {
+            // Note this branch is needed since the other chain's
+            // "setNoLastXChainTxnWithResult" may reset the initialSync object.
+            if (auto f = federator_.lock())
+            {
+                // Inform the other sync object that the last transaction
+                // with a result was found. Note that if start of historic
+                // transactions is found while listening to the mainchain, the
+                // _sidechain_ listener needs to be informed that there is no
+                // last cross chain transaction with result.
+                Federator::ChainType const chainType =
+                    getChainType(!isMainchain_);
+                f->setNoLastXChainTxnWithResult(chainType);
+            }
+        }
+    });
+
+    TER const txnTER = [&msg] {
+        return TER::fromInt(msg[jss::engine_result_code].asInt());
+    }();
+
+    bool const txnSuccess = (txnTER == tesSUCCESS);
+
+    // values < 0 are historical txns. values >= 0 are new transactions. Only
+    // the initial sync needs historical txns.
+    int const txnHistoryIndex = msg[jss::account_history_tx_index].asInt();
+
+    auto const meta = msg[jss::meta];
+
+    // There are two payment types of interest:
+    // 1. User initiated payments on this chain that trigger a transaction on
+    // the other chain.
+    // 2. Federated initated payments on this chain whose status needs to be
+    // checked.
+    enum class PaymentType { user, federator };
+    auto paymentTypeOpt = [&]() -> std::optional<PaymentType> {
+        // Only keep transactions to or from the door account.
+        // Transactions to the account are initiated by users and are are cross
+        // chain transactions. Transaction from the account are initiated by
+        // federators and need to be monitored for errors. There are two types
+        // of transactions that originate from the door account: the second half
+        // of a cross chain payment and a refund of a failed cross chain
+        // payment.
+
+        if (!fieldMatchesStr(msg, jss::type, jss::transaction))
+            return {};
+
+        if (!msg.isMember(jss::transaction))
+            return {};
+        auto const txn = msg[jss::transaction];
+
+        if (!fieldMatchesStr(txn, jss::TransactionType, "Payment"))
+            return {};
+
+        bool const accIsSrc =
+            fieldMatchesStr(txn, jss::Account, doorAccountStr_.c_str());
+        bool const accIsDst =
+            fieldMatchesStr(txn, jss::Destination, doorAccountStr_.c_str());
+
+        if (accIsSrc == accIsDst)
+        {
+            // either account is not involved, or self send
+            return {};
+        }
+
+        if (accIsSrc)
+            return PaymentType::federator;
+        return PaymentType::user;
+    }();
+
+    // There are four types of messages used to control the federator accounts:
+    // 1. AccountSet without modifying account settings. These txns are used to
+    // trigger TicketCreate txns.
+    // 2. TicketCreate to issue tickets.
+    // 3. AccountSet that changes the depositAuth setting of accounts.
+    // 4. SignerListSet to update the signerList of accounts.
+    // 5. AccoutSet that disables the master key. All transactions before this
+    // are used for setup only and should be ignored. This transaction is also
+    // used to help set the initial transaction sequence numbers
+    enum class AccountControlType {
+        trigger,
+        ticket,
+        depositAuth,
+        signerList,
+        disableMasterKey
+    };
+    auto accountControlTypeOpt = [&]() -> std::optional<AccountControlType> {
+        if (!fieldMatchesStr(msg, jss::type, jss::transaction))
+            return {};
+
+        if (!msg.isMember(jss::transaction))
+            return {};
+        auto const txn = msg[jss::transaction];
+
+        if (fieldMatchesStr(txn, jss::TransactionType, "AccountSet"))
+        {
+            if (!(txn.isMember(jss::SetFlag) || txn.isMember(jss::ClearFlag)))
+            {
+                return AccountControlType::trigger;
+            }
+            else
+            {
+                // Get the flags value at the key. If the key is not present,
+                // return 0.
+                auto getFlags =
+                    [&txn](Json::StaticString const& key) -> std::uint32_t {
+                    if (txn.isMember(key))
+                    {
+                        auto const val = txn[key];
+                        try
+                        {
+                            return val.asUInt();
+                        }
+                        catch (...)
+                        {
+                        }
+                    }
+                    return 0;
+                };
+
+                std::uint32_t const setFlags = getFlags(jss::SetFlag);
+                std::uint32_t const clearFlags = getFlags(jss::ClearFlag);
+
+                if (setFlags == asfDepositAuth || clearFlags == asfDepositAuth)
+                    return AccountControlType::depositAuth;
+
+                if (setFlags == asfDisableMaster)
+                    return AccountControlType::disableMasterKey;
+            }
+        }
+        if (fieldMatchesStr(txn, jss::TransactionType, "TicketCreate"))
+            return AccountControlType::ticket;
+        if (fieldMatchesStr(txn, jss::TransactionType, "SignerListSet"))
+            return AccountControlType::signerList;
+
+        return {};
+    }();
+
+    if (!paymentTypeOpt && !accountControlTypeOpt)
+    {
+        JLOGV(
+            j_.warn(),
+            "ignoring listener message",
+            jv("reason", "wrong type, not payment nor account control tx"),
+            jv("msg", msg),
+            jv("chain_name", chainName()));
+        return;
+    }
+    assert(!paymentTypeOpt || !accountControlTypeOpt);
+
+    auto const txnHash = [&]() -> std::optional<uint256> {
+        try
+        {
+            uint256 result;
+            if (result.parseHex(msg[jss::transaction][jss::hash].asString()))
+                return result;
+        }
+        catch (...)
+        {
+        }
+        // TODO: this is an insane input stream
+        // Detect and connect to another server
+        return {};
+    }();
+    if (!txnHash)
+    {
+        JLOG(j_.warn()) << "ignoring listener message, no tx hash";
+        return;
+    }
+
+    auto const seq = [&]() -> std::optional<std::uint32_t> {
+        try
+        {
+            return msg[jss::transaction][jss::Sequence].asUInt();
+        }
+        catch (...)
+        {
+            // TODO: this is an insane input stream
+            // Detect and connect to another server
+            return {};
+        }
+    }();
+    if (!seq)
+    {
+        JLOG(j_.warn()) << "ignoring listener message, no tx seq";
+        return;
+    }
+
+    if (paymentTypeOpt)
+    {
+        PaymentType const paymentType = *paymentTypeOpt;
+
+        std::optional<STAmount> deliveredAmt;
+        if (meta.isMember(jss::delivered_amount))
+        {
+            deliveredAmt =
+                amountFromJson(sfGeneric, meta[jss::delivered_amount]);
+        }
+
+        auto const src = [&]() -> std::optional<AccountID> {
+            try
+            {
+                return parseBase58<AccountID>(
+                    msg[jss::transaction][jss::Account].asString());
+            }
+            catch (...)
+            {
+            }
+            // TODO: this is an insane input stream
+            // Detect and connect to another server
+            return {};
+        }();
+        if (!src)
+        {
+            // TODO: handle the error
+            return;
+        }
+
+        auto const dst = [&]() -> std::optional<AccountID> {
+            try
+            {
+                switch (paymentType)
+                {
+                    case PaymentType::user: {
+                        // This is the destination of the "other chain"
+                        // transfer, which is specified as a memo.
+                        if (!msg.isMember(jss::transaction))
+                        {
+                            return std::nullopt;
+                        }
+                        try
+                        {
+                            // the memo data is a hex encoded version of the
+                            // base58 encoded address. This was chosen for ease
+                            // of encoding by clients.
+                            auto const hexData =
+                                msg[jss::transaction][jss::Memos][0u][jss::Memo]
+                                   [jss::MemoData]
+                                       .asString();
+                            if ((hexData.size() > 100) || (hexData.size() % 2))
+                                return std::nullopt;
+
+                            auto const asciiData = [&]() -> std::string {
+                                std::string result;
+                                result.reserve(40);
+                                auto d = hexData.data();
+                                for (int i = 0; i < hexData.size(); i += 2)
+                                {
+                                    auto highNibble = charUnHex(d[i]);
+                                    auto lowNibble = charUnHex(d[i + 1]);
+                                    if (highNibble < 0 || lowNibble < 0)
+                                        return {};
+                                    char c = (highNibble << 4) | lowNibble;
+                                    result.push_back(c);
+                                }
+                                return result;
+                            }();
+                            return parseBase58<AccountID>(asciiData);
+                        }
+                        catch (...)
+                        {
+                            // User did not specify a destination address in a
+                            // memo
+                            return std::nullopt;
+                        }
+                    }
+                    case PaymentType::federator:
+                        return parseBase58<AccountID>(
+                            msg[jss::transaction][jss::Destination].asString());
+                }
+            }
+            catch (...)
+            {
+            }
+            // TODO: this is an insane input stream
+            // Detect and connect to another server
+            return {};
+        }();
+        if (!dst)
+        {
+            // TODO: handle the error
+            return;
+        }
+
+        switch (paymentType)
+        {
+            case PaymentType::federator: {
+                auto s = txnSuccess ? j_.trace() : j_.error();
+                char const* status = txnSuccess ? "success" : "fail";
+                JLOGV(
+                    s,
+                    "federator txn status",
+                    jv("chain_name", chainName()),
+                    jv("status", status),
+                    jv("msg", msg));
+
+                auto const txnTypeRaw =
+                    detail::getMemoData<uint8_t>(msg[jss::transaction], 0);
+
+                if (!txnTypeRaw || *txnTypeRaw > Federator::txnTypeLast)
+                {
+                    JLOGV(
+                        j_.fatal(),
+                        "expected valid txnType in ChainListener",
+                        jv("msg", msg));
+                    return;
+                }
+
+                Federator::TxnType const txnType =
+                    static_cast<Federator::TxnType>(*txnTypeRaw);
+
+                auto const srcChainTxnHash =
+                    detail::getMemoData<uint256>(msg[jss::transaction], 1);
+
+                if (!srcChainTxnHash)
+                {
+                    JLOGV(
+                        j_.fatal(),
+                        "expected srcChainTxnHash in ChainListener",
+                        jv("msg", msg));
+                    return;
+                }
+                static_assert(
+                    Federator::txnTypeLast == 2, "Add new case below");
+                switch (txnType)
+                {
+                    case Federator::TxnType::xChain: {
+                        using namespace event;
+                        // The dirction looks backwards, but it's not. The
+                        // direction is for the *triggering* transaction.
+                        auto const dir =
+                            isMainchain_ ? Dir::sideToMain : Dir::mainToSide;
+                        XChainTransferResult e{
+                            dir,
+                            *dst,
+                            deliveredAmt,
+                            *seq,
+                            *srcChainTxnHash,
+                            *txnHash,
+                            txnTER,
+                            txnHistoryIndex};
+                        pushEvent(std::move(e), txnHistoryIndex, l);
+                    }
+                    break;
+                    case Federator::TxnType::refund: {
+                        using namespace event;
+                        // The direction is for the triggering transaction.
+                        auto const dir =
+                            isMainchain_ ? Dir::mainToSide : Dir::sideToMain;
+                        auto const dstChainTxnHash =
+                            detail::getMemoData<uint256>(
+                                msg[jss::transaction], 2);
+                        if (!dstChainTxnHash)
+                        {
+                            JLOGV(
+                                j_.fatal(),
+                                "expected valid dstChainTxnHash in "
+                                "ChainListener",
+                                jv("msg", msg));
+                            return;
+                        }
+                        RefundTransferResult e{
+                            dir,
+                            *dst,
+                            deliveredAmt,
+                            *seq,
+                            *srcChainTxnHash,
+                            *dstChainTxnHash,
+                            *txnHash,
+                            txnTER,
+                            txnHistoryIndex};
+                        pushEvent(std::move(e), txnHistoryIndex, l);
+                    }
+                    break;
+                }
+            }
+            break;
+            case PaymentType::user: {
+                if (!txnSuccess)
+                    return;
+
+                if (!deliveredAmt)
+                    return;
+                {
+                    using namespace event;
+                    XChainTransferDetected e{
+                        isMainchain_ ? Dir::mainToSide : Dir::sideToMain,
+                        *src,
+                        *dst,
+                        *deliveredAmt,
+                        *seq,
+                        *txnHash,
+                        txnHistoryIndex};
+                    pushEvent(std::move(e), txnHistoryIndex, l);
+                }
+            }
+            break;
+        }
+    }
+    else
+    {
+        // account control tx
+        auto const ledgerIndex = [&]() -> std::optional<std::uint32_t> {
+            try
+            {
+                return msg["ledger_index"].asInt();
+            }
+            catch (...)
+            {
+                JLOGV(j_.error(), "no ledger_index", jv("message", msg));
+                assert(false);
+                return {};
+            }
+        }();
+        if (!ledgerIndex)
+        {
+            JLOG(j_.warn()) << "ignoring listener message, no ledgerIndex";
+            return;
+        }
+
+        auto const getSourceTag = [&]() -> std::optional<std::uint32_t> {
+            try
+            {
+                return msg[jss::transaction]["SourceTag"].asUInt();
+            }
+            catch (...)
+            {
+                JLOGV(j_.error(), "wrong SourceTag", jv("message", msg));
+                assert(false);
+                return {};
+            }
+        };
+
+        auto const getMemoStr = [&](std::uint32_t index) -> std::string {
+            try
+            {
+                if (msg[jss::transaction][jss::Memos][index] ==
+                    Json::Value::null)
+                    return {};
+                auto str = std::string(msg[jss::transaction][jss::Memos][index]
+                                          [jss::Memo][jss::MemoData]
+                                              .asString());
+                assert(str.length() <= event::MemoStringMax);
+                return str;
+            }
+            catch (...)
+            {
+            }
+            return {};
+        };
+
+        auto const accountControlType = *accountControlTypeOpt;
+        switch (accountControlType)
+        {
+            case AccountControlType::trigger: {
+                JLOGV(
+                    j_.trace(),
+                    "AccountControlType::trigger",
+                    jv("chain_name", chainName()),
+                    jv("account_seq", *seq),
+                    jv("msg", msg));
+                auto sourceTag = getSourceTag();
+                if (!sourceTag)
+                {
+                    JLOG(j_.warn())
+                        << "ignoring listener message, no sourceTag";
+                    return;
+                }
+                auto memoStr = getMemoStr(0);
+                event::TicketCreateTrigger e = {
+                    isMainchain_ ? event::Dir::mainToSide
+                                 : event::Dir::sideToMain,
+                    txnSuccess,
+                    0,
+                    *ledgerIndex,
+                    *txnHash,
+                    txnHistoryIndex,
+                    *sourceTag,
+                    std::move(memoStr)};
+                pushEvent(std::move(e), txnHistoryIndex, l);
+                break;
+            }
+            case AccountControlType::ticket: {
+                JLOGV(
+                    j_.trace(),
+                    "AccountControlType::ticket",
+                    jv("chain_name", chainName()),
+                    jv("account_seq", *seq),
+                    jv("msg", msg));
+                auto sourceTag = getSourceTag();
+                if (!sourceTag)
+                {
+                    JLOG(j_.warn())
+                        << "ignoring listener message, no sourceTag";
+                    return;
+                }
+
+                auto const triggeringTxnHash =
+                    detail::getMemoData<uint256>(msg[jss::transaction], 0);
+                if (!triggeringTxnHash)
+                {
+                    JLOGV(
+                        (txnSuccess ? j_.trace() : j_.error()),
+                        "bootstrap ticket",
+                        jv("chain_name", chainName()),
+                        jv("account_seq", *seq),
+                        jv("msg", msg));
+
+                    if (!txnSuccess)
+                        return;
+
+                    event::BootstrapTicket e = {
+                        isMainchain_,
+                        txnSuccess,
+                        *seq,
+                        *ledgerIndex,
+                        txnHistoryIndex,
+                        *sourceTag};
+                    pushEvent(std::move(e), txnHistoryIndex, l);
+                    return;
+                }
+
+                // The TicketCreate tx is both the result of its triggering
+                // AccountSet tx, and the trigger of another account control tx,
+                // if there is a tx in the memo field.
+                event::TicketCreateResult e = {
+                    isMainchain_ ? event::Dir::sideToMain
+                                 : event::Dir::mainToSide,
+                    txnSuccess,
+                    *seq,
+                    *ledgerIndex,
+                    *triggeringTxnHash,
+                    *txnHash,
+                    txnHistoryIndex,
+                    *sourceTag,
+                    getMemoStr(1)};
+                pushEvent(std::move(e), txnHistoryIndex, l);
+                break;
+            }
+            case AccountControlType::depositAuth: {
+                JLOGV(
+                    j_.trace(),
+                    "AccountControlType::depositAuth",
+                    jv("chain_name", chainName()),
+                    jv("account_seq", *seq),
+                    jv("msg", msg));
+                auto const triggeringTxHash =
+                    detail::getMemoData<uint256>(msg[jss::transaction], 0);
+                if (!triggeringTxHash)
+                {
+                    JLOG(j_.warn())
+                        << "ignoring listener message, no triggeringTxHash";
+                    return;
+                }
+
+                auto opOpt = [&]() -> std::optional<event::AccountFlagOp> {
+                    try
+                    {
+                        if (msg[jss::transaction].isMember(jss::SetFlag) &&
+                            msg[jss::transaction][jss::SetFlag].isIntegral())
+                        {
+                            assert(
+                                msg[jss::transaction][jss::SetFlag].asUInt() ==
+                                asfDepositAuth);
+                            return event::AccountFlagOp::set;
+                        }
+                        if (msg[jss::transaction].isMember(jss::ClearFlag) &&
+                            msg[jss::transaction][jss::ClearFlag].isIntegral())
+                        {
+                            assert(
+                                msg[jss::transaction][jss::ClearFlag]
+                                    .asUInt() == asfDepositAuth);
+
+                            return event::AccountFlagOp::clear;
+                        }
+                    }
+                    catch (...)
+                    {
+                    }
+                    JLOGV(
+                        j_.error(),
+                        "unexpected accountSet tx",
+                        jv("message", msg));
+                    assert(false);
+                    return {};
+                }();
+                if (!opOpt)
+                    return;
+
+                event::DepositAuthResult e{
+                    isMainchain_ ? event::Dir::sideToMain
+                                 : event::Dir::mainToSide,
+                    txnSuccess,
+                    *seq,
+                    *ledgerIndex,
+                    *triggeringTxHash,
+                    txnHistoryIndex,
+                    *opOpt};
+                pushEvent(std::move(e), txnHistoryIndex, l);
+                break;
+            }
+            case AccountControlType::signerList:
+                // TODO
+                break;
+            case AccountControlType::disableMasterKey: {
+                event::DisableMasterKeyResult e{
+                    isMainchain_, *seq, txnHistoryIndex};
+                pushEvent(std::move(e), txnHistoryIndex, l);
+                break;
+            }
+            break;
+        }
+    }
+
+    // Note: Handling "last in history" is done through the lambda given
+    // to `make_scope` earlier in the function
+}
+
+void
+ChainListener::setLastXChainTxnWithResult(uint256 const& hash)
+{
+    // Note that `onMessage` also locks this mutex, and it calls
+    // `setLastXChainTxnWithResult`. However, it calls that function on the
+    // other chain, so the mutex will not be locked twice on the same
+    // thread.
+    std::lock_guard l{m_};
+    if (!initialSync_)
+        return;
+
+    auto const hasReplayed = initialSync_->setLastXChainTxnWithResult(hash);
+    if (hasReplayed)
+        initialSync_.reset();
+}
+
+void
+ChainListener::setNoLastXChainTxnWithResult()
+{
+    // Note that `onMessage` also locks this mutex, and it calls
+    // `setNoLastXChainTxnWithResult`. However, it calls that function on
+    // the other chain, so the mutex will not be locked twice on the same
+    // thread.
+    std::lock_guard l{m_};
+    if (!initialSync_)
+        return;
+
+    bool const hasReplayed = initialSync_->setNoLastXChainTxnWithResult();
+    if (hasReplayed)
+        initialSync_.reset();
+}
+
+Json::Value
+ChainListener::getInfo() const
+{
+    std::lock_guard l{m_};
+
+    Json::Value ret{Json::objectValue};
+    ret[jss::state] = initialSync_ ? "syncing" : "normal";
+    if (initialSync_)
+    {
+        ret[jss::sync_info] = initialSync_->getInfo();
+    }
+    // get the state (in sync, syncing)
+    return ret;
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/ChainListener.h

@@ -0,0 +1,102 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_CHAINLISTENER_H_INCLUDED
+#define RIPPLE_SIDECHAIN_IMPL_CHAINLISTENER_H_INCLUDED
+
+#include <ripple/protocol/AccountID.h>
+
+#include <ripple/app/sidechain/impl/InitialSync.h>
+#include <ripple/beast/utility/Journal.h>
+#include <boost/asio/io_service.hpp>
+#include <boost/asio/ip/address.hpp>
+
+#include <memory>
+#include <mutex>
+#include <string>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+class WebsocketClient;
+
+class ChainListener
+{
+protected:
+    enum class IsMainchain { no, yes };
+
+    bool const isMainchain_;
+    // Sending xrp to the door account will trigger a x-chain transaction
+    AccountID const doorAccount_;
+    std::string const doorAccountStr_;
+    std::weak_ptr<Federator> federator_;
+    mutable std::mutex m_;
+    // Logic to handle potentially collecting and replaying historical
+    // transactions. Will be empty after replaying.
+    std::unique_ptr<InitialSync> GUARDED_BY(m_) initialSync_;
+    beast::Journal j_;
+
+    ChainListener(
+        IsMainchain isMainchain,
+        AccountID const& account,
+        std::weak_ptr<Federator>&& federator,
+        beast::Journal j);
+
+    virtual ~ChainListener();
+
+    std::string const&
+    chainName() const;
+
+    void
+    processMessage(Json::Value const& msg) EXCLUDES(m_);
+
+    template <class E>
+    void
+    pushEvent(E&& e, int txHistoryIndex, std::lock_guard<std::mutex> const&)
+        REQUIRES(m_);
+
+public:
+    void
+    setLastXChainTxnWithResult(uint256 const& hash) EXCLUDES(m_);
+    void
+    setNoLastXChainTxnWithResult() EXCLUDES(m_);
+
+    Json::Value
+    getInfo() const EXCLUDES(m_);
+
+    using RpcCallback = std::function<void(Json::Value const&)>;
+
+    /**
+     * send a RPC and call the callback with the RPC result
+     * @param cmd PRC command
+     * @param params RPC command parameter
+     * @param onResponse callback to process RPC result
+     */
+    virtual void
+    send(
+        std::string const& cmd,
+        Json::Value const& params,
+        RpcCallback onResponse) = 0;
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/DoorKeeper.cpp

@@ -0,0 +1,327 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/DoorKeeper.h>
+
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/impl/TicketHolder.h>
+#include <ripple/basics/Log.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/protocol/LedgerFormats.h>
+#include <ripple/protocol/TxFlags.h>
+
+namespace ripple {
+namespace sidechain {
+
+DoorKeeper::DoorKeeper(
+    bool isMainChain,
+    AccountID const& account,
+    TicketRunner& ticketRunner,
+    Federator& federator,
+    beast::Journal j)
+    : isMainChain_(isMainChain)
+    , accountStr_(toBase58(account))
+    , ticketRunner_(ticketRunner)
+    , federator_(federator)
+    , j_(j)
+{
+}
+
+void
+DoorKeeper::init()
+{
+    std::lock_guard lock(mtx_);
+    if (initData_.status_ != InitializeStatus::waitLedger)
+        return;
+    initData_.status_ = InitializeStatus::waitAccountInfo;
+    rpcAccountInfo(lock);
+}
+
+void
+DoorKeeper::updateQueueLength(std::uint32_t length)
+{
+    DoorStatus oldStatus;
+    auto const tx = [&]() -> std::optional<Json::Value> {
+        enum Action { setFlag, clearFlag, noAction };
+        Action action = noAction;
+        std::lock_guard lock(mtx_);
+        JLOGV(
+            j_.trace(),
+            "updateQueueLength",
+            jv("account:", accountStr_),
+            jv("QLen", length),
+            jv("chain", (isMainChain_ ? "main" : "side")));
+
+        if (initData_.status_ != InitializeStatus::initialized)
+            return {};
+
+        oldStatus = status_;
+        if (length >= HighWaterMark && status_ == DoorStatus::open)
+        {
+            action = setFlag;
+            status_ = DoorStatus::closing;
+        }
+        else if (length <= LowWaterMark && status_ == DoorStatus::closed)
+        {
+            action = clearFlag;
+            status_ = DoorStatus::opening;
+        }
+
+        if (action == noAction)
+            return {};
+
+        XRPAmount const fee{Federator::accountControlTxFee};
+        Json::Value txJson;
+        txJson[jss::TransactionType] = "AccountSet";
+        txJson[jss::Account] = accountStr_;
+        txJson[jss::Sequence] = 0;  // to be filled by ticketRunner
+        txJson[jss::Fee] = to_string(fee);
+        if (action == setFlag)
+            txJson[jss::SetFlag] = asfDepositAuth;
+        else
+            txJson[jss::ClearFlag] = asfDepositAuth;
+        return txJson;
+    }();
+
+    if (tx)
+    {
+        bool triggered = false;
+        if (isMainChain_)
+        {
+            triggered =
+                ticketRunner_.trigger(TicketPurpose::mainDoorKeeper, tx, {});
+        }
+        else
+        {
+            triggered =
+                ticketRunner_.trigger(TicketPurpose::sideDoorKeeper, {}, tx);
+        }
+
+        JLOGV(
+            j_.trace(),
+            "updateQueueLength",
+            jv("account:", accountStr_),
+            jv("QLen", length),
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("tx", *tx),
+            jv("triggered", (triggered ? "yes" : "no")));
+
+        if (!triggered)
+        {
+            std::lock_guard lock(mtx_);
+            status_ = oldStatus;
+        }
+    }
+}
+
+void
+DoorKeeper::onEvent(const event::DepositAuthResult& e)
+{
+    std::lock_guard lock(mtx_);
+    if (initData_.status_ != InitializeStatus::initialized)
+    {
+        JLOG(j_.trace()) << "Queue an event";
+        initData_.toReplay_.push(e);
+    }
+    else
+    {
+        processEvent(e, lock);
+    }
+}
+
+void
+DoorKeeper::rpcAccountInfo(std::lock_guard<std::mutex> const&)
+{
+    Json::Value params = [&] {
+        Json::Value r;
+        r[jss::account] = accountStr_;
+        r[jss::ledger_index] = "validated";
+        r[jss::signer_lists] = false;
+        return r;
+    }();
+
+    rpcChannel_->send(
+        "account_info",
+        params,
+        [chain = isMainChain_ ? Federator::mainChain : Federator::sideChain,
+         wp = federator_.weak_from_this()](Json::Value const& response) {
+            if (auto f = wp.lock())
+                f->getDoorKeeper(chain).accountInfoResult(response);
+        });
+}
+
+void
+DoorKeeper::accountInfoResult(const Json::Value& rpcResult)
+{
+    auto ledgerNFlagsOpt =
+        [&]() -> std::optional<std::pair<std::uint32_t, std::uint32_t>> {
+        try
+        {
+            if (rpcResult.isMember(jss::error))
+            {
+                return {};
+            }
+            if (!rpcResult[jss::validated].asBool())
+            {
+                return {};
+            }
+            if (rpcResult[jss::account_data][jss::Account] != accountStr_)
+            {
+                return {};
+            }
+            if (!rpcResult[jss::account_data][jss::Flags].isIntegral())
+            {
+                return {};
+            }
+            if (!rpcResult[jss::ledger_index].isIntegral())
+            {
+                return {};
+            }
+
+            return std::make_pair(
+                rpcResult[jss::ledger_index].asUInt(),
+                rpcResult[jss::account_data][jss::Flags].asUInt());
+        }
+        catch (...)
+        {
+            return {};
+        }
+    }();
+
+    if (!ledgerNFlagsOpt)
+    {
+        // should not reach here since we only ask account_object after a
+        // validated ledger
+        JLOGV(j_.error(), "account_info result ", jv("result", rpcResult));
+        assert(false);
+        return;
+    }
+
+    auto [ledgerIndex, flags] = *ledgerNFlagsOpt;
+    {
+        JLOGV(
+            j_.trace(),
+            "accountInfoResult",
+            jv("ledgerIndex", ledgerIndex),
+            jv("flags", flags));
+        std::lock_guard lock(mtx_);
+        initData_.ledgerIndex_ = ledgerIndex;
+        status_ = (flags & lsfDepositAuth) == 0 ? DoorStatus::open
+                                                : DoorStatus::closed;
+        while (!initData_.toReplay_.empty())
+        {
+            processEvent(initData_.toReplay_.front(), lock);
+            initData_.toReplay_.pop();
+        }
+        initData_.status_ = InitializeStatus::initialized;
+        JLOG(j_.info()) << "DoorKeeper initialized, status "
+                        << (status_ == DoorStatus::open ? "open" : "closed");
+    }
+}
+
+void
+DoorKeeper::processEvent(
+    const event::DepositAuthResult& e,
+    std::lock_guard<std::mutex> const&)
+{
+    if (e.ledgerIndex_ <= initData_.ledgerIndex_)
+    {
+        JLOGV(
+            j_.trace(),
+            "DepositAuthResult, ignoring an old result",
+            jv("account:", accountStr_),
+            jv("operation",
+               (e.op_ == event::AccountFlagOp::set ? "set" : "clear")));
+        return;
+    }
+
+    JLOGV(
+        j_.trace(),
+        "DepositAuthResult",
+        jv("chain", (isMainChain_ ? "main" : "side")),
+        jv("account:", accountStr_),
+        jv("operation",
+           (e.op_ == event::AccountFlagOp::set ? "set" : "clear")));
+
+    if (!e.success_)
+    {
+        JLOG(j_.error()) << "DepositAuthResult event error, account "
+                         << (isMainChain_ ? "main" : "side") << accountStr_;
+        assert(false);
+        return;
+    }
+
+    switch (e.op_)
+    {
+        case event::AccountFlagOp::set:
+            assert(
+                status_ == DoorStatus::open || status_ == DoorStatus::closing);
+            status_ = DoorStatus::closed;
+            break;
+        case event::AccountFlagOp::clear:
+            assert(
+                status_ == DoorStatus::closed ||
+                status_ == DoorStatus::opening);
+            status_ = DoorStatus::open;
+            break;
+    }
+}
+
+Json::Value
+DoorKeeper::getInfo() const
+{
+    auto DoorStatusToStr = [](DoorKeeper::DoorStatus s) -> std::string {
+        switch (s)
+        {
+            case DoorKeeper::DoorStatus::open:
+                return "open";
+            case DoorKeeper::DoorStatus::opening:
+                return "opening";
+            case DoorKeeper::DoorStatus::closed:
+                return "closed";
+            case DoorKeeper::DoorStatus::closing:
+                return "closing";
+        }
+        return {};
+    };
+
+    Json::Value ret{Json::objectValue};
+    {
+        std::lock_guard lock{mtx_};
+        if (initData_.status_ == InitializeStatus::initialized)
+        {
+            ret["initialized"] = "true";
+            ret["status"] = DoorStatusToStr(status_);
+        }
+        else
+        {
+            ret["initialized"] = "false";
+        }
+    }
+    return ret;
+}
+
+void
+DoorKeeper::setRpcChannel(std::shared_ptr<ChainListener> channel)
+{
+    rpcChannel_ = std::move(channel);
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/DoorKeeper.h

@@ -0,0 +1,128 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_DOOR_OPENER_H
+#define RIPPLE_SIDECHAIN_IMPL_DOOR_OPENER_H
+
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/app/sidechain/impl/ChainListener.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/AccountID.h>
+
+#include <mutex>
+#include <queue>
+#include <string>
+
+namespace ripple {
+namespace sidechain {
+
+class TicketRunner;
+class Federator;
+
+class DoorKeeper
+{
+public:
+    static constexpr std::uint32_t LowWaterMark = 0;
+    static constexpr std::uint32_t HighWaterMark = 100;
+    static_assert(HighWaterMark > LowWaterMark);
+    enum class DoorStatus { open, closing, closed, opening };
+
+private:
+    enum class InitializeStatus { waitLedger, waitAccountInfo, initialized };
+    struct InitializeData
+    {
+        InitializeStatus status_ = InitializeStatus::waitLedger;
+        std::queue<event::DepositAuthResult> toReplay_;
+        std::uint32_t ledgerIndex_ = 0;
+    };
+
+    std::shared_ptr<ChainListener> rpcChannel_;
+    bool const isMainChain_;
+    std::string const accountStr_;
+    mutable std::mutex mtx_;
+    InitializeData GUARDED_BY(mtx_) initData_;
+    DoorStatus GUARDED_BY(mtx_) status_;
+    TicketRunner& ticketRunner_;
+    Federator& federator_;
+    beast::Journal j_;
+
+public:
+    DoorKeeper(
+        bool isMainChain,
+        AccountID const& account,
+        TicketRunner& ticketRunner,
+        Federator& federator,
+        beast::Journal j);
+    ~DoorKeeper() = default;
+
+    /**
+     * start to initialize the doorKeeper by sending accountInfo RPC
+     */
+    void
+    init() EXCLUDES(mtx_);
+
+    /**
+     * process the accountInfo result and set the door status
+     * This is the end of initialization
+     *
+     * @param rpcResult the accountInfo result
+     */
+    void
+    accountInfoResult(Json::Value const& rpcResult) EXCLUDES(mtx_);
+
+    /**
+     * update the doorKeeper about the number of pending XChain payments
+     * The doorKeeper will close the door if there are too many
+     * pending XChain payments and reopen the door later
+     *
+     * @param length the number of pending XChain payments
+     */
+    void
+    updateQueueLength(std::uint32_t length) EXCLUDES(mtx_);
+
+    /**
+     * process a DepositAuthResult event and set the door status.
+     * It queues the event if the doorKeeper is not yet initialized.
+     *
+     * @param e the DepositAuthResult event
+     */
+    void
+    onEvent(event::DepositAuthResult const& e) EXCLUDES(mtx_);
+
+    Json::Value
+    getInfo() const EXCLUDES(mtx_);
+
+    void
+    setRpcChannel(std::shared_ptr<ChainListener> channel);
+
+private:
+    void
+    rpcAccountInfo(std::lock_guard<std::mutex> const&) REQUIRES(mtx_);
+
+    void
+    processEvent(
+        event::DepositAuthResult const& e,
+        std::lock_guard<std::mutex> const&) REQUIRES(mtx_);
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/InitialSync.cpp

@@ -0,0 +1,555 @@
+
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/InitialSync.h>
+
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/contract.h>
+#include <ripple/json/json_writer.h>
+
+#include <type_traits>
+
+namespace ripple {
+namespace sidechain {
+
+InitialSync::InitialSync(
+    std::weak_ptr<Federator> federator,
+    bool isMainchain,
+    beast::Journal j)
+    : federator_{std::move(federator)}, isMainchain_{isMainchain}, j_{j}
+{
+}
+
+bool
+InitialSync::hasTransaction(
+    uint256 const& txnHash,
+    std::lock_guard<std::mutex> const&) const
+{
+    return seenTriggeringTxns_.count(txnHash);
+}
+
+bool
+InitialSync::canReplay(std::lock_guard<std::mutex> const&) const
+{
+    return !(
+        needsLastXChainTxn_ || needsOtherChainLastXChainTxn_ ||
+        needsReplayStartTxnHash_);
+}
+
+void
+InitialSync::stopHistoricalTxns(std::lock_guard<std::mutex> const&)
+{
+    if (!acquiringHistoricData_)
+        return;
+
+    acquiringHistoricData_ = false;
+    if (auto f = federator_.lock())
+    {
+        f->stopHistoricalTxns(getChainType(isMainchain_));
+    }
+}
+
+void
+InitialSync::done()
+{
+    if (auto f = federator_.lock())
+    {
+        f->initialSyncDone(
+            isMainchain_ ? Federator::ChainType::mainChain
+                         : Federator::ChainType::sideChain);
+    }
+}
+
+bool
+InitialSync::setLastXChainTxnWithResult(uint256 const& hash)
+{
+    std::lock_guard l{m_};
+    JLOGV(
+        j_.trace(),
+        "last xchain txn with result",
+        jv("needsOtherChainLastXChainTxn", needsOtherChainLastXChainTxn_),
+        jv("isMainchain", isMainchain_),
+        jv("hash", hash));
+    assert(lastXChainTxnWithResult_.value_or(hash) == hash);
+    if (hasReplayed_ || lastXChainTxnWithResult_)
+        return hasReplayed_;
+
+    lastXChainTxnWithResult_ = hash;
+    needsReplayStartTxnHash_ = false;
+    if (needsLastXChainTxn_)
+    {
+        needsLastXChainTxn_ =
+            !seenTriggeringTxns_.count(*lastXChainTxnWithResult_);
+    }
+
+    if (!acquiringHistoricData_ && needsLastXChainTxn_)
+        LogicError("Initial sync could not find historic XChain transaction");
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+bool
+InitialSync::setNoLastXChainTxnWithResult()
+{
+    std::lock_guard l{m_};
+    JLOGV(
+        j_.trace(),
+        "no last xchain txn with result",
+        jv("needsOtherChainLastXChainTxn", needsOtherChainLastXChainTxn_),
+        jv("isMainchain", isMainchain_));
+    assert(!lastXChainTxnWithResult_);
+    if (hasReplayed_)
+        return hasReplayed_;
+
+    needsLastXChainTxn_ = false;
+    needsReplayStartTxnHash_ = false;
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+void
+InitialSync::replay(std::lock_guard<std::mutex> const& l)
+{
+    if (hasReplayed_)
+        return;
+
+    assert(canReplay(l));
+
+    // Note that this function may push a large number of events to the
+    // federator, and it runs under a lock. However, pushing an event to the
+    // federator just copies it into a collection (it does not handle the event
+    // in the same thread). So this should run relatively quickly.
+    stopHistoricalTxns(l);
+    hasReplayed_ = true;
+    JLOGV(
+        j_.trace(),
+        "InitialSync replay,",
+        jv("chain_name", (isMainchain_ ? "Mainchain" : "Sidechain")),
+        jv("lastXChainTxnWithResult_",
+           (lastXChainTxnWithResult_ ? strHex(*lastXChainTxnWithResult_)
+                                     : "not set")));
+
+    if (lastXChainTxnWithResult_)
+        assert(seenTriggeringTxns_.count(*lastXChainTxnWithResult_));
+
+    if (lastXChainTxnWithResult_ &&
+        seenTriggeringTxns_.count(*lastXChainTxnWithResult_))
+    {
+        // Remove the XChainTransferDetected event associated with this txn, and
+        // all the XChainTransferDetected events before it. They have already
+        // been submitted. If they are not removed, they will never collect
+        // enough signatures to be submitted (since the other federators have
+        // already submitted it), and it will prevent subsequent event from
+        // replaying.
+        std::vector<decltype(pendingEvents_)::const_iterator> toRemove;
+        toRemove.reserve(pendingEvents_.size());
+        std::vector<decltype(pendingEvents_)::const_iterator> toRemoveTrigger;
+        bool matched = false;
+        for (auto i = pendingEvents_.cbegin(), e = pendingEvents_.cend();
+             i != e;
+             ++i)
+        {
+            auto const et = eventType(i->second);
+            if (et == event::EventType::trigger)
+            {
+                toRemove.push_back(i);
+            }
+            else if (et == event::EventType::resultAndTrigger)
+            {
+                toRemoveTrigger.push_back(i);
+            }
+            else
+            {
+                continue;
+            }
+
+            auto const txnHash = sidechain::txnHash(i->second);
+            if (!txnHash)
+            {
+                // All triggering events should have a txnHash
+                assert(0);
+                continue;
+            }
+            JLOGV(
+                j_.trace(),
+                "InitialSync replay, remove trigger event from pendingEvents_",
+                jv("chain_name", (isMainchain_ ? "Mainchain" : "Sidechain")),
+                jv("txnHash", *txnHash));
+            if (*lastXChainTxnWithResult_ == *txnHash)
+            {
+                matched = true;
+                break;
+            }
+        }
+        assert(matched);
+        if (matched)
+        {
+            for (auto i : toRemoveTrigger)
+            {
+                if (auto ticketResult = std::get_if<event::TicketCreateResult>(
+                        &(pendingEvents_.erase(i, i)->second));
+                    ticketResult)
+                {
+                    ticketResult->removeTrigger();
+                }
+            }
+            for (auto i = toRemove.begin(), e = toRemove.end(); i != e; ++i)
+            {
+                pendingEvents_.erase(*i);
+            }
+        }
+    }
+
+    if (auto f = federator_.lock())
+    {
+        for (auto&& [_, e] : pendingEvents_)
+            f->push(std::move(e));
+    }
+
+    seenTriggeringTxns_.clear();
+    pendingEvents_.clear();
+    done();
+}
+
+bool
+InitialSync::onEvent(event::XChainTransferDetected&& e)
+{
+    return onTriggerEvent(std::move(e));
+}
+
+bool
+InitialSync::onEvent(event::XChainTransferResult&& e)
+{
+    return onResultEvent(std::move(e), 1);
+}
+
+bool
+InitialSync::onEvent(event::TicketCreateTrigger&& e)
+{
+    return onTriggerEvent(std::move(e));
+}
+
+bool
+InitialSync::onEvent(event::TicketCreateResult&& e)
+{
+    static_assert(std::is_rvalue_reference_v<decltype(e)>, "");
+
+    std::lock_guard l{m_};
+    if (hasReplayed_)
+    {
+        assert(0);
+        return hasReplayed_;
+    }
+
+    JLOGV(
+        j_.trace(), "InitialSync TicketCreateResult", jv("event", e.toJson()));
+
+    if (needsOtherChainLastXChainTxn_)
+    {
+        if (auto f = federator_.lock())
+        {
+            // Inform the other sync object that the last transaction with a
+            // result was found. e.dir_ is for the triggering transaction.
+            Federator::ChainType const chainType = srcChainType(e.dir_);
+            f->setLastXChainTxnWithResult(
+                chainType, e.txnSeq_, 2, e.srcChainTxnHash_);
+        }
+        needsOtherChainLastXChainTxn_ = false;
+    }
+
+    if (!e.memoStr_.empty())
+    {
+        seenTriggeringTxns_.insert(e.txnHash_);
+        if (lastXChainTxnWithResult_ && needsLastXChainTxn_)
+        {
+            if (e.txnHash_ == *lastXChainTxnWithResult_)
+            {
+                needsLastXChainTxn_ = false;
+                JLOGV(
+                    j_.trace(),
+                    "InitialSync TicketCreateResult, found the trigger tx",
+                    jv("txHash", e.txnHash_),
+                    jv("chain_name",
+                       (isMainchain_ ? "Mainchain" : "Sidechain")));
+            }
+        }
+    }
+    pendingEvents_[e.rpcOrder_] = std::move(e);
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+bool
+InitialSync::onEvent(event::DepositAuthResult&& e)
+{
+    return onResultEvent(std::move(e), 1);
+}
+
+bool
+InitialSync::onEvent(event::BootstrapTicket&& e)
+{
+    std::lock_guard l{m_};
+
+    JLOGV(j_.trace(), "InitialSync onBootstrapTicket", jv("event", e.toJson()));
+
+    if (hasReplayed_)
+    {
+        assert(0);
+        return hasReplayed_;
+    }
+
+    pendingEvents_[e.rpcOrder_] = std::move(e);
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+bool
+InitialSync::onEvent(event::DisableMasterKeyResult&& e)
+{
+    std::lock_guard l{m_};
+
+    if (hasReplayed_)
+    {
+        assert(0);
+        return hasReplayed_;
+    }
+
+    JLOGV(
+        j_.trace(),
+        "InitialSync onDisableMasterKeyResultEvent",
+        jv("event", e.toJson()));
+    assert(!disableMasterKeySeq_);
+    disableMasterKeySeq_ = e.txnSeq_;
+
+    pendingEvents_[e.rpcOrder_] = std::move(e);
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+template <class T>
+bool
+InitialSync::onTriggerEvent(T&& e)
+{
+    static_assert(std::is_rvalue_reference_v<decltype(e)>, "");
+
+    std::lock_guard l{m_};
+    if (hasReplayed_)
+    {
+        assert(0);
+        return hasReplayed_;
+    }
+
+    JLOGV(j_.trace(), "InitialSync onTriggerEvent", jv("event", e.toJson()));
+    seenTriggeringTxns_.insert(e.txnHash_);
+    if (lastXChainTxnWithResult_ && needsLastXChainTxn_)
+    {
+        if (e.txnHash_ == *lastXChainTxnWithResult_)
+        {
+            needsLastXChainTxn_ = false;
+            JLOGV(
+                j_.trace(),
+                "InitialSync onTriggerEvent, found the trigger tx",
+                jv("txHash", e.txnHash_),
+                jv("chain_name", (isMainchain_ ? "Mainchain" : "Sidechain")));
+        }
+    }
+    pendingEvents_[e.rpcOrder_] = std::move(e);
+
+    if (canReplay(l))
+    {
+        replay(l);
+    }
+    return hasReplayed_;
+}
+
+template <class T>
+bool
+InitialSync::onResultEvent(T&& e, std::uint32_t seqTook)
+{
+    static_assert(std::is_rvalue_reference_v<decltype(e)>, "");
+
+    std::lock_guard l{m_};
+    if (hasReplayed_)
+    {
+        assert(0);
+        return hasReplayed_;
+    }
+
+    JLOGV(j_.trace(), "InitialSync onResultEvent", jv("event", e.toJson()));
+
+    if (needsOtherChainLastXChainTxn_)
+    {
+        if (auto f = federator_.lock())
+        {
+            // Inform the other sync object that the last transaction with a
+            // result was found. e.dir_ is for the triggering transaction.
+            Federator::ChainType const chainType = srcChainType(e.dir_);
+            f->setLastXChainTxnWithResult(
+                chainType, e.txnSeq_, seqTook, e.srcChainTxnHash_);
+        }
+        needsOtherChainLastXChainTxn_ = false;
+    }
+
+    pendingEvents_[e.rpcOrder_] = std::move(e);
+
+    if (canReplay(l))
+        replay(l);
+
+    return hasReplayed_;
+}
+
+bool
+InitialSync::onEvent(event::RefundTransferResult&& e)
+{
+    std::lock_guard l{m_};
+    if (hasReplayed_)
+    {
+        assert(0);
+    }
+    else
+    {
+        pendingEvents_[e.rpcOrder_] = std::move(e);
+
+        if (canReplay(l))
+            replay(l);
+    }
+    return hasReplayed_;
+}
+
+bool
+InitialSync::onEvent(event::StartOfHistoricTransactions&& e)
+{
+    std::lock_guard l{m_};
+    if (lastXChainTxnWithResult_)
+        LogicError("Initial sync could not find historic XChain transaction");
+
+    if (needsOtherChainLastXChainTxn_)
+    {
+        if (auto f = federator_.lock())
+        {
+            // Inform the other sync object that the last transaction
+            // with a result was found. Note that if start of historic
+            // transactions is found while listening to the mainchain, the
+            // _sidechain_ listener needs to be informed that there is no last
+            // cross chain transaction with result.
+            Federator::ChainType const chainType = getChainType(!isMainchain_);
+            f->setNoLastXChainTxnWithResult(chainType);
+        }
+        needsOtherChainLastXChainTxn_ = false;
+    }
+
+    acquiringHistoricData_ = false;
+    needsOtherChainLastXChainTxn_ = false;
+
+    if (canReplay(l))
+    {
+        replay(l);
+    }
+
+    return hasReplayed_;
+}
+
+namespace detail {
+
+Json::Value
+getInfo(FederatorEvent const& event)
+{
+    return std::visit(
+        [](auto const& e) {
+            using eventType = decltype(e);
+            Json::Value ret{Json::objectValue};
+            if constexpr (std::is_same_v<
+                              eventType,
+                              event::XChainTransferDetected>)
+            {
+                ret[jss::type] = "xchain_transfer_detected";
+                ret[jss::amount] = to_string(e.amt_);
+                ret[jss::destination_account] = to_string(e.dst_);
+                ret[jss::hash] = strHex(e.txnHash_);
+                ret[jss::sequence] = e.txnSeq_;
+                ret["rpc_order"] = e.rpcOrder_;
+            }
+            else if constexpr (std::is_same_v<
+                                   eventType,
+                                   event::XChainTransferResult>)
+            {
+                ret[jss::type] = "xchain_transfer_result";
+                ret[jss::amount] = to_string(e.amt_);
+                ret[jss::destination_account] = to_string(e.dst_);
+                ret[jss::hash] = strHex(e.txnHash_);
+                ret["triggering_tx_hash"] = strHex(e.triggeringTxnHash_);
+                ret[jss::sequence] = e.txnSeq_;
+                ret[jss::result] = transHuman(e.ter_);
+                ret["rpc_order"] = e.rpcOrder_;
+            }
+            else
+            {
+                ret[jss::type] = "other_event";
+            }
+            return ret;
+        },
+        event);
+}
+
+}  // namespace detail
+
+Json::Value
+InitialSync::getInfo() const
+{
+    Json::Value ret{Json::objectValue};
+    {
+        std::lock_guard l{m_};
+        ret["last_x_chain_txn_with_result"] = lastXChainTxnWithResult_
+            ? strHex(*lastXChainTxnWithResult_)
+            : "None";
+        Json::Value triggerinTxns{Json::arrayValue};
+        for (auto const& h : seenTriggeringTxns_)
+        {
+            triggerinTxns.append(strHex(h));
+        }
+        ret["seen_triggering_txns"] = triggerinTxns;
+        ret["needs_last_x_chain_txn"] = needsLastXChainTxn_;
+        ret["needs_other_chain_last_x_chain_txn"] =
+            needsOtherChainLastXChainTxn_;
+        ret["acquiring_historic_data"] = acquiringHistoricData_;
+        ret["needs_replay_start_txn_hash"] = needsReplayStartTxnHash_;
+    }
+    return ret;
+}
+
+}  // namespace sidechain
+
+}  // namespace ripple

src/ripple/app/sidechain/impl/InitialSync.h

@@ -0,0 +1,214 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_INITIALSYNC_H_INCLUDED
+#define RIPPLE_SIDECHAIN_IMPL_INITIALSYNC_H_INCLUDED
+
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/basics/ThreadSaftyAnalysis.h>
+#include <ripple/basics/UnorderedContainers.h>
+#include <ripple/beast/utility/Journal.h>
+
+#include <map>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+class WebsocketClient;
+
+// This class handles the logic of getting a federator that joins the network
+// into a "normal" state of handling new cross chain transactions and results.
+// There will be two instance of this class, one for the main chain and one for
+// the side chain.
+//
+// When a federator joins the network of other federators, the network can be in
+// one of three states:
+//
+// 1) The initial sidechain startup.
+// 2) Running normally with a quorum of federators. This federator that's
+//    joining just increases the quorum.
+// 3) A stalled sidechain without enough federators to make forward progress.
+//    This federator may or may not increase the quorum enough so cross chain
+//    transactions can continue. In the meantime, cross chain transactions may
+//    continue to accumulate.
+//
+// No matter the state of the federators network, connecting to the network goes
+// through the same steps. There are two instances of this class, one
+// for the main chain and one for the side chain.
+//
+// The RPC command used to fetch transactions will initially be configured to
+// retrieve both historical transactions and new transactions. Once the
+// information needed from the historical transactions are retrieved, it will be
+// changed to only stream new transactions.
+//
+// There are two states this class can be in: pre-replay and post-replay. In
+// pre-replay mode, the class collects information from both historic and
+// transactions that will be used for helping the this instance and the "other"
+// instance of this class known when to stop collecting historic data, as well
+// as collecting transactions for replaying.
+//
+// Historic data needs to be collected until:
+//
+// 1) The most recent historic `XChainTransferResult` event is detected (or the
+// account's first transaction is detected). This is used to inform the "other"
+// instance of this class which `XChainTransferDetected` event is the first that
+// may need to be replayed. Since the previous `XChainTransferDetected` events
+// have results on the other chain, we can definitively say the federators have
+// handled these events and they don't need to be replayed.
+//
+// 2) Once the `lastXChainTxnWithResult_` is know, historic transactions need be
+// acquired until that transaction is seen on a `XChainTransferDetected` event.
+//
+// Once historic data collection has completed, the collected transactions are
+// replayed to the federator, and this class is not longer needed. All new
+// transactions should simply be forwarded to the federator.
+//
+class InitialSync
+{
+private:
+    std::weak_ptr<Federator> federator_;
+    // Holds all the eventsseen so far. These events will be replayed to the
+    // federator upon switching to `normal` mode. Will be cleared while
+    // replaying.
+    std::map<std::int32_t, FederatorEvent> GUARDED_BY(m_) pendingEvents_;
+    // Holds all triggering cross chain transactions seen so far. This is used
+    // to determine if the `XChainTransferDetected` event with the
+    // `lastXChainTxnWithResult_` has has been seen or not. Will be cleared
+    // while replaying
+    hash_set<uint256> GUARDED_BY(m_) seenTriggeringTxns_;
+    // Hash of the last cross chain transaction on this chain with a result on
+    // the "other" chain. Note: this is set when the `InitialSync` for the
+    // "other" chain encounters the transaction.
+    std::optional<uint256> GUARDED_BY(m_) lastXChainTxnWithResult_;
+    // Track if we need to keep acquiring historic transactions for the
+    // `lastXChainTxnWithResult_`. This is true if the lastXChainTxnWithResult_
+    // is unknown, or it is known and the transaction is not part of that
+    // collection yet.
+    bool GUARDED_BY(m_) needsLastXChainTxn_{true};
+    // Track if we need to keep acquiring historic transactions for the other
+    // chain's `lastXChainTxnWithResult_` hash value. This is true if no
+    // cross chain transaction results are known and the first historical
+    // transaction has not been encountered.
+    bool GUARDED_BY(m_) needsOtherChainLastXChainTxn_{true};
+    // Track if the transaction to start the replay from is known. This is true
+    // until `lastXChainTxnWithResult_` is known and the other listener has not
+    // encountered the first historical transaction
+    bool GUARDED_BY(m_) needsReplayStartTxnHash_{true};
+    // True if the historical transactions have been replayed to the federator
+    bool GUARDED_BY(m_) hasReplayed_{false};
+    // Track the state of the transaction data we are acquiring.
+    // If this is `false`, only new transactions events will be streamed.
+    // Note: there will be a period where this is `false` but historic txns will
+    // continue to come in until the rpc command has responded to the request to
+    // shut off historic data.
+    bool GUARDED_BY(m_) acquiringHistoricData_{true};
+    // All transactions before "DisableMasterKey" are setup transactions and
+    // should be ignored
+    std::optional<std::uint32_t> GUARDED_BY(m_) disableMasterKeySeq_;
+    bool const isMainchain_;
+    mutable std::mutex m_;
+    beast::Journal j_;
+    // See description on class for explanation of states
+
+public:
+    InitialSync(
+        std::weak_ptr<Federator> federator,
+        bool isMainchain,
+        beast::Journal j);
+
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    setLastXChainTxnWithResult(uint256 const& hash) EXCLUDES(m_);
+
+    // There have not been any cross chain transactions.
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    setNoLastXChainTxnWithResult() EXCLUDES(m_);
+
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    onEvent(event::XChainTransferDetected&& e) EXCLUDES(m_);
+
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    onEvent(event::XChainTransferResult&& e) EXCLUDES(m_);
+
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    onEvent(event::RefundTransferResult&& e) EXCLUDES(m_);
+
+    // Return `hasReplayed_`. This is used to determine if events should
+    // continue to be routed to this object. Once replayed, events can be
+    // processed normally.
+    [[nodiscard]] bool
+    onEvent(event::StartOfHistoricTransactions&& e) EXCLUDES(m_);
+    // Return `hasReplayed_`.
+    [[nodiscard]] bool
+    onEvent(event::TicketCreateTrigger&& e) EXCLUDES(m_);
+    // Return `hasReplayed_`.
+    [[nodiscard]] bool
+    onEvent(event::TicketCreateResult&& e) EXCLUDES(m_);
+    // Return `hasReplayed_`.
+    [[nodiscard]] bool
+    onEvent(event::DepositAuthResult&& e) EXCLUDES(m_);
+    [[nodiscard]] bool
+    onEvent(event::BootstrapTicket&& e) EXCLUDES(m_);
+    [[nodiscard]] bool
+    onEvent(event::DisableMasterKeyResult&& e) EXCLUDES(m_);
+
+    Json::Value
+    getInfo() const EXCLUDES(m_);
+
+private:
+    // Replay when historical transactions are no longer being acquired,
+    // and the transaction to start the replay from is known.
+    bool
+    canReplay(std::lock_guard<std::mutex> const&) const REQUIRES(m_);
+    void
+    replay(std::lock_guard<std::mutex> const&) REQUIRES(m_);
+    bool
+    hasTransaction(uint256 const& txnHash, std::lock_guard<std::mutex> const&)
+        const REQUIRES(m_);
+    void
+    stopHistoricalTxns(std::lock_guard<std::mutex> const&) REQUIRES(m_);
+    template <class T>
+    bool
+    onTriggerEvent(T&& e);
+    template <class T>
+    bool
+    onResultEvent(T&& e, std::uint32_t seqTook);
+    void
+    done();
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/MainchainListener.cpp

@@ -0,0 +1,144 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/MainchainListener.h>
+
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/app/sidechain/impl/WebsocketClient.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/XRPAmount.h>
+#include <ripple/json/Output.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/jss.h>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+
+MainchainListener::MainchainListener(
+    AccountID const& account,
+    std::weak_ptr<Federator>&& federator,
+    beast::Journal j)
+    : ChainListener(
+          ChainListener::IsMainchain::yes,
+          account,
+          std::move(federator),
+          j)
+{
+}
+void
+MainchainListener::onMessage(Json::Value const& msg)
+{
+    auto callbackOpt = [&]() -> std::optional<RpcCallback> {
+        if (msg.isMember(jss::id) && msg[jss::id].isIntegral())
+        {
+            auto callbackId = msg[jss::id].asUInt();
+            std::lock_guard lock(callbacksMtx_);
+            auto i = callbacks_.find(callbackId);
+            if (i != callbacks_.end())
+            {
+                auto cb = i->second;
+                callbacks_.erase(i);
+                return cb;
+            }
+        }
+        return {};
+    }();
+
+    if (callbackOpt)
+    {
+        JLOG(j_.trace()) << "Mainchain onMessage, reply to a callback: " << msg;
+        assert(msg.isMember(jss::result));
+        (*callbackOpt)(msg[jss::result]);
+    }
+    else
+    {
+        processMessage(msg);
+    }
+}
+
+void
+MainchainListener::init(
+    boost::asio::io_service& ios,
+    boost::asio::ip::address const& ip,
+    std::uint16_t port)
+{
+    wsClient_ = std::make_unique<WebsocketClient>(
+        [self = shared_from_this()](Json::Value const& msg) {
+            self->onMessage(msg);
+        },
+        ios,
+        ip,
+        port,
+        /*headers*/ std::unordered_map<std::string, std::string>{},
+        j_);
+
+    Json::Value params;
+    params[jss::account_history_tx_stream] = Json::objectValue;
+    params[jss::account_history_tx_stream][jss::account] = doorAccountStr_;
+    send("subscribe", params);
+}
+
+// destructor must be defined after WebsocketClient size is known (i.e. it can
+// not be defaulted in the header or the unique_ptr declration of
+// WebsocketClient won't work)
+MainchainListener::~MainchainListener() = default;
+
+void
+MainchainListener::shutdown()
+{
+    if (wsClient_)
+        wsClient_->shutdown();
+}
+
+std::uint32_t
+MainchainListener::send(std::string const& cmd, Json::Value const& params)
+{
+    return wsClient_->send(cmd, params);
+}
+
+void
+MainchainListener::stopHistoricalTxns()
+{
+    Json::Value params;
+    params[jss::stop_history_tx_only] = true;
+    params[jss::account_history_tx_stream] = Json::objectValue;
+    params[jss::account_history_tx_stream][jss::account] = doorAccountStr_;
+    send("unsubscribe", params);
+}
+
+void
+MainchainListener::send(
+    std::string const& cmd,
+    Json::Value const& params,
+    RpcCallback onResponse)
+{
+    JLOGV(
+        j_.trace(), "Mainchain send", jv("command", cmd), jv("params", params));
+
+    auto id = wsClient_->send(cmd, params);
+    std::lock_guard lock(callbacksMtx_);
+    callbacks_.emplace(id, onResponse);
+}
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/MainchainListener.h

@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_MAINCHAINLISTENER_H_INCLUDED
+#define RIPPLE_SIDECHAIN_IMPL_MAINCHAINLISTENER_H_INCLUDED
+
+#include <ripple/app/sidechain/impl/ChainListener.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/protocol/AccountID.h>
+
+#include <boost/asio/io_service.hpp>
+#include <boost/asio/ip/address.hpp>
+
+#include <memory>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+class WebsocketClient;
+
+class MainchainListener : public ChainListener,
+                          public std::enable_shared_from_this<MainchainListener>
+{
+    std::unique_ptr<WebsocketClient> wsClient_;
+
+    mutable std::mutex callbacksMtx_;
+    std::map<std::uint32_t, RpcCallback> GUARDED_BY(callbacksMtx_) callbacks_;
+
+    void
+    onMessage(Json::Value const& msg) EXCLUDES(callbacksMtx_);
+
+public:
+    MainchainListener(
+        AccountID const& account,
+        std::weak_ptr<Federator>&& federator,
+        beast::Journal j);
+
+    ~MainchainListener();
+
+    void
+    init(
+        boost::asio::io_service& ios,
+        boost::asio::ip::address const& ip,
+        std::uint16_t port);
+
+    // Returns command id that will be returned in the response
+    std::uint32_t
+    send(std::string const& cmd, Json::Value const& params)
+        EXCLUDES(callbacksMtx_);
+
+    void
+    shutdown();
+
+    void
+    stopHistoricalTxns();
+
+    void
+    send(
+        std::string const& cmd,
+        Json::Value const& params,
+        RpcCallback onResponse) override;
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/SidechainListener.cpp

@@ -0,0 +1,130 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/SidechainListener.h>
+
+#include <ripple/app/main/Application.h>
+#include <ripple/app/misc/NetworkOPs.h>
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/XRPAmount.h>
+#include <ripple/json/Output.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/jss.h>
+#include <ripple/resource/Fees.h>
+#include <ripple/rpc/Context.h>
+#include <ripple/rpc/RPCHandler.h>
+#include <ripple/rpc/impl/RPCHelpers.h>
+
+namespace ripple {
+namespace sidechain {
+
+SidechainListener::SidechainListener(
+    Source& source,
+    AccountID const& account,
+    std::weak_ptr<Federator>&& federator,
+    Application& app,
+    beast::Journal j)
+    : InfoSub(source)
+    , ChainListener(
+          ChainListener::IsMainchain::no,
+          account,
+          std::move(federator),
+          j)
+    , app_(app)
+{
+}
+
+void
+SidechainListener::init(NetworkOPs& netOPs)
+{
+    auto e = netOPs.subAccountHistory(shared_from_this(), doorAccount_);
+    if (e != rpcSUCCESS)
+        LogicError("Could not subscribe to side chain door account history.");
+}
+
+void
+SidechainListener::send(Json::Value const& msg, bool)
+{
+    processMessage(msg);
+}
+
+void
+SidechainListener::stopHistoricalTxns(NetworkOPs& netOPs)
+{
+    netOPs.unsubAccountHistory(
+        shared_from_this(), doorAccount_, /*history only*/ true);
+}
+
+void
+SidechainListener::send(
+    std::string const& cmd,
+    Json::Value const& params,
+    RpcCallback onResponse)
+{
+    std::weak_ptr<SidechainListener> selfWeak = shared_from_this();
+    auto job = [cmd, params, onResponse, selfWeak](Job&) {
+        auto self = selfWeak.lock();
+        if (!self)
+            return;
+
+        JLOGV(
+            self->j_.trace(),
+            "Sidechain send",
+            jv("command", cmd),
+            jv("params", params));
+
+        Json::Value const request = [&] {
+            Json::Value r(params);
+            r[jss::method] = cmd;
+            r[jss::jsonrpc] = "2.0";
+            r[jss::ripplerpc] = "2.0";
+            return r;
+        }();
+        Resource::Charge loadType = Resource::feeReferenceRPC;
+        Resource::Consumer c;
+        RPC::JsonContext context{
+            {self->j_,
+             self->app_,
+             loadType,
+             self->app_.getOPs(),
+             self->app_.getLedgerMaster(),
+             c,
+             Role::ADMIN,
+             {},
+             {},
+             RPC::apiMaximumSupportedVersion},
+            std::move(request)};
+
+        Json::Value jvResult;
+        RPC::doCommand(context, jvResult);
+        JLOG(self->j_.trace()) << "Sidechain response: " << jvResult;
+        if (self->app_.config().standalone())
+            self->app_.getOPs().acceptLedger();
+        onResponse(jvResult);
+    };
+    app_.getJobQueue().addJob(jtRPC, "federator rpc", job);
+}
+
+}  // namespace sidechain
+
+}  // namespace ripple

src/ripple/app/sidechain/impl/SidechainListener.h

@@ -0,0 +1,74 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_SIDECHAINLISTENER_H_INCLUDED
+#define RIPPLE_SIDECHAIN_IMPL_SIDECHAINLISTENER_H_INCLUDED
+
+#include <ripple/app/sidechain/impl/ChainListener.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/net/InfoSub.h>
+#include <ripple/protocol/AccountID.h>
+
+#include <memory>
+
+namespace ripple {
+
+class NetworkOPs;
+class Application;
+
+namespace sidechain {
+
+class Federator;
+
+class SidechainListener : public InfoSub,
+                          public ChainListener,
+                          public std::enable_shared_from_this<SidechainListener>
+{
+    Application& app_;
+
+public:
+    SidechainListener(
+        Source& source,
+        AccountID const& account,
+        std::weak_ptr<Federator>&& federator,
+        Application& app,
+        beast::Journal j);
+
+    void
+    init(NetworkOPs& netOPs);
+
+    ~SidechainListener() = default;
+
+    void
+    send(Json::Value const& msg, bool) override;
+
+    void
+    stopHistoricalTxns(NetworkOPs& netOPs);
+
+    void
+    send(
+        std::string const& cmd,
+        Json::Value const& params,
+        RpcCallback onResponse) override;
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/SignatureCollector.cpp

@@ -0,0 +1,325 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/SignatureCollector.h>
+
+#include <ripple/app/main/Application.h>
+#include <ripple/app/misc/HashRouter.h>
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/impl/SignerList.h>
+#include <ripple/basics/Slice.h>
+#include <ripple/core/Job.h>
+#include <ripple/core/JobQueue.h>
+#include <ripple/json/json_value.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/overlay/Message.h>
+#include <ripple/overlay/Overlay.h>
+#include <ripple/overlay/Peer.h>
+#include <ripple/protocol/STAccount.h>
+#include <ripple/protocol/STObject.h>
+#include <ripple/protocol/STParsedJSON.h>
+#include <ripple/protocol/Serializer.h>
+#include <mutex>
+
+namespace ripple {
+namespace sidechain {
+
+std::chrono::seconds messageExpire = std::chrono::minutes{10};
+
+uint256
+computeMessageSuppression(MessageId const& mId, Slice const& signature)
+{
+    Serializer s(128);
+    s.addBitString(mId);
+    s.addVL(signature);
+    return s.getSHA512Half();
+}
+
+SignatureCollector::SignatureCollector(
+    bool isMainChain,
+    SecretKey const& mySecKey,
+    PublicKey const& myPubKey,
+    beast::abstract_clock<std::chrono::steady_clock>& c,
+    SignerList& signers,
+    Federator& federator,
+    Application& app,
+    beast::Journal j)
+    : isMainChain_(isMainChain)
+    , mySecKey_(mySecKey)
+    , myPubKey_(myPubKey)
+    , messages_(c)
+    , signers_(signers)
+    , federator_(federator)
+    , app_(app)
+    , j_(j)
+{
+}
+
+void
+SignatureCollector::signAndSubmit(Json::Value const& txJson)
+{
+    auto job = [tx = txJson,
+                myPK = myPubKey_,
+                mySK = mySecKey_,
+                chain =
+                    isMainChain_ ? Federator::mainChain : Federator::sideChain,
+                f = federator_.weak_from_this(),
+                j = j_](Job&) mutable {
+        auto federator = f.lock();
+        if (!federator)
+            return;
+
+        STParsedJSONObject parsed(std::string(jss::tx_json), tx);
+        if (parsed.object == std::nullopt)
+        {
+            JLOGV(j.fatal(), "cannot parse transaction", jv("tx", tx));
+            assert(0);
+            return;
+        }
+        try
+        {
+            parsed.object->setFieldVL(sfSigningPubKey, Slice(nullptr, 0));
+            STTx tx(std::move(parsed.object.value()));
+
+            MessageId mId{tx.getSigningHash()};
+            Buffer sig{tx.getMultiSignature(calcAccountID(myPK), myPK, mySK)};
+            federator->getSignatureCollector(chain).processSig(
+                mId, myPK, std::move(sig), std::move(tx));
+        }
+        catch (...)
+        {
+            JLOGV(j.fatal(), "invalid transaction", jv("tx", tx));
+            assert(0);
+        }
+    };
+
+    app_.getJobQueue().addJob(jtFEDERATORSIGNATURE, "federator signature", job);
+}
+
+bool
+SignatureCollector::processSig(
+    MessageId const& mId,
+    PublicKey const& pk,
+    Buffer const& sig,
+    std::optional<STTx> const& txOpt)
+{
+    JLOGV(
+        j_.trace(),
+        "processSig",
+        jv("public key", strHex(pk)),
+        jv("message", mId));
+    if (!signers_.isFederator(pk))
+    {
+        return false;
+    }
+
+    auto valid = addSig(mId, pk, sig, txOpt);
+    if (txOpt)
+        shareSig(mId, sig);
+    return valid;
+}
+
+void
+SignatureCollector::expire()
+{
+    std::lock_guard lock(mtx_);
+    beast::expire(messages_, messageExpire);
+}
+
+bool
+SignatureCollector::addSig(
+    MessageId const& mId,
+    PublicKey const& pk,
+    Buffer const& sig,
+    std::optional<STTx> const& txOpt)
+{
+    JLOGV(
+        j_.trace(),
+        "addSig",
+        jv("message", mId),
+        jv("public key", strHex(pk)),
+        jv("sig", strHex(sig)));
+
+    std::lock_guard lock(mtx_);
+    auto txi = messages_.find(mId);
+    if (txi == messages_.end())
+    {
+        PeerSignatureMap sigMaps;
+        sigMaps.emplace(pk, sig);
+        MultiSigMessage m{sigMaps, txOpt};
+        messages_.emplace(mId, std::move(m));
+        return true;
+    }
+
+    auto const verifySingle = [&](PublicKey const& pk,
+                                  Buffer const& sig) -> bool {
+        Serializer s;
+        s.add32(HashPrefix::txMultiSign);
+        (*txi->second.tx_).addWithoutSigningFields(s);
+        s.addBitString(calcAccountID(pk));
+        return verify(pk, s.slice(), sig, true);
+    };
+
+    MultiSigMessage& message = txi->second;
+    if (txOpt)
+    {
+        message.tx_.emplace(std::move(*txOpt));
+
+        for (auto i = message.sigMaps_.begin(); i != message.sigMaps_.end();)
+        {
+            if (verifySingle(i->first, i->second))
+                ++i;
+            else
+            {
+                JLOGV(
+                    j_.trace(),
+                    "verifySingle failed",
+                    jv("public key", strHex(i->first)));
+                message.sigMaps_.erase(i);
+            }
+        }
+    }
+    else
+    {
+        if (message.tx_)
+        {
+            if (!verifySingle(pk, sig))
+            {
+                JLOGV(
+                    j_.trace(),
+                    "verifySingle failed",
+                    jv("public key", strHex(pk)));
+                return false;
+            }
+        }
+    }
+    message.sigMaps_.emplace(pk, sig);
+
+    if (!message.submitted_ && message.tx_ &&
+        message.sigMaps_.size() >= signers_.quorum())
+    {
+        // message.submitted_ = true;
+        submit(mId, lock);
+    }
+    return true;
+}
+
+void
+SignatureCollector::shareSig(MessageId const& mId, Buffer const& sig)
+{
+    JLOGV(j_.trace(), "shareSig", jv("message", mId), jv("sig", strHex(sig)));
+
+    std::shared_ptr<Message> toSend = [&]() -> std::shared_ptr<Message> {
+        protocol::TMFederatorAccountCtrlSignature m;
+        m.set_chain(isMainChain_ ? ::protocol::fct_MAIN : ::protocol::fct_SIDE);
+        m.set_publickey(myPubKey_.data(), myPubKey_.size());
+        m.set_messageid(mId.data(), mId.size());
+        m.set_signature(sig.data(), sig.size());
+
+        return std::make_shared<Message>(
+            m, protocol::mtFederatorAccountCtrlSignature);
+    }();
+
+    Overlay& overlay = app_.overlay();
+    HashRouter& hashRouter = app_.getHashRouter();
+    auto const suppression = computeMessageSuppression(mId, sig);
+
+    overlay.foreach([&](std::shared_ptr<Peer> const& p) {
+        hashRouter.addSuppressionPeer(suppression, p->id());
+        JLOGV(
+            j_.trace(),
+            "sending signature to peer",
+            jv("pid", p->id()),
+            jv("mid", mId));
+        p->send(toSend);
+    });
+}
+
+void
+SignatureCollector::submit(
+    MessageId const& mId,
+    std::lock_guard<std::mutex> const&)
+{
+    JLOGV(j_.trace(), "submit", jv("message", mId));
+
+    assert(messages_.find(mId) != messages_.end());
+    auto& message = messages_[mId];
+    assert(!message.submitted_);
+    message.submitted_ = true;
+
+    STArray signatures;
+    auto sigCount = message.sigMaps_.size();
+    assert(sigCount >= signers_.quorum());
+    signatures.reserve(sigCount);
+
+    for (auto const& item : message.sigMaps_)
+    {
+        STObject obj{sfSigner};
+        obj[sfAccount] = calcAccountID(item.first);
+        obj[sfSigningPubKey] = item.first;
+        obj[sfTxnSignature] = item.second;
+        signatures.push_back(std::move(obj));
+    };
+
+    std::sort(
+        signatures.begin(),
+        signatures.end(),
+        [](STObject const& lhs, STObject const& rhs) {
+            return lhs[sfAccount] < rhs[sfAccount];
+        });
+
+    message.tx_->setFieldArray(sfSigners, std::move(signatures));
+
+    auto sp = message.tx_->getSeqProxy();
+    if (sp.isTicket())
+    {
+        Json::Value r;
+        r[jss::tx_blob] = strHex(message.tx_->getSerializer().peekData());
+
+        JLOGV(j_.trace(), "submit", jv("tx", r));
+        auto callback = [&](Json::Value const& response) {
+            JLOGV(
+                j_.trace(),
+                "SignatureCollector::submit ",
+                jv("response", response));
+        };
+        rpcChannel_->send("submit", r, callback);
+    }
+    else
+    {
+        JLOGV(
+            j_.trace(),
+            "forward to federator to submit",
+            jv("tx", strHex(message.tx_->getSerializer().peekData())));
+        federator_.addTxToSend(
+            (isMainChain_ ? Federator::ChainType::mainChain
+                          : Federator::ChainType::sideChain),
+            sp.value(),
+            *(message.tx_));
+    }
+}
+
+void
+SignatureCollector::setRpcChannel(std::shared_ptr<ChainListener> channel)
+{
+    rpcChannel_ = std::move(channel);
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/SignatureCollector.h

@@ -0,0 +1,146 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_SIGNATURE_COLLECTOR_H
+#define RIPPLE_SIDECHAIN_IMPL_SIGNATURE_COLLECTOR_H
+
+#include <ripple/app/sidechain/impl/ChainListener.h>
+#include <ripple/basics/Buffer.h>
+#include <ripple/basics/UnorderedContainers.h>
+#include <ripple/beast/clock/abstract_clock.h>
+#include <ripple/beast/container/aged_unordered_map.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/protocol/PublicKey.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/SecretKey.h>
+#include <ripple.pb.h>
+
+#include <mutex>
+#include <optional>
+#include <set>
+
+namespace Json {
+class Value;
+}
+
+namespace ripple {
+
+class Application;
+
+namespace sidechain {
+
+class SignerList;
+class Federator;
+
+using PeerSignatureMap = hash_map<PublicKey, Buffer>;
+using MessageId = uint256;
+struct MultiSigMessage
+{
+    PeerSignatureMap sigMaps_;
+    std::optional<STTx> tx_;
+    bool submitted_ = false;
+};
+
+class SignatureCollector
+{
+    std::shared_ptr<ChainListener> rpcChannel_;
+    bool const isMainChain_;
+    SecretKey const mySecKey_;
+    PublicKey const myPubKey_;
+    mutable std::mutex mtx_;
+    beast::aged_unordered_map<
+        MessageId,
+        MultiSigMessage,
+        std::chrono::steady_clock,
+        beast::uhash<>>
+        GUARDED_BY(mtx_) messages_;
+
+    SignerList& signers_;
+    Federator& federator_;
+    Application& app_;
+    beast::Journal j_;
+
+public:
+    SignatureCollector(
+        bool isMainChain,
+        SecretKey const& mySecKey,
+        PublicKey const& myPubKey,
+        beast::abstract_clock<std::chrono::steady_clock>& c,
+        SignerList& signers,
+        Federator& federator,
+        Application& app,
+        beast::Journal j);
+
+    /**
+     * sign the tx, and share with network
+     * once quorum signatures are collected, the tx will be submitted
+     * @param tx the transaction to be signed and later submitted
+     */
+    void
+    signAndSubmit(Json::Value const& tx);
+
+    /**
+     * verify the signature and remember it.
+     * If quorum signatures are collected for the same MessageId,
+     * a tx will be submitted.
+     *
+     * @param mId identify the tx
+     * @param pk public key of signer
+     * @param sig signature
+     * @param txOpt the transaction, only used by the local node
+     * @return if the signature is from a federator
+     */
+    bool
+    processSig(
+        MessageId const& mId,
+        PublicKey const& pk,
+        Buffer const& sig,
+        std::optional<STTx> const& txOpt);
+
+    /**
+     * remove stale signatures
+     */
+    void
+    expire() EXCLUDES(mtx_);  // TODO retry logic
+
+    void
+    setRpcChannel(std::shared_ptr<ChainListener> channel);
+
+private:
+    // verify a signature (if it is from a peer) and add to a collection
+    bool
+    addSig(
+        MessageId const& mId,
+        PublicKey const& pk,
+        Buffer const& sig,
+        std::optional<STTx> const& txOpt) EXCLUDES(mtx_);
+
+    // share a signature to the network
+    void
+    shareSig(MessageId const& mId, Buffer const& sig);
+    // submit a tx since it collected quorum signatures
+    void
+    submit(MessageId const& mId, std::lock_guard<std::mutex> const&)
+        REQUIRES(mtx_);
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/SignerList.cpp

@@ -0,0 +1,51 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/SignerList.h>
+
+namespace ripple {
+namespace sidechain {
+
+SignerList::SignerList(
+    AccountID const& account,
+    hash_set<PublicKey> const& signers,
+    beast::Journal j)
+    : account_(account)
+    , signers_(signers)
+    , quorum_(static_cast<std::uint32_t>(std::ceil(signers.size() * 0.8)))
+    , j_(j)
+{
+    (void)j_;
+}
+
+bool
+SignerList::isFederator(PublicKey const& pk) const
+{
+    std::lock_guard<std::mutex> lock(mtx_);
+    return signers_.find(pk) != signers_.end();
+}
+
+std::uint32_t
+SignerList::quorum() const
+{
+    std::lock_guard<std::mutex> lock(mtx_);
+    return quorum_;
+}
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/SignerList.h

@@ -0,0 +1,63 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_SIGNER_LIST_H
+#define RIPPLE_SIDECHAIN_IMPL_SIGNER_LIST_H
+
+#include <ripple/basics/ThreadSaftyAnalysis.h>
+#include <ripple/basics/UnorderedContainers.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/PublicKey.h>
+#include <ripple.pb.h>
+
+#include <mutex>
+
+namespace ripple {
+namespace sidechain {
+
+/**
+ * grow to handle singer list changes
+ */
+class SignerList
+{
+    AccountID const account_;
+    mutable std::mutex mtx_;
+    hash_set<PublicKey> GUARDED_BY(mtx_) signers_;
+    std::uint32_t GUARDED_BY(mtx_) quorum_;
+    beast::Journal j_;
+
+public:
+    SignerList(
+        AccountID const& account,
+        hash_set<PublicKey> const& signers,
+        beast::Journal j);
+    ~SignerList() = default;
+
+    bool
+    isFederator(PublicKey const& pk) const EXCLUDES(mtx_);
+
+    std::uint32_t
+    quorum() const EXCLUDES(mtx_);
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/TicketHolder.cpp

@@ -0,0 +1,791 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/TicketHolder.h>
+
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/app/sidechain/impl/SignatureCollector.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/StringUtilities.h>
+#include <ripple/json/json_writer.h>
+#include <ripple/protocol/STParsedJSON.h>
+
+namespace ripple {
+namespace sidechain {
+
+std::string
+TicketPurposeToStr(TicketPurpose tp)
+{
+    switch (tp)
+    {
+        case TicketPurpose::mainDoorKeeper:
+            return "mainDoorKeeper";
+        case TicketPurpose::sideDoorKeeper:
+            return "sideDoorKeeper";
+        case TicketPurpose::updateSignerList:
+            return "updateSignerList";
+        default:
+            break;
+    }
+    return "unknown";
+}
+
+TicketHolder::TicketHolder(
+    bool isMainChain,
+    AccountID const& account,
+    Federator& federator,
+    beast::Journal j)
+    : isMainChain_(isMainChain)
+    , accountStr_(toBase58(account))
+    , federator_(federator)
+    , j_(j)
+{
+}
+
+void
+TicketHolder::init()
+{
+    std::lock_guard lock(mtx_);
+    if (initData_.status_ != InitializeStatus::waitLedger)
+        return;
+    initData_.status_ = InitializeStatus::waitAccountObject;
+    rpcAccountObject();
+}
+
+std::optional<std::uint32_t>
+TicketHolder::getTicket(TicketPurpose purpose, PeekOrTake pt)
+{
+    std::lock_guard lock(mtx_);
+
+    if (initData_.status_ != InitializeStatus::initialized)
+    {
+        JLOGV(
+            j_.debug(),
+            "TicketHolder getTicket but ticket holder not initialized",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("purpose", TicketPurposeToStr(purpose)));
+
+        if (initData_.status_ == InitializeStatus::needToQueryTx)
+            rpcTx(lock);
+        return {};
+    }
+
+    auto index = static_cast<std::underlying_type_t<TicketPurpose>>(purpose);
+    if (tickets_[index].status_ == AutoRenewedTicket::Status::available)
+    {
+        if (pt == PeekOrTake::take)
+        {
+            JLOGV(
+                j_.trace(),
+                "getTicket",
+                jv("chain", (isMainChain_ ? "main" : "side")),
+                jv("seq", tickets_[index].seq_));
+            tickets_[index].status_ = AutoRenewedTicket::Status::taken;
+        }
+        return tickets_[index].seq_;
+    }
+    if (pt == PeekOrTake::take)
+    {
+        JLOGV(
+            j_.trace(),
+            "getTicket",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("no ticket for ", TicketPurposeToStr(purpose)));
+    }
+    return {};
+}
+
+void
+TicketHolder::onEvent(event::TicketCreateResult const& e)
+{
+    std::lock_guard lock(mtx_);
+    if (initData_.status_ != InitializeStatus::initialized)
+    {
+        JLOG(j_.trace()) << "TicketHolder queues an event";
+        initData_.toReplay_.emplace(e);
+        return;
+    }
+    processEvent(e, lock);
+}
+
+void
+TicketHolder::onEvent(event::BootstrapTicket const& e)
+{
+    std::lock_guard lock(mtx_);
+    if (initData_.status_ != InitializeStatus::initialized)
+    {
+        JLOG(j_.trace()) << "TicketHolder queues an event";
+        initData_.bootstrapTicketToReplay_.emplace(e);
+        return;
+    }
+    processEvent(e, lock);
+}
+
+Json::Value
+TicketHolder::getInfo() const
+{
+    Json::Value ret{Json::objectValue};
+    {
+        std::lock_guard lock{mtx_};
+        if (initData_.status_ == InitializeStatus::initialized)
+        {
+            ret["initialized"] = "true";
+            Json::Value tickets{Json::arrayValue};
+            for (auto const& t : tickets_)
+            {
+                Json::Value tj{Json::objectValue};
+                tj["ticket_seq"] = t.seq_;
+                tj["status"] = t.status_ == AutoRenewedTicket::Status::taken
+                    ? "taken"
+                    : "available";
+                tickets.append(tj);
+            }
+            ret["tickets"] = tickets;
+        }
+        else
+        {
+            ret["initialized"] = "false";
+        }
+    }
+    return ret;
+}
+
+void
+TicketHolder::rpcAccountObject()
+{
+    Json::Value params = [&] {
+        Json::Value r;
+        r[jss::account] = accountStr_;
+        r[jss::ledger_index] = "validated";
+        r[jss::type] = "ticket";
+        r[jss::limit] = 250;
+        return r;
+    }();
+
+    rpcChannel_->send(
+        "account_objects",
+        params,
+        [isMainChain = isMainChain_,
+         f = federator_.weak_from_this()](Json::Value const& response) {
+            auto federator = f.lock();
+            if (!federator)
+                return;
+            federator->getTicketRunner().accountObjectResult(
+                isMainChain, response);
+        });
+}
+
+void
+TicketHolder::accountObjectResult(Json::Value const& rpcResult)
+{
+    auto ledgerNAccountObjectOpt =
+        [&]() -> std::optional<std::pair<std::uint32_t, Json::Value>> {
+        try
+        {
+            if (rpcResult.isMember(jss::error))
+            {
+                return {};
+            }
+            if (!rpcResult[jss::validated].asBool())
+            {
+                return {};
+            }
+            if (rpcResult[jss::account] != accountStr_)
+            {
+                return {};
+            }
+            if (!rpcResult[jss::ledger_index].isIntegral())
+            {
+                return {};
+            }
+            if (!rpcResult.isMember(jss::account_objects) ||
+                !rpcResult[jss::account_objects].isArray())
+            {
+                return {};
+            }
+            return std::make_pair(
+                rpcResult[jss::ledger_index].asUInt(),
+                rpcResult[jss::account_objects]);
+        }
+        catch (...)
+        {
+            return {};
+        }
+    }();
+
+    if (!ledgerNAccountObjectOpt)
+    {
+        // can reach here?
+        // should not since we only ask account_object after a validated ledger
+        JLOGV(j_.error(), "AccountObject", jv("result", rpcResult));
+        assert(false);
+        return;
+    }
+
+    auto& [ledgerIndex, accountObjects] = *ledgerNAccountObjectOpt;
+    std::lock_guard<std::mutex> lock(mtx_);
+    if (initData_.status_ != InitializeStatus::waitAccountObject)
+    {
+        JLOG(j_.warn()) << "unexpected AccountObject";
+        return;
+    }
+
+    initData_.ledgerIndex_ = ledgerIndex;
+    for (auto const& o : accountObjects)
+    {
+        if (!o.isMember("LedgerEntryType") ||
+            o["LedgerEntryType"] != jss::Ticket)
+            continue;
+        // the following fields are mandatory
+        uint256 txHash;
+        if (!txHash.parseHex(o["PreviousTxnID"].asString()))
+        {
+            JLOGV(
+                j_.error(),
+                "AccountObject cannot parse tx hash",
+                jv("result", rpcResult));
+            assert(false);
+            return;
+        }
+        std::uint32_t ticketSeq = o["TicketSequence"].asUInt();
+        if (initData_.tickets_.find(txHash) != initData_.tickets_.end())
+        {
+            JLOGV(
+                j_.error(),
+                "AccountObject cannot parse tx hash",
+                jv("result", rpcResult));
+            assert(false);
+            return;
+        }
+        initData_.tickets_.emplace(txHash, ticketSeq);
+        JLOGV(
+            j_.trace(),
+            "AccountObject, add",
+            jv("tx hash", txHash),
+            jv("ticketSeq", ticketSeq));
+    }
+
+    if (initData_.tickets_.empty())
+    {
+        JLOG(j_.debug()) << "Door account has no tickets in current ledger, "
+                            "unlikely but could happen";
+        replay(lock);
+    }
+    else
+    {
+        rpcTx(lock);
+    }
+}
+
+void
+TicketHolder::rpcTx(std::lock_guard<std::mutex> const&)
+{
+    assert(!initData_.tickets_.empty());
+    initData_.status_ = InitializeStatus::waitTx;
+    for (auto const& t : initData_.tickets_)
+    {
+        JLOG(j_.trace()) << "TicketHolder query tx " << t.first;
+        Json::Value params;
+        params[jss::transaction] = strHex(t.first);
+        rpcChannel_->send(
+            "tx",
+            params,
+            [isMainChain = isMainChain_,
+             f = federator_.weak_from_this()](Json::Value const& response) {
+                auto federator = f.lock();
+                if (!federator)
+                    return;
+                federator->getTicketRunner().txResult(isMainChain, response);
+            });
+    }
+}
+
+void
+TicketHolder::txResult(Json::Value const& rpcResult)
+{
+    std::lock_guard<std::mutex> lock(mtx_);
+    if (initData_.status_ != InitializeStatus::waitTx &&
+        initData_.status_ != InitializeStatus::needToQueryTx)
+        return;
+    auto txOpt = [&]() -> std::optional<std::pair<TicketPurpose, uint256>> {
+        try
+        {
+            if (rpcResult.isMember(jss::error))
+            {
+                return {};
+            }
+            if (rpcResult[jss::Account] != accountStr_)
+            {
+                return {};
+            }
+
+            if (rpcResult[jss::TransactionType] != "TicketCreate")
+            {
+                return {};
+            }
+
+            if (!rpcResult["SourceTag"].isIntegral())
+            {
+                return {};
+            }
+            std::uint32_t tp = rpcResult["SourceTag"].asUInt();
+            if (tp >= static_cast<std::underlying_type_t<TicketPurpose>>(
+                          TicketPurpose::TP_NumberOfItems))
+            {
+                return {};
+            }
+
+            uint256 txHash;
+            if (!txHash.parseHex(rpcResult[jss::hash].asString()))
+            {
+                return {};
+            }
+            return std::make_pair(static_cast<TicketPurpose>(tp), txHash);
+        }
+        catch (...)
+        {
+            return {};
+        }
+    }();
+
+    if (!txOpt)
+    {
+        JLOGV(
+            j_.warn(),
+            "TicketCreate can not be found or has wrong format",
+            jv("result", rpcResult));
+        if (initData_.status_ == InitializeStatus::waitTx)
+            initData_.status_ = InitializeStatus::needToQueryTx;
+        return;
+    }
+
+    auto [tPurpose, txHash] = *txOpt;
+    if (initData_.tickets_.find(txHash) == initData_.tickets_.end())
+    {
+        JLOGV(
+            j_.debug(),
+            "Repeated TicketCreate tx result",
+            jv("result", rpcResult));
+        return;
+    }
+
+    auto& ticket = initData_.tickets_[txHash];
+    JLOGV(
+        j_.trace(),
+        "TicketHolder txResult",
+        jv("purpose", TicketPurposeToStr(tPurpose)),
+        jv("txHash", txHash));
+
+    auto index = static_cast<std::underlying_type_t<TicketPurpose>>(tPurpose);
+    tickets_[index].seq_ = ticket;
+    tickets_[index].status_ = AutoRenewedTicket::Status::available;
+    initData_.tickets_.erase(txHash);
+
+    if (initData_.tickets_.empty())
+    {
+        replay(lock);
+    }
+}
+
+void
+TicketHolder::replay(std::lock_guard<std::mutex> const& lock)
+{
+    assert(initData_.tickets_.empty());
+    // replay bootstrap tickets first if any
+    while (!initData_.bootstrapTicketToReplay_.empty())
+    {
+        auto e = initData_.bootstrapTicketToReplay_.front();
+        processEvent(e, lock);
+        initData_.bootstrapTicketToReplay_.pop();
+    }
+
+    while (!initData_.toReplay_.empty())
+    {
+        auto e = initData_.toReplay_.front();
+        processEvent(e, lock);
+        initData_.toReplay_.pop();
+    }
+    initData_.status_ = InitializeStatus::initialized;
+    JLOG(j_.info()) << "TicketHolder initialized";
+}
+
+template <class E>
+void
+TicketHolder::processEvent(E const& e, std::lock_guard<std::mutex> const&)
+{
+    std::uint32_t const tSeq = e.txnSeq_ + 1;
+    if (e.sourceTag_ >= static_cast<std::underlying_type_t<TicketPurpose>>(
+                            TicketPurpose::TP_NumberOfItems))
+    {
+        JLOGV(
+            j_.error(),
+            "Wrong sourceTag",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("sourceTag", e.sourceTag_));
+        assert(false);
+        return;
+    }
+
+    auto purposeStr =
+        TicketPurposeToStr(static_cast<TicketPurpose>(e.sourceTag_));
+
+    if (e.ledgerIndex_ <= initData_.ledgerIndex_)
+    {
+        JLOGV(
+            j_.trace(),
+            "TicketHolder, ignoring an old ticket",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("ticket seq", tSeq),
+            jv("purpose", purposeStr));
+        return;
+    }
+
+    if (!e.success_)
+    {
+        JLOGV(
+            j_.error(),
+            "CreateTicket failed",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("ticket seq", tSeq),
+            jv("purpose", purposeStr));
+        assert(false);
+        return;
+    }
+
+    JLOGV(
+        j_.trace(),
+        "TicketHolder, got a ticket",
+        jv("chain", (isMainChain_ ? "main" : "side")),
+        jv("ticket seq", tSeq),
+        jv("purpose", purposeStr));
+
+    std::uint32_t const ticketPurposeToIndex = e.sourceTag_;
+
+    if (e.eventType() == event::EventType::bootstrap &&
+        tickets_[ticketPurposeToIndex].seq_ != 0)
+    {
+        JLOGV(
+            j_.error(),
+            "Got a bootstrap ticket too late",
+            jv("chain", (isMainChain_ ? "main" : "side")),
+            jv("ticket seq", tSeq),
+            jv("purpose", purposeStr));
+        assert(false);
+        return;
+    }
+    tickets_[ticketPurposeToIndex].seq_ = tSeq;
+    tickets_[ticketPurposeToIndex].status_ =
+        AutoRenewedTicket::Status::available;
+}
+
+void
+TicketHolder::setRpcChannel(std::shared_ptr<ChainListener> channel)
+{
+    rpcChannel_ = std::move(channel);
+}
+
+TicketRunner::TicketRunner(
+    const AccountID& mainAccount,
+    const AccountID& sideAccount,
+    Federator& federator,
+    beast::Journal j)
+    : mainAccountStr_(toBase58(mainAccount))
+    , sideAccountStr_(toBase58(sideAccount))
+    , federator_(federator)
+    , mainHolder_(true, mainAccount, federator, j)
+    , sideHolder_(false, sideAccount, federator, j)
+    , j_(j)
+{
+}
+
+void
+TicketRunner::setRpcChannel(
+    bool isMainChain,
+    std::shared_ptr<ChainListener> channel)
+{
+    if (isMainChain)
+        mainHolder_.setRpcChannel(std::move(channel));
+    else
+        sideHolder_.setRpcChannel(std::move(channel));
+}
+
+void
+TicketRunner::init(bool isMainChain)
+{
+    if (isMainChain)
+        mainHolder_.init();
+    else
+        sideHolder_.init();
+}
+
+void
+TicketRunner::accountObjectResult(
+    bool isMainChain,
+    Json::Value const& rpcResult)
+{
+    if (isMainChain)
+        mainHolder_.accountObjectResult(rpcResult);
+    else
+        sideHolder_.accountObjectResult(rpcResult);
+}
+
+void
+TicketRunner::txResult(bool isMainChain, Json::Value const& rpcResult)
+{
+    if (isMainChain)
+        mainHolder_.txResult(rpcResult);
+    else
+        sideHolder_.txResult(rpcResult);
+}
+
+bool
+TicketRunner::trigger(
+    TicketPurpose purpose,
+    std::optional<Json::Value> const& mainTxJson,
+    std::optional<Json::Value> const& sideTxJson)
+{
+    if (!mainTxJson && !sideTxJson)
+    {
+        assert(false);
+        return false;
+    }
+
+    auto ticketPair =
+        [&]() -> std::optional<std::pair<std::uint32_t, std::uint32_t>> {
+        std::lock_guard<std::mutex> lock(mtx_);
+        if (!mainHolder_.getTicket(purpose, TicketHolder::PeekOrTake::peek) ||
+            !sideHolder_.getTicket(purpose, TicketHolder::PeekOrTake::peek))
+        {
+            JLOG(j_.trace()) << "TicketRunner tickets no ready";
+            return {};
+        }
+        auto mainTicket =
+            mainHolder_.getTicket(purpose, TicketHolder::PeekOrTake::take);
+        auto sideTicket =
+            sideHolder_.getTicket(purpose, TicketHolder::PeekOrTake::take);
+        assert(mainTicket && sideTicket);
+        JLOGV(
+            j_.trace(),
+            "TicketRunner trigger",
+            jv("main ticket", *mainTicket),
+            jv("side ticket", *sideTicket),
+            jv("purpose", TicketPurposeToStr(purpose)));
+        return {{*mainTicket, *sideTicket}};
+    }();
+
+    if (!ticketPair)
+        return false;
+
+    auto sendTriggerTx = [&](std::string const& accountStr,
+                             std::uint32_t ticketSequence,
+                             std::optional<Json::Value> const& memoJson,
+                             SignatureCollector& signatureCollector) {
+        XRPAmount const fee{Federator::accountControlTxFee};
+        Json::Value txJson;
+        txJson[jss::TransactionType] = "AccountSet";
+        txJson[jss::Account] = accountStr;
+        txJson[jss::Sequence] = 0;
+        txJson[jss::Fee] = to_string(fee);
+        txJson["SourceTag"] =
+            static_cast<std::underlying_type_t<TicketPurpose>>(purpose);
+        txJson["TicketSequence"] = ticketSequence;
+        if (memoJson)
+        {
+            Serializer s;
+            try
+            {
+                STParsedJSONObject parsed(std::string(jss::tx_json), *memoJson);
+                if (!parsed.object)
+                {
+                    JLOGV(
+                        j_.fatal(), "invalid transaction", jv("tx", *memoJson));
+                    assert(0);
+                    return;
+                }
+                parsed.object->setFieldVL(sfSigningPubKey, Slice(nullptr, 0));
+                parsed.object->add(s);
+            }
+            catch (...)
+            {
+                JLOGV(j_.fatal(), "invalid transaction", jv("tx", *memoJson));
+                assert(0);
+                return;
+            }
+
+            Json::Value memos{Json::arrayValue};
+            Json::Value memo;
+            auto const dataStr = strHex(s.peekData());
+            memo[jss::Memo][jss::MemoData] = dataStr;
+            memos.append(memo);
+            txJson[jss::Memos] = memos;
+            JLOGV(
+                j_.trace(),
+                "TicketRunner",
+                jv("tx", *memoJson),
+                jv("tx packed", dataStr),
+                jv("packed size", dataStr.length()));
+            assert(
+                memo[jss::Memo][jss::MemoData].asString().length() <=
+                event::MemoStringMax);
+        }
+        signatureCollector.signAndSubmit(txJson);
+    };
+
+    sendTriggerTx(
+        mainAccountStr_,
+        ticketPair->first,
+        mainTxJson,
+        federator_.getSignatureCollector(Federator::ChainType::mainChain));
+    sendTriggerTx(
+        sideAccountStr_,
+        ticketPair->second,
+        sideTxJson,
+        federator_.getSignatureCollector(Federator::ChainType::sideChain));
+    return true;
+}
+
+void
+TicketRunner::onEvent(
+    std::uint32_t accountSeq,
+    const event::TicketCreateTrigger& e)
+{
+    Json::Value txJson;
+    XRPAmount const fee{Federator::accountControlTxFee};
+    txJson[jss::TransactionType] = "TicketCreate";
+    txJson[jss::Account] =
+        e.dir_ == event::Dir::mainToSide ? sideAccountStr_ : mainAccountStr_;
+    txJson[jss::Sequence] = accountSeq;
+    txJson[jss::Fee] = to_string(fee);
+    txJson["TicketCount"] = 1;
+    txJson["SourceTag"] = e.sourceTag_;
+    {
+        Json::Value memos{Json::arrayValue};
+        {
+            Json::Value memo;
+            memo[jss::Memo][jss::MemoData] = to_string(e.txnHash_);
+            memos.append(memo);
+        }
+        if (!e.memoStr_.empty())
+        {
+            Json::Value memo;
+            memo[jss::Memo][jss::MemoData] = e.memoStr_;
+            memos.append(memo);
+        }
+        txJson[jss::Memos] = memos;
+    }
+    JLOGV(
+        j_.trace(),
+        "TicketRunner TicketTriggerDetected",
+        jv("chain", (e.dir_ == event::Dir::mainToSide ? "main" : "side")),
+        jv("seq", accountSeq),
+        jv("CreateTicket tx", txJson));
+
+    if (e.dir_ == event::Dir::mainToSide)
+        federator_.getSignatureCollector(Federator::ChainType::sideChain)
+            .signAndSubmit(txJson);
+    else
+        federator_.getSignatureCollector(Federator::ChainType::mainChain)
+            .signAndSubmit(txJson);
+}
+
+void
+TicketRunner::onEvent(
+    std::uint32_t accountSeq,
+    const event::TicketCreateResult& e)
+{
+    auto const [fromChain, toChain] = e.dir_ == event::Dir::mainToSide
+        ? std::make_pair(Federator::sideChain, Federator::mainChain)
+        : std::make_pair(Federator::mainChain, Federator::sideChain);
+
+    auto ticketSeq = e.txnSeq_ + 1;
+    JLOGV(
+        j_.trace(),
+        "TicketRunner CreateTicketResult",
+        jv("chain",
+           (fromChain == Federator::ChainType::mainChain ? "main" : "side")),
+        jv("ticket seq", ticketSeq));
+
+    if (fromChain == Federator::ChainType::mainChain)
+        mainHolder_.onEvent(e);
+    else
+        sideHolder_.onEvent(e);
+
+    federator_.addSeqToSkip(fromChain, ticketSeq);
+
+    if (accountSeq)
+    {
+        assert(!e.memoStr_.empty());
+        auto txData = strUnHex(e.memoStr_);
+        if (!txData || !txData->size())
+        {
+            assert(false);
+            return;
+        }
+        SerialIter sitTrans(makeSlice(*txData));
+        STTx tx(sitTrans);
+        tx.setFieldU32(sfSequence, accountSeq);
+
+        auto txJson = tx.getJson(JsonOptions::none);
+        // trigger hash
+        Json::Value memos{Json::arrayValue};
+        {
+            Json::Value memo;
+            memo[jss::Memo][jss::MemoData] = to_string(e.txnHash_);
+            memos.append(memo);
+        }
+        txJson[jss::Memos] = memos;
+
+        JLOGV(
+            j_.trace(),
+            "TicketRunner AccountControlTrigger",
+            jv("chain",
+               (toChain == Federator::ChainType::mainChain ? "side" : "main")),
+            jv("tx with added memos", txJson.toStyledString()));
+
+        federator_.getSignatureCollector(toChain).signAndSubmit(txJson);
+    }
+}
+
+void
+TicketRunner::onEvent(const event::BootstrapTicket& e)
+{
+    auto ticketSeq = e.txnSeq_ + 1;
+    JLOGV(
+        j_.trace(),
+        "TicketRunner BootstrapTicket",
+        jv("chain", (e.isMainchain_ ? "main" : "side")),
+        jv("ticket seq", ticketSeq));
+
+    if (e.isMainchain_)
+        mainHolder_.onEvent(e);
+    else
+        sideHolder_.onEvent(e);
+}
+
+Json::Value
+TicketRunner::getInfo(bool isMainchain) const
+{
+    if (isMainchain)
+        return mainHolder_.getInfo();
+    else
+        return sideHolder_.getInfo();
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/TicketHolder.h

@@ -0,0 +1,255 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IMPL_TICKET_BOOTH_H
+#define RIPPLE_SIDECHAIN_IMPL_TICKET_BOOTH_H
+
+#include <ripple/app/sidechain/FederatorEvents.h>
+#include <ripple/app/sidechain/impl/ChainListener.h>
+#include <ripple/basics/UnorderedContainers.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/AccountID.h>
+
+#include <limits>
+#include <mutex>
+#include <optional>
+#include <queue>
+#include <string>
+
+namespace ripple {
+namespace sidechain {
+
+class Federator;
+
+enum class TicketPurpose : std::uint32_t {
+    mainDoorKeeper,
+    sideDoorKeeper,
+    updateSignerList,
+    TP_NumberOfItems
+};
+std::string
+TicketPurposeToStr(TicketPurpose tp);
+
+struct AutoRenewedTicket
+{
+    enum class Status { available, taken };
+
+    std::uint32_t seq_;
+    Status status_;
+
+    AutoRenewedTicket() : seq_(0), status_(Status::taken)
+    {
+    }
+};
+
+class TicketHolder
+{
+    enum class InitializeStatus {
+        waitLedger,
+        waitAccountObject,
+        waitTx,
+        needToQueryTx,
+        initialized
+    };
+
+    struct InitializeData
+    {
+        InitializeStatus status_ = InitializeStatus::waitLedger;
+        hash_map<uint256, std::uint32_t> tickets_;
+        std::queue<event::TicketCreateResult> toReplay_;
+        std::queue<event::BootstrapTicket> bootstrapTicketToReplay_;
+        std::uint32_t ledgerIndex_ = 0;
+    };
+
+    std::shared_ptr<ChainListener> rpcChannel_;
+    bool isMainChain_;
+    std::string const accountStr_;
+    AutoRenewedTicket
+        tickets_[static_cast<std::underlying_type_t<TicketPurpose>>(
+            TicketPurpose::TP_NumberOfItems)];
+    InitializeData initData_;
+    Federator& federator_;
+    beast::Journal j_;
+    mutable std::mutex mtx_;
+
+public:
+    TicketHolder(
+        bool isMainChain,
+        AccountID const& account,
+        Federator& federator,
+        beast::Journal j);
+
+    /**
+     * start to initialize the ticketHolder by sending accountObject RPC
+     */
+    void
+    init() EXCLUDES(mtx_);
+    /**
+     * process accountObject result and find the tickets.
+     * Initialization is not completed, because a ticket ledger object
+     * does not have information about its purpose.
+     * The purpose is in the TicketCreate tx what created the ticket.
+     * So the ticketHolder queries the TicketCreate tx for each ticket found.
+     * @param rpcResult accountObject result
+     */
+    void
+    accountObjectResult(Json::Value const& rpcResult) EXCLUDES(mtx_);
+    /**
+     * process tx RPC result
+     * Initialization is completed once all TicketCreate txns are found,
+     * one for every ticket found in the previous initialization step.
+     * @param rpcResult tx result
+     */
+    void
+    txResult(Json::Value const& rpcResult) EXCLUDES(mtx_);
+
+    enum class PeekOrTake { peek, take };
+    /**
+     * take or peek the ticket for a purpose
+     * @param purpose the ticket purpose
+     * @param pt take or peek
+     * @return the ticket if exist and not taken
+     */
+    std::optional<std::uint32_t>
+    getTicket(TicketPurpose purpose, PeekOrTake pt) EXCLUDES(mtx_);
+
+    /**
+     * process a TicketCreateResult event, update the ticket number and status
+     * It queues the event if the ticketHolder is not yet initialized.
+     *
+     * @param e the TicketCreateResult event
+     */
+    void
+    onEvent(event::TicketCreateResult const& e) EXCLUDES(mtx_);
+    /**
+     * process a ticket created during network bootstrap
+     * @param e the BootstrapTicket event
+     */
+    void
+    onEvent(event::BootstrapTicket const& e) EXCLUDES(mtx_);
+
+    Json::Value
+    getInfo() const EXCLUDES(mtx_);
+
+    void
+    setRpcChannel(std::shared_ptr<ChainListener> channel);
+
+private:
+    void
+    rpcAccountObject();
+
+    void
+    rpcTx(std::lock_guard<std::mutex> const&) REQUIRES(mtx_);
+
+    // replay accumulated events before finish initialization
+    void
+    replay(std::lock_guard<std::mutex> const&) REQUIRES(mtx_);
+    template <class E>
+    void
+    processEvent(E const& e, std::lock_guard<std::mutex> const&) REQUIRES(mtx_);
+};
+
+class TicketRunner
+{
+    std::string const mainAccountStr_;
+    std::string const sideAccountStr_;
+    Federator& federator_;
+    TicketHolder mainHolder_;
+    TicketHolder sideHolder_;
+    beast::Journal j_;
+    // Only one thread at a time can grab tickets
+    mutable std::mutex mtx_;
+
+public:
+    TicketRunner(
+        AccountID const& mainAccount,
+        AccountID const& sideAccount,
+        Federator& federator,
+        beast::Journal j);
+
+    // set RpcChannel for a ticketHolder
+    void
+    setRpcChannel(bool isMainChain, std::shared_ptr<ChainListener> channel);
+    // init a ticketHolder
+    void
+    init(bool isMainChain);
+    // pass a accountObject RPC result to a ticketHolder
+    void
+    accountObjectResult(bool isMainChain, Json::Value const& rpcResult);
+    // pass a tx RPC result to a ticketHolder
+    void
+    txResult(bool isMainChain, Json::Value const& rpcResult);
+
+    /**
+     * Start to run a protocol that submit a federator account control tx
+     * to the network.
+     *
+     * Comparing to a normal tx submission that takes one step, a federator
+     * account control tx (such as depositAuth and signerListSet) takes 3 steps:
+     * 1. use a ticket to send a accountSet no-op tx as a trigger
+     * 2. create a new ticket
+     * 3. submit the account control tx
+     *
+     * @param ticketPurpose the purpose of ticket. The purpose describes
+     * the account control tx use case.
+     * @param mainTxJson account control tx for main chain
+     * @param sideTxJson account control tx for side chain
+     * @note mainTxJson and sideTxJson cannot both be empty
+     * @return if the protocol started
+     */
+    [[nodiscard]] bool
+    trigger(
+        TicketPurpose ticketPurpose,
+        std::optional<Json::Value> const& mainTxJson,
+        std::optional<Json::Value> const& sideTxJson) EXCLUDES(mtx_);
+
+    /**
+     * process a TicketCreateTrigger event, by submitting TicketCreate tx
+     *
+     * This event is generated when the accountSet no-op tx
+     * (as the protocol trigger) appears in the tx stream,
+     * i.e. sorted with regular XChain payments.
+     */
+    void
+    onEvent(std::uint32_t accountSeq, event::TicketCreateTrigger const& e);
+    /**
+     * process a TicketCreateResult event, update the ticketHolder.
+     *
+     * This event is generated when the TicketCreate tx appears
+     * in the tx stream.
+     */
+    void
+    onEvent(std::uint32_t accountSeq, event::TicketCreateResult const& e);
+
+    /**
+     * process a ticket created during network bootstrap
+     */
+    void
+    onEvent(event::BootstrapTicket const& e);
+
+    Json::Value
+    getInfo(bool isMainchain) const;
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/app/sidechain/impl/WebsocketClient.cpp

@@ -0,0 +1,175 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2016 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/sidechain/impl/WebsocketClient.h>
+
+#include <ripple/basics/Log.h>
+#include <ripple/json/Output.h>
+#include <ripple/json/json_reader.h>
+#include <ripple/json/to_string.h>
+#include <ripple/protocol/jss.h>
+#include <ripple/server/Port.h>
+
+#include <boost/beast/websocket.hpp>
+
+#include <condition_variable>
+#include <string>
+#include <unordered_map>
+
+#include <iostream>
+
+namespace ripple {
+namespace sidechain {
+
+template <class ConstBuffers>
+std::string
+WebsocketClient::buffer_string(ConstBuffers const& b)
+{
+    using boost::asio::buffer;
+    using boost::asio::buffer_size;
+    std::string s;
+    s.resize(buffer_size(b));
+    buffer_copy(buffer(&s[0], s.size()), b);
+    return s;
+}
+
+void
+WebsocketClient::cleanup()
+{
+    ios_.post(strand_.wrap([this] {
+        if (!peerClosed_)
+        {
+            {
+                std::lock_guard l{m_};
+                ws_.async_close({}, strand_.wrap([&](error_code ec) {
+                    stream_.cancel(ec);
+
+                    std::lock_guard l(shutdownM_);
+                    isShutdown_ = true;
+                    shutdownCv_.notify_one();
+                }));
+            }
+        }
+        else
+        {
+            std::lock_guard<std::mutex> l(shutdownM_);
+            isShutdown_ = true;
+            shutdownCv_.notify_one();
+        }
+    }));
+}
+
+void
+WebsocketClient::shutdown()
+{
+    cleanup();
+    std::unique_lock l{shutdownM_};
+    shutdownCv_.wait(l, [this] { return isShutdown_; });
+}
+
+WebsocketClient::WebsocketClient(
+    std::function<void(Json::Value const&)> callback,
+    boost::asio::io_service& ios,
+    boost::asio::ip::address const& ip,
+    std::uint16_t port,
+    std::unordered_map<std::string, std::string> const& headers,
+    beast::Journal j)
+    : ios_(ios)
+    , strand_(ios_)
+    , stream_(ios_)
+    , ws_(stream_)
+    , callback_(callback)
+    , j_{j}
+{
+    try
+    {
+        boost::asio::ip::tcp::endpoint const ep{ip, port};
+        stream_.connect(ep);
+        ws_.set_option(boost::beast::websocket::stream_base::decorator(
+            [&](boost::beast::websocket::request_type& req) {
+                for (auto const& h : headers)
+                    req.set(h.first, h.second);
+            }));
+        ws_.handshake(
+            ep.address().to_string() + ":" + std::to_string(ep.port()), "/");
+        ws_.async_read(
+            rb_,
+            strand_.wrap(std::bind(
+                &WebsocketClient::onReadMsg, this, std::placeholders::_1)));
+    }
+    catch (std::exception&)
+    {
+        cleanup();
+        Rethrow();
+    }
+}
+
+WebsocketClient::~WebsocketClient()
+{
+    cleanup();
+}
+
+std::uint32_t
+WebsocketClient::send(std::string const& cmd, Json::Value params)
+{
+    params[jss::method] = cmd;
+    params[jss::jsonrpc] = "2.0";
+    params[jss::ripplerpc] = "2.0";
+
+    auto const id = nextId_++;
+    params[jss::id] = id;
+    auto const s = to_string(params);
+
+    std::lock_guard l{m_};
+    ws_.write_some(true, boost::asio::buffer(s));
+    return id;
+}
+
+void
+WebsocketClient::onReadMsg(error_code const& ec)
+{
+    if (ec)
+    {
+        JLOGV(j_.trace(), "WebsocketClient::onReadMsg error", jv("ec", ec));
+        if (ec == boost::beast::websocket::error::closed)
+            peerClosed_ = true;
+        return;
+    }
+
+    Json::Value jv;
+    Json::Reader jr;
+    jr.parse(buffer_string(rb_.data()), jv);
+    rb_.consume(rb_.size());
+    callback_(jv);
+
+    std::lock_guard l{m_};
+    ws_.async_read(
+        rb_,
+        strand_.wrap(std::bind(
+            &WebsocketClient::onReadMsg, this, std::placeholders::_1)));
+}
+
+// Called when the read op terminates
+void
+WebsocketClient::onReadDone()
+{
+}
+
+}  // namespace sidechain
+}  // namespace ripple

src/ripple/app/sidechain/impl/WebsocketClient.h

@@ -0,0 +1,105 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_SIDECHAIN_IO_WEBSOCKET_CLIENT_H_INCLUDED
+#define RIPPLE_SIDECHAIN_IO_WEBSOCKET_CLIENT_H_INCLUDED
+
+#include <ripple/basics/ThreadSaftyAnalysis.h>
+#include <ripple/core/Config.h>
+
+#include <boost/asio/io_service.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/strand.hpp>
+#include <boost/beast/core/multi_buffer.hpp>
+#include <boost/beast/websocket/stream.hpp>
+#include <boost/optional.hpp>
+
+#include <chrono>
+#include <condition_variable>
+#include <functional>
+#include <memory>
+
+namespace ripple {
+namespace sidechain {
+
+class WebsocketClient
+{
+    using error_code = boost::system::error_code;
+
+    template <class ConstBuffers>
+    static std::string
+    buffer_string(ConstBuffers const& b);
+
+    // mutex for ws_
+    std::mutex m_;
+
+    // mutex for shutdown
+    std::mutex shutdownM_;
+    bool isShutdown_ = false;
+    std::condition_variable shutdownCv_;
+
+    boost::asio::io_service& ios_;
+    boost::asio::io_service::strand strand_;
+    boost::asio::ip::tcp::socket stream_;
+    boost::beast::websocket::stream<boost::asio::ip::tcp::socket&> GUARDED_BY(
+        m_) ws_;
+    boost::beast::multi_buffer rb_;
+
+    std::atomic<bool> peerClosed_{false};
+
+    std::function<void(Json::Value const&)> callback_;
+    std::atomic<std::uint32_t> nextId_{0};
+
+    beast::Journal j_;
+
+    void
+    cleanup();
+
+public:
+    // callback will be called from a io_service thread
+    WebsocketClient(
+        std::function<void(Json::Value const&)> callback,
+        boost::asio::io_service& ios,
+        boost::asio::ip::address const& ip,
+        std::uint16_t port,
+        std::unordered_map<std::string, std::string> const& headers,
+        beast::Journal j);
+
+    ~WebsocketClient();
+
+    // Returns command id that will be returned in the response
+    std::uint32_t
+    send(std::string const& cmd, Json::Value params) EXCLUDES(m_);
+
+    void
+    shutdown() EXCLUDES(shutdownM_);
+
+private:
+    void
+    onReadMsg(error_code const& ec) EXCLUDES(m_);
+
+    // Called when the read op terminates
+    void
+    onReadDone();
+};
+
+}  // namespace sidechain
+}  // namespace ripple
+
+#endif

src/ripple/basics/ThreadSaftyAnalysis.h

@@ -0,0 +1,82 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED
+#define RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED
+
+#ifdef RIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS
+#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
+#else
+#define THREAD_ANNOTATION_ATTRIBUTE__(x)  // no-op
+#endif
+
+#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x))
+
+#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
+
+#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))
+
+#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))
+
+#define ACQUIRED_BEFORE(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))
+
+#define ACQUIRED_AFTER(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))
+
+#define REQUIRES(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__))
+
+#define REQUIRES_SHARED(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__))
+
+#define ACQUIRE(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__))
+
+#define ACQUIRE_SHARED(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__))
+
+#define RELEASE(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__))
+
+#define RELEASE_SHARED(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__))
+
+#define RELEASE_GENERIC(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(release_generic_capability(__VA_ARGS__))
+
+#define TRY_ACQUIRE(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__))
+
+#define TRY_ACQUIRE_SHARED(...) \
+    THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__))
+
+#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__))
+
+#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x))
+
+#define ASSERT_SHARED_CAPABILITY(x) \
+    THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x))
+
+#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
+
+#define NO_THREAD_SAFETY_ANALYSIS \
+    THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis)
+
+#endif

src/ripple/core/Job.h

@@ -60,20 +60,21 @@ enum JobType {
     jtREPLAY_TASK,        // A Ledger replay task/subtask
     jtLEDGER_DATA,        // Received data for a ledger we're acquiring
     jtTRANSACTION,        // A transaction received from the network
-    jtMISSING_TXN,        // Request missing transactions
-    jtREQUESTED_TXN,      // Reply with requested transactions
-    jtBATCH,              // Apply batched transactions
-    jtADVANCE,            // Advance validated/acquired ledgers
-    jtPUBLEDGER,          // Publish a fully-accepted ledger
-    jtTXN_DATA,           // Fetch a proposed set
-    jtWAL,                // Write-ahead logging
-    jtVALIDATION_t,       // A validation from a trusted source
-    jtWRITE,              // Write out hashed objects
-    jtACCEPT,             // Accept a consensus ledger
-    jtPROPOSAL_t,         // A proposal from a trusted source
-    jtNETOP_CLUSTER,      // NetworkOPs cluster peer report
-    jtNETOP_TIMER,        // NetworkOPs net timer processing
-    jtADMIN,              // An administrative operation
+    jtFEDERATORSIGNATURE,  // A signature from a sidechain federator
+    jtMISSING_TXN,         // Request missing transactions
+    jtREQUESTED_TXN,       // Reply with requested transactions
+    jtBATCH,               // Apply batched transactions
+    jtADVANCE,             // Advance validated/acquired ledgers
+    jtPUBLEDGER,           // Publish a fully-accepted ledger
+    jtTXN_DATA,            // Fetch a proposed set
+    jtWAL,                 // Write-ahead logging
+    jtVALIDATION_t,        // A validation from a trusted source
+    jtWRITE,               // Write out hashed objects
+    jtACCEPT,              // Accept a consensus ledger
+    jtPROPOSAL_t,          // A proposal from a trusted source
+    jtNETOP_CLUSTER,       // NetworkOPs cluster peer report
+    jtNETOP_TIMER,         // NetworkOPs net timer processing
+    jtADMIN,               // An administrative operation
 
     // Special job types which are not dispatched by the job pool
     jtPEER,

src/ripple/core/JobTypes.h

@@ -90,6 +90,13 @@ private:
         add(jtUPDATE_PF,         "updatePaths",                 1,     0ms,     0ms);
         add(jtTRANSACTION,       "transaction",          maxLimit,   250ms,  1000ms);
         add(jtBATCH,             "batch",                maxLimit,   250ms,  1000ms);
+        // TODO chose ave latency and peak latency numbers
+        add(jtFEDERATORSIGNATURE,
+            "federatorSignature",
+            maxLimit,
+            false,
+            250ms,
+            1000ms);
         add(jtADVANCE,           "advanceLedger",        maxLimit,     0ms,     0ms);
         add(jtPUBLEDGER,         "publishNewLedger",     maxLimit,  3000ms,  4500ms);
         add(jtTXN_DATA,          "fetchTxnData",                5,     0ms,     0ms);

src/ripple/net/impl/RPCCall.cpp

@@ -1250,6 +1250,7 @@ public:
             {"deposit_authorized", &RPCParser::parseDepositAuthorized, 2, 3},
             {"download_shard", &RPCParser::parseDownloadShard, 2, -1},
             {"feature", &RPCParser::parseFeature, 0, 2},
+            {"federator_info", &RPCParser::parseAsIs, 0, 0},
             {"fetch_info", &RPCParser::parseFetchInfo, 0, 1},
             {"gateway_balances", &RPCParser::parseGatewayBalances, 1, -1},
             {"get_counts", &RPCParser::parseGetCounts, 0, 1},

src/ripple/overlay/impl/Message.cpp

@@ -87,6 +87,7 @@ Message::compress()
             case protocol::mtVALIDATORLISTCOLLECTION:
             case protocol::mtREPLAY_DELTA_RESPONSE:
             case protocol::mtTRANSACTIONS:
+            case protocol::mtFederatorXChainTxnSignature:
                 return true;
             case protocol::mtPING:
             case protocol::mtCLUSTER:
@@ -102,6 +103,7 @@ Message::compress()
             case protocol::mtGET_PEER_SHARD_INFO_V2:
             case protocol::mtPEER_SHARD_INFO_V2:
             case protocol::mtHAVE_TRANSACTIONS:
+            case protocol::mtFederatorAccountCtrlSignature:
                 break;
         }
         return false;

src/ripple/overlay/impl/PeerImp.cpp

@@ -27,6 +27,7 @@
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/ValidatorList.h>
+#include <ripple/app/sidechain/Federator.h>
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/UptimeClock.h>
 #include <ripple/basics/base64.h>
@@ -45,6 +46,7 @@
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/beast/core/ostream.hpp>
 
+#include "protocol/SField.h"
 #include <algorithm>
 #include <memory>
 #include <mutex>
@@ -2928,6 +2930,339 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMSquelch> const& m)
         << "onMessage: TMSquelch " << slice << " " << id() << " " << duration;
 }
 
+void
+PeerImp::onMessage(
+    std::shared_ptr<protocol::TMFederatorXChainTxnSignature> const& m)
+{
+    std::shared_ptr<sidechain::Federator> federator =
+        app_.getSidechainFederator();
+
+    auto sidechainJ = app_.journal("SidechainFederator");
+
+    auto badData = [&](std::string msg) {
+        fee_ = Resource::feeBadData;
+        JLOG(p_journal_.warn()) << msg;
+    };
+
+    auto getTxnType = [](::protocol::TMFederatorTxnType tt)
+        -> std::optional<sidechain::Federator::TxnType> {
+        switch (tt)
+        {
+            case ::protocol::TMFederatorTxnType::ftxnt_XCHAIN:
+                return sidechain::Federator::TxnType::xChain;
+            case ::protocol::TMFederatorTxnType::ftxnt_REFUND:
+                return sidechain::Federator::TxnType::refund;
+            default:
+                return {};
+        }
+    };
+
+    auto getChainType = [](::protocol::TMFederatorChainType ct)
+        -> std::optional<sidechain::Federator::ChainType> {
+        switch (ct)
+        {
+            case ::protocol::TMFederatorChainType::fct_SIDE:
+                return sidechain::Federator::sideChain;
+            case ::protocol::TMFederatorChainType::fct_MAIN:
+                return sidechain::Federator::mainChain;
+            default:
+                return {};
+        }
+    };
+
+    auto getPublicKey =
+        [](std::string const& data) -> std::optional<PublicKey> {
+        if (data.empty())
+            return {};
+        auto const s = makeSlice(data);
+        if (!publicKeyType(s))
+            return {};
+        return PublicKey(s);
+    };
+
+    auto getHash = [](std::string const& data) -> std::optional<uint256> {
+        if (data.size() != 32)
+            return {};
+        return uint256(data);
+    };
+
+    auto getAccountId =
+        [](std::string const& data) -> std::optional<AccountID> {
+        if (data.size() != 20)
+            return {};
+        return AccountID(data);
+    };
+
+    auto const signingPK = getPublicKey(m->signingpk());
+    if (!signingPK)
+    {
+        return badData("Invalid federator key");
+    }
+    auto const txnType = getTxnType(m->txntype());
+    if (!txnType)
+    {
+        return badData("Invalid txn type");
+    }
+    auto const dstChain = getChainType(m->dstchain());
+    if (!dstChain)
+    {
+        return badData("Invalid dst chain");
+    }
+    auto const srcChainTxnHash = getHash(m->srcchaintxnhash());
+    if (!srcChainTxnHash)
+    {
+        return badData("Invalid src chain txn hash");
+    }
+    auto const dstChainTxnHash = getHash(m->dstchaintxnhash());
+    if (*txnType == sidechain::Federator::TxnType::refund && !dstChainTxnHash)
+    {
+        return badData("Invalid dst chain txn hash for refund");
+    }
+    if (*txnType == sidechain::Federator::TxnType::xChain && dstChainTxnHash)
+    {
+        return badData("Invalid dst chain txn hash for xchain");
+    }
+    auto const srcChainSrcAccount = getAccountId(m->srcchainsrcaccount());
+    if (!srcChainSrcAccount)
+    {
+        return badData("Invalid src chain src account");
+    }
+    auto const dstChainSrcAccount = getAccountId(m->dstchainsrcaccount());
+    if (!dstChainSrcAccount)
+    {
+        return badData("Invalid dst chain src account");
+    }
+    auto const dstChainDstAccount = getAccountId(m->dstchaindstaccount());
+    if (!dstChainDstAccount)
+    {
+        return badData("Invalid dst account");
+    }
+    auto const seq = m->seq();
+    auto const amt = [&m]() -> std::optional<STAmount> {
+        try
+        {
+            SerialIter iter{m->amount().data(), m->amount().size()};
+            STAmount amt{iter, sfGeneric};
+            if (!iter.empty() || amt.signum() <= 0)
+            {
+                return std::nullopt;
+            }
+            if (amt.native() && amt.xrp() > INITIAL_XRP)
+            {
+                return std::nullopt;
+            }
+            return amt;
+        }
+        catch (std::exception const&)
+        {
+        }
+        return std::nullopt;
+    }();
+
+    if (!amt)
+    {
+        return badData("Invalid amount");
+    }
+
+    Buffer sig{m->signature().data(), m->signature().size()};
+
+    JLOGV(
+        sidechainJ.trace(),
+        "Received signature from peer",
+        jv("id", id()),
+        jv("sig", strHex(sig.data(), sig.data() + sig.size())));
+
+    if (federator && federator->alreadySent(*dstChain, seq))
+    {
+        // already sent this transaction, no need to forward signature
+        return;
+    }
+
+    uint256 const suppression = sidechain::crossChainTxnSignatureId(
+        *signingPK,
+        *srcChainTxnHash,
+        dstChainTxnHash,
+        *amt,
+        *dstChainSrcAccount,
+        *dstChainDstAccount,
+        seq,
+        sig);
+    app_.getHashRouter().addSuppressionPeer(suppression, id_);
+
+    app_.getJobQueue().addJob(
+        jtFEDERATORSIGNATURE,
+        "federator signature",
+        [self = shared_from_this(),
+         federator = std::move(federator),
+         suppression,
+         txnType,
+         dstChain,
+         signingPK,
+         srcChainTxnHash,
+         dstChainTxnHash,
+         amt,
+         srcChainSrcAccount,
+         dstChainDstAccount,
+         seq,
+         m,
+         j = sidechainJ,
+         sig = std::move(sig)](Job&) mutable {
+            auto& hashRouter = self->app_.getHashRouter();
+            if (auto const toSkip = hashRouter.shouldRelay(suppression))
+            {
+                auto const toSend = std::make_shared<Message>(
+                    *m, protocol::mtFederatorXChainTxnSignature);
+                self->overlay_.foreach([&](std::shared_ptr<Peer> const& p) {
+                    hashRouter.addSuppressionPeer(suppression, p->id());
+                    if (toSkip->count(p->id()))
+                    {
+                        JLOGV(
+                            j.trace(),
+                            "Not forwarding signature to peer",
+                            jv("id", p->id()),
+                            jv("suppression", suppression));
+                        return;
+                    }
+                    JLOGV(
+                        j.trace(),
+                        "Forwarding signature to peer",
+                        jv("id", p->id()),
+                        jv("suppression", suppression));
+                    p->send(toSend);
+                });
+            }
+
+            if (federator)
+            {
+                // Signature is checked in `addPendingTxnSig`
+                federator->addPendingTxnSig(
+                    *txnType,
+                    *dstChain,
+                    *signingPK,
+                    *srcChainTxnHash,
+                    dstChainTxnHash,
+                    *amt,
+                    *srcChainSrcAccount,
+                    *dstChainDstAccount,
+                    seq,
+                    std::move(sig));
+            }
+        });
+}
+
+void
+PeerImp::onMessage(
+    std::shared_ptr<protocol::TMFederatorAccountCtrlSignature> const& m)
+{
+    std::shared_ptr<sidechain::Federator> federator =
+        app_.getSidechainFederator();
+
+    auto sidechainJ = app_.journal("SidechainFederator");
+
+    auto badData = [&](std::string msg) {
+        fee_ = Resource::feeBadData;
+        JLOG(p_journal_.warn()) << msg;
+    };
+    auto getChainType = [](::protocol::TMFederatorChainType ct)
+        -> std::optional<sidechain::Federator::ChainType> {
+        switch (ct)
+        {
+            case ::protocol::TMFederatorChainType::fct_SIDE:
+                return sidechain::Federator::sideChain;
+            case ::protocol::TMFederatorChainType::fct_MAIN:
+                return sidechain::Federator::mainChain;
+            default:
+                return {};
+        }
+    };
+    auto const dstChain = getChainType(m->chain());
+    if (!dstChain)
+    {
+        return badData("Invalid dst chain");
+    }
+    auto getPublicKey =
+        [](std::string const& data) -> std::optional<PublicKey> {
+        if (data.empty())
+            return {};
+        auto const s = makeSlice(data);
+        if (!publicKeyType(s))
+            return {};
+        return PublicKey(s);
+    };
+
+    auto getHash = [](std::string const& data) -> std::optional<uint256> {
+        if (data.size() != 32)
+            return {};
+        return uint256(data);
+    };
+
+    auto const pk = getPublicKey(m->publickey());
+    if (!pk)
+    {
+        return badData("Invalid federator key");
+    }
+    auto const mId = getHash(m->messageid());
+    if (!mId)
+    {
+        return badData("Invalid txn hash");
+    }
+
+    Buffer sig{m->signature().data(), m->signature().size()};
+
+    JLOGV(
+        sidechainJ.trace(),
+        "Received signature from peer",
+        jv("id", id()),
+        jv("sig", strHex(sig.data(), sig.data() + sig.size())));
+
+    uint256 const suppression = sidechain::computeMessageSuppression(*mId, sig);
+    app_.getHashRouter().addSuppressionPeer(suppression, id_);
+
+    app_.getJobQueue().addJob(
+        jtFEDERATORSIGNATURE,
+        "federator signature",
+        [self = shared_from_this(),
+         federator = std::move(federator),
+         suppression,
+         pk,
+         mId,
+         m,
+         j = sidechainJ,
+         chain = *dstChain,
+         sig = std::move(sig)](Job&) mutable {
+            auto& hashRouter = self->app_.getHashRouter();
+            if (auto const toSkip = hashRouter.shouldRelay(suppression))
+            {
+                auto const toSend = std::make_shared<Message>(
+                    *m, protocol::mtFederatorAccountCtrlSignature);
+                self->overlay_.foreach([&](std::shared_ptr<Peer> const& p) {
+                    hashRouter.addSuppressionPeer(suppression, p->id());
+                    if (toSkip->count(p->id()))
+                    {
+                        JLOGV(
+                            j.trace(),
+                            "Not forwarding signature to peer",
+                            jv("id", p->id()),
+                            jv("suppression", suppression));
+                        return;
+                    }
+                    JLOGV(
+                        j.trace(),
+                        "Forwarding signature to peer",
+                        jv("id", p->id()),
+                        jv("suppression", suppression));
+                    p->send(toSend);
+                });
+            }
+
+            if (federator)
+            {
+                // Signature is checked in `addPendingTxnSig`
+                federator->addPendingTxnSig(chain, *pk, *mId, std::move(sig));
+            }
+        });
+}
+
 //--------------------------------------------------------------------------
 
 void

src/ripple/overlay/impl/PeerImp.h

@@ -584,6 +584,12 @@ public:
     onMessage(std::shared_ptr<protocol::TMReplayDeltaRequest> const& m);
     void
     onMessage(std::shared_ptr<protocol::TMReplayDeltaResponse> const& m);
+    void
+    onMessage(
+        std::shared_ptr<protocol::TMFederatorXChainTxnSignature> const& m);
+    void
+    onMessage(
+        std::shared_ptr<protocol::TMFederatorAccountCtrlSignature> const& m);
 
 private:
     //--------------------------------------------------------------------------

src/ripple/overlay/impl/ProtocolMessage.h

@@ -113,6 +113,10 @@ protocolMessageName(int type)
             return "get_peer_shard_info_v2";
         case protocol::mtPEER_SHARD_INFO_V2:
             return "peer_shard_info_v2";
+        case protocol::mtFederatorXChainTxnSignature:
+            return "federator_xchain_txn_signature";
+        case protocol::mtFederatorAccountCtrlSignature:
+            return "federator_account_ctrl_signature";
         default:
             break;
     }
@@ -493,6 +497,14 @@ invokeProtocolMessage(
             success = detail::invoke<protocol::TMPeerShardInfoV2>(
                 *header, buffers, handler);
             break;
+        case protocol::mtFederatorXChainTxnSignature:
+            success = detail::invoke<protocol::TMFederatorXChainTxnSignature>(
+                *header, buffers, handler);
+            break;
+        case protocol::mtFederatorAccountCtrlSignature:
+            success = detail::invoke<protocol::TMFederatorAccountCtrlSignature>(
+                *header, buffers, handler);
+            break;
         default:
             handler.onMessageUnknown(header->message_type);
             success = true;

src/ripple/overlay/impl/TrafficCount.cpp

@@ -163,6 +163,9 @@ TrafficCount::categorize(
     if (type == protocol::mtTRANSACTIONS)
         return TrafficCount::category::requested_transactions;
 
+    if (type == protocol::mtFederatorXChainTxnSignature)
+        return TrafficCount::category::federator_xchain_txn_signature;
+
     return TrafficCount::category::unknown;
 }
 

src/ripple/overlay/impl/TrafficCount.h

@@ -157,6 +157,8 @@ public:
         // TMTransactions
         requested_transactions,
 
+        federator_xchain_txn_signature,
+
         unknown  // must be last
     };
 
@@ -243,12 +245,13 @@ protected:
         {"getobject_share"},                     // category::share_hash
         {"getobject_get"},                       // category::get_hash
         {"proof_path_request"},                  // category::proof_path_request
-        {"proof_path_response"},     // category::proof_path_response
-        {"replay_delta_request"},    // category::replay_delta_request
-        {"replay_delta_response"},   // category::replay_delta_response
-        {"have_transactions"},       // category::have_transactions
-        {"requested_transactions"},  // category::transactions
-        {"unknown"}                  // category::unknown
+        {"proof_path_response"},             // category::proof_path_response
+        {"replay_delta_request"},            // category::replay_delta_request
+        {"replay_delta_response"},           // category::replay_delta_response
+        {"have_transactions"},               // category::have_transactions
+        {"requested_transactions"},          // category::transactions
+        {"federator_xchain_txn_signature"},  // category::federator_xchain_txn_signature
+        {"unknown"}                          // category::unknown
     }};
 };
 

src/ripple/proto/ripple.proto

@@ -33,6 +33,8 @@ enum MessageType
     mtPEER_SHARD_INFO_V2        = 62;
     mtHAVE_TRANSACTIONS         = 63;
     mtTRANSACTIONS              = 64;
+    mtFederatorXChainTxnSignature = 65;
+    mtFederatorAccountCtrlSignature = 66;
 }
 
 // token, iterations, target, challenge = issue demand for proof of work
@@ -450,3 +452,42 @@ message TMHaveTransactions
     repeated bytes hashes = 1;
 }
 
+enum TMFederatorChainType
+{
+   fct_SIDE = 1;
+   fct_MAIN = 2;
+}
+
+enum TMFederatorTxnType
+{
+   ftxnt_XCHAIN = 1; // cross chain
+   ftxnt_REFUND = 2; 
+}
+
+message TMFederatorXChainTxnSignature
+{
+    required TMFederatorTxnType txnType = 1;
+    required TMFederatorChainType dstChain = 2;
+    required bytes signingPK = 3; // federator's signing public key (unencoded binary data)
+    // txn hash for the src chain (unencoded bigendian binary data)
+    // This will be origional transaction from the src account to the door account
+    required bytes srcChainTxnHash = 4; 
+    // txn hash for the dst chain (unencoded bigendian binary data)
+    // This will be empty for XCHAIN transaction, and will the failed transaction from the
+    // door account to the dst account for refund txns.
+    optional bytes dstChainTxnHash = 5; 
+    required bytes amount = 6; // STAmount in wire serialized format
+    required bytes srcChainSrcAccount = 7; // account id (unencoded bigendian binary data)
+    required bytes dstChainSrcAccount = 8; // account id (unencoded bigendian binary data)
+    required bytes dstChainDstAccount = 9; // account id (unencoded bigendian binary data)
+    required uint32 seq = 10; // sequence number
+    required bytes signature = 11; // (unencoded bigendian binary data)
+}
+
+message TMFederatorAccountCtrlSignature
+{
+    required TMFederatorChainType chain = 1;
+    required bytes publicKey = 2;
+    required bytes messageId = 3;
+    required bytes signature = 4;
+}

src/ripple/protocol/STTx.h

@@ -20,6 +20,7 @@
 #ifndef RIPPLE_PROTOCOL_STTX_H_INCLUDED
 #define RIPPLE_PROTOCOL_STTX_H_INCLUDED
 
+#include <ripple/basics/Buffer.h>
 #include <ripple/basics/Expected.h>
 #include <ripple/protocol/PublicKey.h>
 #include <ripple/protocol/STObject.h>
@@ -77,6 +78,20 @@ public:
     Blob
     getSignature() const;
 
+    Buffer
+    getSignature(PublicKey const& publicKey, SecretKey const& secretKey) const;
+
+    // Get one of the multi-signatures
+    Buffer
+    getMultiSignature(
+        AccountID const& signingID,
+        PublicKey const& publicKey,
+        SecretKey const& secretKey) const;
+
+    // unconditionally set signature. No error checking.
+    void
+    setSignature(Buffer const& sig);
+
     uint256
     getSigningHash() const;
 

src/ripple/protocol/impl/STTx.cpp

@@ -160,6 +160,16 @@ getSigningData(STTx const& that)
     return s.getData();
 }
 
+static Blob
+getMultiSigningData(STTx const& that, AccountID const& signingID)
+{
+    Serializer s;
+    s.add32(HashPrefix::txMultiSign);
+    that.addWithoutSigningFields(s);
+    s.addBitString(signingID);
+    return s.getData();
+}
+
 uint256
 STTx::getSigningHash() const
 {
@@ -179,6 +189,30 @@ STTx::getSignature() const
     }
 }
 
+Buffer
+STTx::getSignature(PublicKey const& publicKey, SecretKey const& secretKey) const
+{
+    auto const data = getSigningData(*this);
+    return ripple::sign(publicKey, secretKey, makeSlice(data));
+}
+
+Buffer
+STTx::getMultiSignature(
+    AccountID const& signingID,
+    PublicKey const& publicKey,
+    SecretKey const& secretKey) const
+{
+    auto const data = getMultiSigningData(*this, signingID);
+    return ripple::sign(publicKey, secretKey, makeSlice(data));
+}
+
+void
+STTx::setSignature(Buffer const& sig)
+{
+    setFieldVL(sfTxnSignature, sig);
+    tid_ = getHash(HashPrefix::transactionID);
+}
+
 SeqProxy
 STTx::getSeqProxy() const
 {
@@ -197,12 +231,7 @@ STTx::getSeqProxy() const
 void
 STTx::sign(PublicKey const& publicKey, SecretKey const& secretKey)
 {
-    auto const data = getSigningData(*this);
-
-    auto const sig = ripple::sign(publicKey, secretKey, makeSlice(data));
-
-    setFieldVL(sfTxnSignature, sig);
-    tid_ = getHash(HashPrefix::transactionID);
+    setSignature(getSignature(publicKey, secretKey));
 }
 
 Expected<void, std::string>

src/ripple/protocol/jss.h

@@ -70,6 +70,10 @@ JSS(Invalid);                //
 JSS(LastLedgerSequence);     // in: TransactionSign; field
 JSS(LedgerHashes);           // ledger type.
 JSS(LimitAmount);            // field.
+JSS(Memo);                   // txn common field
+JSS(Memos);                  // txn common field
+JSS(MemoType);               // txn common field
+JSS(MemoData);               // txn common field
 JSS(Offer);                  // ledger type.
 JSS(OfferCancel);            // transaction type.
 JSS(OfferCreate);            // transaction type.
@@ -91,6 +95,10 @@ JSS(SetFlag);                // field.
 JSS(SetRegularKey);          // transaction type.
 JSS(SignerList);             // ledger type.
 JSS(SignerListSet);          // transaction type.
+JSS(SignerEntry);            // transaction type.
+JSS(SignerEntries);          // transaction type.
+JSS(SignerQuorum);           // transaction type.
+JSS(SignerWeight);           // transaction type.
 JSS(SigningPubKey);          // field.
 JSS(TakerGets);              // field.
 JSS(TakerPays);              // field.
@@ -312,6 +320,7 @@ JSS(last_close);                  // out: NetworkOPs
 JSS(last_refresh_time);           // out: ValidatorSite
 JSS(last_refresh_status);         // out: ValidatorSite
 JSS(last_refresh_message);        // out: ValidatorSite
+JSS(last_transaction_sent_seq);   // out: federator_info
 JSS(ledger);                      // in: NetworkOPs, LedgerCleaner,
                                   //     RPCHelpers
                                   // out: NetworkOPs, PeerImp
@@ -339,6 +348,7 @@ JSS(limit);                       // in/out: AccountTx*, AccountOffers,
 JSS(limit_peer);                  // out: AccountLines
 JSS(lines);                       // out: AccountLines
 JSS(list);                        // out: ValidatorList
+JSS(listener_info);               // out: federator_info
 JSS(load);                        // out: NetworkOPs, PeerImp
 JSS(load_base);                   // out: NetworkOPs
 JSS(load_factor);                 // out: NetworkOPs
@@ -355,6 +365,7 @@ JSS(local_txs);                   // out: GetCounts
 JSS(local_static_keys);           // out: ValidatorList
 JSS(lowest_sequence);             // out: AccountInfo
 JSS(lowest_ticket);               // out: AccountInfo
+JSS(mainchain);                   // out: federator_info
 JSS(majority);                    // out: RPC feature
 JSS(manifest);                    // out: ValidatorInfo, Manifest
 JSS(marker);                      // in/out: AccountTx, AccountOffers,
@@ -436,6 +447,7 @@ JSS(peers);                      // out: InboundLedger, handlers/Peers, Overlay
 JSS(peer_disconnects);           // Severed peer connection counter.
 JSS(peer_disconnects_resources);  // Severed peer connections because of
                                   // excess resource consumption.
+JSS(pending_transactions);        // out: federator_info
 JSS(port);                        // in: Connect
 JSS(previous);                    // out: Reservations
 JSS(previous_ledger);             // out: LedgerPropose
@@ -508,7 +520,9 @@ JSS(server_version);            // out: NetworkOPs
 JSS(settle_delay);              // out: AccountChannels
 JSS(severity);                  // in: LogLevel
 JSS(shards);                    // in/out: GetCounts, DownloadShard
+JSS(sidechain);                 // out: federator_info
 JSS(signature);                 // out: NetworkOPs, ChannelAuthorize
+JSS(signatures);                // out: federator_info
 JSS(signature_verified);        // out: ChannelVerify
 JSS(signing_key);               // out: NetworkOPs
 JSS(signing_keys);              // out: ValidatorList
@@ -536,6 +550,7 @@ JSS(sub_index);             // in: LedgerEntry
 JSS(subcommand);            // in: PathFind
 JSS(success);               // rpc
 JSS(supported);             // out: AmendmentTableImpl
+JSS(sync_info);             // out: federator_info
 JSS(system_time_offset);    // out: NetworkOPs
 JSS(tag);                   // out: Peers
 JSS(taker);                 // in: Subscribe, BookOffers

src/ripple/rpc/handlers/FederatorInfo.cpp

@@ -0,0 +1,46 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/main/Application.h>
+#include <ripple/app/sidechain/Federator.h>
+#include <ripple/json/json_value.h>
+#include <ripple/net/RPCErr.h>
+#include <ripple/protocol/jss.h>
+#include <ripple/rpc/Context.h>
+
+namespace ripple {
+
+Json::Value
+doFederatorInfo(RPC::JsonContext& context)
+{
+    Json::Value ret{Json::objectValue};
+
+    if (auto f = context.app.getSidechainFederator())
+    {
+        ret[jss::info] = f->getInfo();
+    }
+    else
+    {
+        ret[jss::info] = "Not configured as a sidechain federator";
+    }
+
+    return ret;
+}
+
+}  // namespace ripple

src/ripple/rpc/handlers/Handlers.h

@@ -59,6 +59,8 @@ doDownloadShard(RPC::JsonContext&);
 Json::Value
 doFeature(RPC::JsonContext&);
 Json::Value
+doFederatorInfo(RPC::JsonContext&);
+Json::Value
 doFee(RPC::JsonContext&);
 Json::Value
 doFetchInfo(RPC::JsonContext&);

src/ripple/rpc/impl/Handler.cpp

@@ -89,6 +89,7 @@ Handler const handlerArray[]{
 #endif
     {"get_counts", byRef(&doGetCounts), Role::ADMIN, NO_CONDITION},
     {"feature", byRef(&doFeature), Role::ADMIN, NO_CONDITION},
+    {"federator_info", byRef(&doFederatorInfo), Role::USER, NO_CONDITION},
     {"fee", byRef(&doFee), Role::USER, NEEDS_CURRENT_LEDGER},
     {"fetch_info", byRef(&doFetchInfo), Role::ADMIN, NO_CONDITION},
     {"ledger_accept",

src/ripple/rpc/impl/ServerHandlerImp.cpp

@@ -320,6 +320,7 @@ ServerHandlerImp::onWSMessage(
     if (size > RPC::Tuning::maxRequestSize ||
         !Json::Reader{}.parse(jv, buffers) || !jv.isObject())
     {
+        Json::Reader{}.parse(jv, buffers);  // swd debug
         Json::Value jvResult(Json::objectValue);
         jvResult[jss::type] = jss::error;
         jvResult[jss::error] = "jsonInvalid";