New improved Pathfinding engine

modules/ripple_app/ledger/OrderBookDB.cpp

@@ -34,10 +34,12 @@ void OrderBookDB::setup (Ledger::ref ledger)
 
     mDestMap.clear ();
     mSourceMap.clear ();
+    mXRPBooks.clear ();
 
     WriteLog (lsDEBUG, OrderBookDB) << "OrderBookDB>";
 
     // walk through the entire ledger looking for orderbook entries
+    int books = 0;
     uint256 currentIndex = ledger->getFirstLedgerIndex ();
 
     while (currentIndex.isNonZero ())
@@ -62,13 +64,16 @@ void OrderBookDB::setup (Ledger::ref ledger)
 
                 mSourceMap[currencyIssuer_ct (ci, ii)].push_back (book);
                 mDestMap[currencyIssuer_ct (co, io)].push_back (book);
+                if (co.isZero())
+                    mXRPBooks.insert(currencyIssuer_ct (ci, ii));
+                ++books;
             }
         }
 
         currentIndex = ledger->getNextLedgerIndex (currentIndex);
     }
 
-    WriteLog (lsDEBUG, OrderBookDB) << "OrderBookDB<";
+    WriteLog (lsDEBUG, OrderBookDB) << "OrderBookDB< " << books << " books found";
 }
 
 // return list of all orderbooks that want this issuerID and currencyID
@@ -85,6 +90,13 @@ void OrderBookDB::getBooksByTakerPays (const uint160& issuerID, const uint160& c
         bookRet.clear ();
 }
 
+bool OrderBookDB::isBookToXRP(const uint160& issuerID, const uint160& currencyID)
+{
+    ScopedLockType sl (mLock, __FILE__, __LINE__);
+
+    return mXRPBooks.count(currencyIssuer_ct(currencyID, issuerID)) > 0;
+}
+
 // return list of all orderbooks that give this issuerID and currencyID
 void OrderBookDB::getBooksByTakerGets (const uint160& issuerID, const uint160& currencyID,
                                        std::vector<OrderBook::pointer>& bookRet)

modules/ripple_app/ledger/OrderBookDB.h

@@ -50,6 +50,8 @@ public:
     void getBooksByTakerGets (const uint160& issuerID, const uint160& currencyID,
                               std::vector<OrderBook::pointer>& bookRet);
 
+    bool isBookToXRP (const uint160& issuerID, const uint160& currencyID);
+
     BookListeners::pointer getBookListeners (const uint160& currencyPays, const uint160& currencyGets,
             const uint160& issuerPays, const uint160& issuerGets);
 
@@ -61,11 +63,12 @@ public:
 
 private:
     boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mSourceMap;   // by ci/ii
+    boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mDestMap;     // by co/io
+    boost::unordered_set< currencyIssuer_t > mXRPBooks; // does an order book to XRP exist
     typedef RippleRecursiveMutex LockType;
     typedef LockType::ScopedLockType ScopedLockType;
     LockType mLock;
 
-    boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mDestMap;     // by co/io
 
     // issuerPays, issuerGets, currencyPays, currencyGets
     std::map<uint160, std::map<uint160, std::map<uint160, std::map<uint160, BookListeners::pointer> > > > mListeners;

modules/ripple_app/ledger/ripple_LedgerEntrySet.cpp

@@ -1142,7 +1142,7 @@ uint32 LedgerEntrySet::rippleTransferRate (const uint160& uIssuerID)
                                   ? sleAccount->getFieldU32 (sfTransferRate)
                                   : QUALITY_ONE;
 
-    WriteLog (lsDEBUG, LedgerEntrySet) << boost::str (boost::format ("rippleTransferRate: uIssuerID=%s account_exists=%d transfer_rate=%f")
+    WriteLog (lsTRACE, LedgerEntrySet) << boost::str (boost::format ("rippleTransferRate: uIssuerID=%s account_exists=%d transfer_rate=%f")
                                        % RippleAddress::createHumanAccountID (uIssuerID)
                                        % !!sleAccount
                                        % (uQuality / 1000000000.0));

modules/ripple_app/main/ripple_Application.cpp

@@ -456,6 +456,7 @@ public:
             updateTables ();
 
         mFeatures->addInitialFeatures ();
+        Pathfinder::initPathTable ();
 
         if (getConfig ().START_UP == Config::FRESH)
         {

modules/ripple_app/paths/ripple_PathRequest.cpp

@@ -16,6 +16,8 @@ PathRequest::PathRequest (const boost::shared_ptr<InfoSub>& subscriber)
     , jvStatus (Json::objectValue)
     , bValid (false)
     , bNew (true)
+    , iLastLevel (0)
+    , bLastSuccess (false)
 {
 }
 
@@ -266,6 +268,43 @@ bool PathRequest::doUpdate (RippleLineCache::ref cache, bool fast)
 
     Json::Value jvArray = Json::arrayValue;
 
+    int iLevel = iLastLevel;
+    bool loaded = getApp().getFeeTrack().isLoadedLocal();
+
+    if (iLevel == 0)
+    { // first pass
+        if (loaded)
+            iLevel = getConfig().PATH_SEARCH_FAST;
+        else if (!fast)
+            iLevel = getConfig().PATH_SEARCH_OLD;
+        else if (getConfig().PATH_SEARCH < getConfig().PATH_SEARCH_MAX)
+            iLevel = getConfig().PATH_SEARCH + 1; // start with an extra boost
+        else
+            iLevel = getConfig().PATH_SEARCH;
+    }
+    else if ((iLevel == getConfig().PATH_SEARCH_FAST) && !fast)
+    { // leaving fast pathfinding
+        iLevel = getConfig().PATH_SEARCH;
+        if (loaded && (iLevel > getConfig().PATH_SEARCH_FAST))
+            --iLevel;
+        else if (!loaded && (iLevel < getConfig().PATH_SEARCH))
+            ++iLevel;
+    }
+    else if (bLastSuccess)
+    { // decrement, if possible
+        if ((iLevel > getConfig().PATH_SEARCH) || (loaded && (iLevel > getConfig().PATH_SEARCH_FAST)))
+            --iLevel;
+    }
+    else
+    { // adjust as needed
+        if (!loaded && (iLevel < getConfig().PATH_SEARCH_MAX))
+            ++iLevel;
+        if (loaded && (iLevel > getConfig().PATH_SEARCH_FAST))
+            --iLevel;
+    }
+
+    bool found = false;
+
     BOOST_FOREACH (const currIssuer_t & currIssuer, sourceCurrencies)
     {
         {
@@ -273,12 +312,12 @@ bool PathRequest::doUpdate (RippleLineCache::ref cache, bool fast)
             WriteLog (lsDEBUG, PathRequest) << "Trying to find paths: " << test.getFullText ();
         }
         bool valid;
-        STPathSet spsPaths;
+        STPathSet& spsPaths = mContext[currIssuer];
         Pathfinder pf (cache, raSrcAccount, raDstAccount,
                        currIssuer.first, currIssuer.second, saDstAmount, valid);
         CondLog (!valid, lsINFO, PathRequest) << "PF request not valid";
 
-        if (valid && pf.findPaths (getConfig ().PATH_SEARCH_SIZE - (fast ? 1 : 0), 3, spsPaths))
+        if (valid && pf.findPaths (iLevel, 4, spsPaths))
         {
             LedgerEntrySet                      lesSandbox (cache->getLedger (), tapNONE);
             std::vector<PathState::pointer>     vpsExpanded;
@@ -298,6 +337,7 @@ bool PathRequest::doUpdate (RippleLineCache::ref cache, bool fast)
                 Json::Value jvEntry (Json::objectValue);
                 jvEntry["source_amount"]    = saMaxAmountAct.getJson (0);
                 jvEntry["paths_computed"]   = spsPaths.getJson (0);
+                found  = true;
                 jvArray.append (jvEntry);
             }
             else
@@ -310,6 +350,10 @@ bool PathRequest::doUpdate (RippleLineCache::ref cache, bool fast)
             WriteLog (lsINFO, PathRequest) << "No paths found";
         }
     }
+
+    iLastLevel = iLevel;
+    bLastSuccess = found;
+
     jvStatus["alternatives"] = jvArray;
     return true;
 }

modules/ripple_app/paths/ripple_PathRequest.h

@@ -20,7 +20,7 @@ class RippleLineCache;
 class PathRequest : public boost::enable_shared_from_this<PathRequest>
 {
 public:
-    typedef boost::weak_ptr<PathRequest>        wptr;
+    typedef boost::weak_ptr<PathRequest>    wptr;
     typedef boost::shared_ptr<PathRequest>  pointer;
     typedef const pointer&                  ref;
     typedef const wptr&                     wref;
@@ -56,15 +56,19 @@ private:
     Json::Value                     jvStatus;                   // Last result
 
     // Client request parameters
-    RippleAddress                   raSrcAccount;
+    RippleAddress                     raSrcAccount;
-    RippleAddress                   raDstAccount;
+    RippleAddress                     raDstAccount;
-    STAmount                        saDstAmount;
+    STAmount                          saDstAmount;
-    std::set<currIssuer_t>          sciSourceCurrencies;
+    std::set<currIssuer_t>            sciSourceCurrencies;
-    std::vector<Json::Value>        vjvBridges;
+    std::vector<Json::Value>          vjvBridges;
+    std::map<currIssuer_t, STPathSet> mContext;
 
     bool                            bValid;
     bool                            bNew;
 
+    int                             iLastLevel;
+    bool                            bLastSuccess;
+
     // Track all requests
     static std::set<wptr>           sRequests;
 

modules/ripple_app/paths/ripple_PathState.cpp

@@ -391,7 +391,7 @@ void PathState::setExpanded (
     const uint160   uOutIssuerID    = saOutReq.getIssuer ();
     const uint160   uSenderIssuerID = !!uMaxCurrencyID ? uSenderID : ACCOUNT_XRP;   // Sender is always issuer for non-XRP.
 
-    WriteLog (lsDEBUG, RippleCalc) << boost::str (boost::format ("setExpanded> %s") % spSourcePath.getJson (0));
+    WriteLog (lsTRACE, RippleCalc) << boost::str (boost::format ("setExpanded> %s") % spSourcePath.getJson (0));
 
     lesEntries  = lesSource.duplicate ();
 
@@ -467,7 +467,7 @@ void PathState::setExpanded (
     {
         if (tesSUCCESS == terStatus)
         {
-            WriteLog (lsDEBUG, RippleCalc) << boost::str (boost::format ("setExpanded: element in path:"));
+            WriteLog (lsTRACE, RippleCalc) << boost::str (boost::format ("setExpanded: element in path:"));
             terStatus   = pushNode (speElement.getNodeType (), speElement.getAccountID (), speElement.getCurrency (), speElement.getIssuerID ());
         }
     }

modules/ripple_app/paths/ripple_Pathfinder.h

@@ -47,24 +47,54 @@ public:
                 const RippleAddress& srcAccountID, const RippleAddress& dstAccountID,
                 const uint160& srcCurrencyID, const uint160& srcIssuerID, const STAmount& dstAmount, bool& bValid);
 
-    bool findPaths (const unsigned int iMaxSteps, const unsigned int iMaxPaths, STPathSet& spsDst);
+    static void initPathTable();
-
+    bool findPaths (int iLevel, const unsigned int iMaxPaths, STPathSet& spsDst);
-    bool bDefaultPath (const STPath& spPath);
 
 private:
-    //  void addOptions(PathOption::pointer tail);
+
+    enum PaymentType
+    {
+        pt_XRP_to_XRP,
+        pt_XRP_to_nonXRP,
+        pt_nonXRP_to_XRP,
+        pt_nonXRP_to_same,
+        pt_nonXRP_to_nonXRP
+    };
+
+    enum NodeType
+    {
+        nt_SOURCE,  // The source account with an issuer account, if required
+        nt_ACCOUNTS,     // Accounts that connect from this source/currency
+        nt_BOOKS,        // Order books that connect to this currency
+        nt_XRP_BOOK,     // The order book from this currency to XRP
+        nt_DEST_BOOK,    // The order book to the destination currency/issuer
+        nt_DESTINATION   // The destination account only
+    };
+
+    typedef std::vector<NodeType>         PathType_t;
+    typedef std::pair<int, PathType_t>    CostedPath_t;
+    typedef std::vector<CostedPath_t>     CostedPathList_t;
 
     // returns true if any building paths are now complete?
     bool checkComplete (STPathSet& retPathSet);
 
-    //  void addPathOption(PathOption::pointer pathOption);
+    static std::string pathTypeToString(PathType_t const&);
 
     bool matchesOrigin (const uint160& currency, const uint160& issuer);
 
     int getPathsOut (const uint160& currency, const uint160& accountID,
                      bool isDestCurrency, const uint160& dest);
 
-private:
+    void addLink(const STPath& currentPath, STPathSet& incompletePaths, int addFlags);
+    void addLink(const STPathSet& currentPaths, STPathSet& incompletePaths, int addFlags);
+    STPathSet& getPaths(const PathType_t& type, bool addComplete = true);
+    STPathSet filterPaths(int iMaxPaths);
+
+    // Our main table of paths
+
+    static std::map<PaymentType, CostedPathList_t> mPathTable;
+    static PathType_t makePath(char const*);
+
     uint160             mSrcAccountID;
     uint160             mDstAccountID;
     STAmount            mDstAmount;
@@ -73,15 +103,21 @@ private:
     STAmount            mSrcAmount;
 
     Ledger::pointer     mLedger;
-    PathState::pointer  mPsDefault;
     LoadEvent::pointer  m_loadEvent;
     RippleLineCache::pointer    mRLCache;
 
+    STPathElement                     mSource;
+    STPathSet                         mCompletePaths;
+    std::map< PathType_t, STPathSet > mPaths;
+
     boost::unordered_map<uint160, AccountItems::pointer>    mRLMap;
     boost::unordered_map<std::pair<uint160, uint160>, int>  mPOMap;
 
-    //  std::list<PathOption::pointer> mBuildingPaths;
+    static const uint32 afADD_ACCOUNTS = 0x001;  // Add ripple paths
-    //  std::list<PathOption::pointer> mCompletePaths;
+    static const uint32 afADD_BOOKS    = 0x002;  // Add order books
+    static const uint32 afOB_XRP       = 0x010;  // Add order book to XRP only
+    static const uint32 afOB_LAST      = 0x040;  // Must link to destination currency
+    static const uint32 afAC_LAST      = 0x080;  // Destination account only
 };
 
 boost::unordered_set<uint160> usAccountDestCurrencies (const RippleAddress& raAccountID, Ledger::ref lrLedger,

modules/ripple_app/peers/ripple_Peer.cpp

@@ -1250,14 +1250,7 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, Application::Sco
                 return;
         }
 
-        if (getApp().getMasterTransaction().fetch(txID, true))
+        WriteLog (lsDEBUG, Peer) << "Got transaction from peer " << getDisplayName () << " : " << txID;
-        {
-            WriteLog (lsDEBUG, Peer) << "Peer " << getDisplayName() << " send old TX " << txID;
-            applyLoadCharge (LT_InvalidRequest);
-            return;
-        }
-
-        WriteLog (lsDEBUG, Peer) << "Got new transaction from peer " << getDisplayName () << " : " << txID;
 
         if (mCluster)
             flags |= SF_TRUSTED | SF_SIGGOOD;
@@ -1943,6 +1936,7 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, Application::ScopedL
         uint256 txHash;
         memcpy (txHash.begin (), packet.ledgerhash ().data (), 32);
         map = getApp().getOPs ().getTXMap (txHash);
+        masterLockHolder.unlock();
 
         if (!map)
         {
@@ -1982,6 +1976,12 @@ void PeerImp::recvGetLedger (protocol::TMGetLedger& packet, Application::ScopedL
     }
     else
     {
+        if (getApp().getFeeTrack().isLoadedLocal() && !mCluster)
+        {
+            WriteLog (lsDEBUG, Peer) << "Too busy to fetch ledger data";
+            return;
+        }
+
         // Figure out what ledger they want
         WriteLog (lsTRACE, Peer) << "Received request for ledger data " << getIP ();
         Ledger::pointer ledger;
@@ -2280,7 +2280,7 @@ void PeerImp::recvLedger (const boost::shared_ptr<protocol::TMLedgerData>& packe
 }
 
 bool PeerImp::hasLedger (uint256 const& hash, uint32 seq) const
-{
+{ // FIXME: mRecentLedgers needs some kind of synchronization
     if ((seq != 0) && (seq >= mMinLedger) && (seq <= mMaxLedger))
         return true;
 

modules/ripple_app/rpc/RPCHandler.cpp

@@ -159,7 +159,7 @@ Json::Value RPCHandler::transactionSign (Json::Value params, bool bSubmit, bool
                 Pathfinder pf (cache, raSrcAddressID, dstAccountID,
                                saSendMax.getCurrency (), saSendMax.getIssuer (), saSend, bValid);
 
-                if (!bValid || !pf.findPaths (getConfig ().PATH_SEARCH_SIZE, 3, spsPaths))
+                if (!bValid || !pf.findPaths (getConfig ().PATH_SEARCH_OLD, 5, spsPaths))
                 {
                     WriteLog (lsDEBUG, RPCHandler) << "transactionSign: build_path: No paths found.";
 
@@ -1513,7 +1513,10 @@ Json::Value RPCHandler::doRipplePathFind (Json::Value params, LoadType* loadType
             bool        bValid;
             Pathfinder  pf (cache, raSrc, raDst, uSrcCurrencyID, uSrcIssuerID, saDstAmount, bValid);
 
-            if (!bValid || !pf.findPaths (getConfig ().PATH_SEARCH_SIZE, 3, spsComputed))
+            int level = getConfig().PATH_SEARCH_OLD;
+            if ((getConfig().PATH_SEARCH_MAX > level) && getApp().getFeeTrack().isLoadedLocal())
+                ++level;
+            if (!bValid || !pf.findPaths (level, 4, spsComputed))
             {
                 WriteLog (lsWARNING, RPCHandler) << "ripple_path_find: No paths found.";
             }

modules/ripple_core/functional/ripple_Config.cpp

@@ -72,7 +72,11 @@ Config::Config ()
 
     LEDGER_HISTORY          = 256;
 
-    PATH_SEARCH_SIZE        = DEFAULT_PATH_SEARCH_SIZE;
+    PATH_SEARCH_OLD         = DEFAULT_PATH_SEARCH_OLD;
+    PATH_SEARCH             = DEFAULT_PATH_SEARCH;
+    PATH_SEARCH_FAST        = DEFAULT_PATH_SEARCH_FAST;
+    PATH_SEARCH_MAX         = DEFAULT_PATH_SEARCH_MAX;
+
     ACCOUNT_PROBE_MAX       = 10;
 
     VALIDATORS_SITE         = DEFAULT_VALIDATORS_SITE;
@@ -502,8 +506,14 @@ void Config::load ()
                     LEDGER_HISTORY = lexicalCastThrow <uint32> (strTemp);
             }
 
-            if (SectionSingleB (secConfig, SECTION_PATH_SEARCH_SIZE, strTemp))
+            if (SectionSingleB (secConfig, SECTION_PATH_SEARCH_OLD, strTemp))
-                PATH_SEARCH_SIZE    = lexicalCastThrow <int> (strTemp);
+                PATH_SEARCH_OLD     = lexicalCastThrow <int> (strTemp);
+            if (SectionSingleB (secConfig, SECTION_PATH_SEARCH, strTemp))
+                PATH_SEARCH         = lexicalCastThrow <int> (strTemp);
+            if (SectionSingleB (secConfig, SECTION_PATH_SEARCH_FAST, strTemp))
+                PATH_SEARCH_FAST    = lexicalCastThrow <int> (strTemp);
+            if (SectionSingleB (secConfig, SECTION_PATH_SEARCH_MAX, strTemp))
+                PATH_SEARCH_MAX     = lexicalCastThrow <int> (strTemp);
 
             if (SectionSingleB (secConfig, SECTION_ACCOUNT_PROBE_MAX, strTemp))
                 ACCOUNT_PROBE_MAX   = lexicalCastThrow <int> (strTemp);

modules/ripple_core/functional/ripple_Config.h

@@ -44,8 +44,10 @@ const int SYSTEM_WEBSOCKET_PUBLIC_PORT  = 6563; // XXX Going away.
 // Might connect with fewer for testing.
 #define DEFAULT_PEER_CONNECT_LOW_WATER  10
 
-// Grows exponentially worse.
+#define DEFAULT_PATH_SEARCH_OLD         7
-#define DEFAULT_PATH_SEARCH_SIZE        4
+#define DEFAULT_PATH_SEARCH             7
+#define DEFAULT_PATH_SEARCH_FAST        2
+#define DEFAULT_PATH_SEARCH_MAX         10
 
 enum SizedItemName
 {
@@ -291,7 +293,10 @@ public:
     //----------------------------------------------------------------------------
 
     // Path searching
-    int                         PATH_SEARCH_SIZE;
+    int                         PATH_SEARCH_OLD;
+    int                         PATH_SEARCH;
+    int                         PATH_SEARCH_FAST;
+    int                         PATH_SEARCH_MAX;
 
     // Validation
     RippleAddress               VALIDATION_SEED, VALIDATION_PUB, VALIDATION_PRIV;

modules/ripple_core/functional/ripple_ConfigSections.h

@@ -37,7 +37,10 @@ struct ConfigSection
 #define SECTION_NETWORK_QUORUM          "network_quorum"
 #define SECTION_NODE_SEED               "node_seed"
 #define SECTION_NODE_SIZE               "node_size"
-#define SECTION_PATH_SEARCH_SIZE        "path_search_size"
+#define SECTION_PATH_SEARCH_OLD         "path_search_old"
+#define SECTION_PATH_SEARCH             "path_search"
+#define SECTION_PATH_SEARCH_FAST        "path_search_fast"
+#define SECTION_PATH_SEARCH_MAX         "path_search_max"
 #define SECTION_PEER_CONNECT_LOW_WATER  "peer_connect_low_water"
 #define SECTION_PEER_IP                 "peer_ip"
 #define SECTION_PEER_PORT               "peer_port"

modules/ripple_data/protocol/ripple_SerializedTypes.h

@@ -1164,6 +1164,12 @@ public:
         ;
     }
 
+    STPathElement ()
+        : mType (0)
+    {
+        ;
+    }
+
     int getNodeType () const
     {
         return mType;
@@ -1193,8 +1199,8 @@ public:
 
     bool operator== (const STPathElement& t) const
     {
-        return mType == t.mType && mAccountID == t.mAccountID && mCurrencyID == t.mCurrencyID &&
+        return ((mType & typeAccount) == (t.mType & typeAccount)) && 
-               mIssuerID == t.mIssuerID;
+            (mAccountID == t.mAccountID) && (mCurrencyID == t.mCurrencyID) && (mIssuerID == t.mIssuerID);
     }
 
 private:
@@ -1368,6 +1374,15 @@ public:
     {
         value.push_back (e);
     }
+    void addUniquePath (const STPath& e)
+    {
+        BOOST_FOREACH(const STPath& p, value)
+        {
+            if (p == e)
+                return;
+        }
+        value.push_back (e);
+    }
 
     bool assembleAdd(STPath const& base, STPathElement const& tail)
     { // assemble base+tail and add it to the set if it's not a duplicate
@@ -1398,6 +1413,15 @@ public:
 
     void printDebug ();
 
+    STPath& operator[](size_t n)
+    {
+        return value[n];
+    }
+    STPath const& operator[](size_t n) const
+    {
+        return value[n];
+    }
+
     std::vector<STPath>::iterator begin ()
     {
         return value.begin ();

rippled-example.cfg

@@ -443,7 +443,7 @@
 #
 #-------------------------------------------------------------------------------
 #
-# 5. Ripple Protcol
+# 5. Ripple Protocol
 #
 #------------------
 #
@@ -554,6 +554,25 @@
 #
 #
 #
+# [path_search]
+#   When searching for paths, the default search aggressiveness. This can take
+#   exponentially more resources as the size is increased.
+#
+#   The default is: 7
+#
+# [path_search_fast]
+# [path_search_max]
+#   When seaching for paths, the minimum and maximum search aggressiveness.
+#
+#   The default for 'path_search_fast' is 2. The default for 'path_search_max' is 10.
+#
+# [path_search_old]
+#
+#   For clients that use the legacy path finding interfaces, the search
+#   agressiveness to use. The default is 7.
+#
+#
+#
 #-------------------------------------------------------------------------------
 #
 # 6. HTTPS Client