Merge branch 'develop' into ximinez/fix/validator-cache

- If any [validator_list_keys] are not available after all
  [validator_list_sites] have had a chance to be queried, then fall
  back to loading cache files. Currently, cache files are only used if
  no sites are defined, or the request to one of them has an error. It
  does not include cases where not enough sites are defined, or if a
  site returns an invalid VL (or something else entirely).
- Resolves #5320

src/tests/libxrpl/net/HTTPClient.cpp

@@ -10,7 +10,6 @@
 #include <gtest/gtest.h>
 
 #include <atomic>
-#include <iostream>
 #include <map>
 #include <thread>
 
@@ -18,40 +17,6 @@ using namespace xrpl;
 
 namespace {
 
-struct logger
-{
-    std::string name;
-    logger const* const parent = nullptr;
-    static std::size_t depth;
-    logger(std::string n) : name(n)
-    {
-        std::clog << indent() << name << " begin\n";
-        ++depth;
-    }
-
-    logger(logger const& p, std::string n) : parent(&p), name(n)
-    {
-        std::clog << indent() << parent->name << " : " << name << " begin\n";
-        ++depth;
-    }
-
-    ~logger()
-    {
-        --depth;
-        if (parent)
-            std::clog << indent() << parent->name << " : " << name << " end\n";
-        else
-            std::clog << indent() << name << " end\n";
-    }
-
-    std::string
-    indent()
-    {
-        return std::string(depth, ' ');
-    }
-};
-std::size_t logger::depth = 0;
-
 // Simple HTTP server using Beast for testing
 class TestHTTPServer
 {
@@ -70,7 +35,6 @@ private:
 public:
     TestHTTPServer() : acceptor_(ioc_), port_(0)
     {
-        logger l("TestHTTPServer()");
         // Bind to any available port
         endpoint_ = {boost::asio::ip::tcp::v4(), 0};
         acceptor_.open(endpoint_.protocol());
@@ -86,7 +50,6 @@ public:
 
     ~TestHTTPServer()
     {
-        logger l("~TestHTTPServer()");
         stop();
     }
 
@@ -124,7 +87,6 @@ private:
     void
     stop()
     {
-        logger l("TestHTTPServer::stop");
         running_ = false;
         acceptor_.close();
     }
@@ -132,7 +94,6 @@ private:
     void
     accept()
     {
-        logger l("TestHTTPServer::accept");
         if (!running_)
             return;
 
@@ -154,37 +115,31 @@ private:
     void
     handleConnection(boost::asio::ip::tcp::socket socket)
     {
-        logger l("TestHTTPServer::handleConnection");
         try
         {
-            std::optional<logger> r(std::in_place, l, "read the http request");
             // Read the HTTP request
             boost::beast::flat_buffer buffer;
             boost::beast::http::request<boost::beast::http::string_body> req;
             boost::beast::http::read(socket, buffer, req);
 
             // Create response
-            r.emplace(l, "create response");
             boost::beast::http::response<boost::beast::http::string_body> res;
             res.version(req.version());
             res.result(status_code_);
             res.set(boost::beast::http::field::server, "TestServer");
 
             // Add custom headers
-            r.emplace(l, "add custom headers");
             for (auto const& [name, value] : custom_headers_)
             {
                 res.set(name, value);
             }
 
             // Set body and prepare payload first
-            r.emplace(l, "set body and prepare payload");
             res.body() = response_body_;
             res.prepare_payload();
 
             // Override Content-Length with custom headers after prepare_payload
             // This allows us to test case-insensitive header parsing
-            r.emplace(l, "override content-length");
             for (auto const& [name, value] : custom_headers_)
             {
                 if (boost::iequals(name, "Content-Length"))
@@ -195,25 +150,19 @@ private:
             }
 
             // Send response
-            r.emplace(l, "send response");
             boost::beast::http::write(socket, res);
 
             // Shutdown socket gracefully
-            r.emplace(l, "shutdown socket");
             boost::system::error_code ec;
             socket.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec);
         }
         catch (std::exception const&)
         {
             // Connection handling errors are expected
-            logger c(l, "catch");
         }
 
         if (running_)
-        {
-            logger r(l, "accept");
             accept();
-        }
     }
 };
 
@@ -227,16 +176,12 @@ runHTTPTest(
     std::string& result_data,
     boost::system::error_code& result_error)
 {
-    logger l("runHTTPTest");
     // Create a null journal for testing
     beast::Journal j{beast::Journal::getNullSink()};
 
-    std::optional<logger> r(std::in_place, l, "initializeSSLContext");
-
     // Initialize HTTPClient SSL context
     HTTPClient::initializeSSLContext("", "", false, j);
 
-    r.emplace(l, "HTTPClient::get");
     HTTPClient::get(
         false,  // no SSL
         server.ioc(),
@@ -261,7 +206,6 @@ runHTTPTest(
     while (!completed &&
            std::chrono::steady_clock::now() - start < std::chrono::seconds(10))
     {
-        r.emplace(l, "ioc.run_one");
         if (server.ioc().run_one() == 0)
         {
             break;
@@ -275,8 +219,6 @@ runHTTPTest(
 
 TEST(HTTPClient, case_insensitive_content_length)
 {
-    logger l("HTTPClient case insensitive Content-Length");
-
     // Test different cases of Content-Length header
     std::vector<std::string> header_cases = {
         "Content-Length",  // Standard case
@@ -288,7 +230,6 @@ TEST(HTTPClient, case_insensitive_content_length)
 
     for (auto const& header_name : header_cases)
     {
-        logger h(l, header_name);
         TestHTTPServer server;
         std::string test_body = "Hello World!";
         server.setResponseBody(test_body);
@@ -317,7 +258,6 @@ TEST(HTTPClient, case_insensitive_content_length)
 
 TEST(HTTPClient, basic_http_request)
 {
-    logger l("HTTPClient basic HTTP request");
     TestHTTPServer server;
     std::string test_body = "Test response body";
     server.setResponseBody(test_body);
@@ -339,7 +279,6 @@ TEST(HTTPClient, basic_http_request)
 
 TEST(HTTPClient, empty_response)
 {
-    logger l("HTTPClient empty response");
     TestHTTPServer server;
     server.setResponseBody("");  // Empty body
     server.setHeader("Content-Length", "0");
@@ -360,7 +299,6 @@ TEST(HTTPClient, empty_response)
 
 TEST(HTTPClient, different_status_codes)
 {
-    logger l("HTTPClient different status codes");
     std::vector<unsigned int> status_codes = {200, 404, 500};
 
     for (auto status : status_codes)

src/xrpld/app/misc/detail/ValidatorSite.cpp

@@ -129,7 +129,12 @@ ValidatorSite::load(
     {
         try
         {
-            sites_.emplace_back(uri);
+            // This is not super efficient, but it doesn't happen often.
+            bool found = std::ranges::any_of(sites_, [&uri](auto const& site) {
+                return site.loadedResource->uri == uri;
+            });
+            if (!found)
+                sites_.emplace_back(uri);
         }
         catch (std::exception const& e)
         {
@@ -191,6 +196,17 @@ ValidatorSite::setTimer(
     std::lock_guard<std::mutex> const& site_lock,
     std::lock_guard<std::mutex> const& state_lock)
 {
+    if (!sites_.empty() &&  //
+        std::ranges::all_of(sites_, [](auto const& site) {
+            return site.lastRefreshStatus.has_value();
+        }))
+    {
+        // If all of the sites have been handled at least once (including
+        // errors and timeouts), call missingSite, which will load the cache
+        // files for any lists that are still unavailable.
+        missingSite(site_lock);
+    }
+
     auto next = std::min_element(
         sites_.begin(), sites_.end(), [](Site const& a, Site const& b) {
             return a.nextRefresh < b.nextRefresh;
@@ -303,13 +319,16 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         // processes a network error. Usually, this function runs first,
         // but on extremely rare occasions, the response handler can run
         // first, which will leave activeResource empty.
-        auto const& site = sites_[siteIdx];
+        auto& site = sites_[siteIdx];
         if (site.activeResource)
             JLOG(j_.warn()) << "Request for " << site.activeResource->uri
                             << " took too long";
         else
             JLOG(j_.error()) << "Request took too long, but a response has "
                                 "already been processed";
+        if (!site.lastRefreshStatus)
+            site.lastRefreshStatus.emplace(Site::Status{
+                clock_type::now(), ListDisposition::invalid, "timeout"});
     }
 
     std::lock_guard lock_state{state_mutex_};