feat: Graceful shutdown with old web server (#2786)

- Stop accepting connections during graceful shutdown in the old web server
- Stop all the services before Clio exits
- Move cache saving into stop callback

tests/unit/CMakeLists.txt

@@ -11,6 +11,7 @@ target_sources(
           data/BackendCountersTests.cpp
           data/BackendInterfaceTests.cpp
           data/LedgerCacheTests.cpp
+          data/LedgerCacheSaverTests.cpp
           data/cassandra/AsyncExecutorTests.cpp
           data/cassandra/ExecutionStrategyTests.cpp
           data/cassandra/LedgerHeaderCacheTests.cpp

tests/unit/app/StopperTests.cpp

@@ -24,7 +24,7 @@
 #include "util/MockPrometheus.hpp"
 #include "util/MockSubscriptionManager.hpp"
 #include "util/config/ConfigDefinition.hpp"
-#include "web/ng/Server.hpp"
+#include "web/interface/Concepts.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/io_context.hpp>
@@ -61,16 +61,22 @@ TEST_F(StopperTest, stopCalledMultipleTimes)
 }
 
 struct StopperMakeCallbackTest : util::prometheus::WithPrometheus, SyncAsioContextTest {
-    struct ServerMock : web::ng::ServerTag {
+    struct ServerMock : web::ServerTag {
         MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
     };
 
+    struct MockLedgerCacheSaver {
+        MOCK_METHOD(void, save, ());
+        MOCK_METHOD(void, waitToFinish, ());
+    };
+
 protected:
     testing::StrictMock<ServerMock> serverMock_;
     testing::StrictMock<MockLoadBalancer> loadBalancerMock_;
     testing::StrictMock<MockETLService> etlServiceMock_;
     testing::StrictMock<MockSubscriptionManager> subscriptionManagerMock_;
     testing::StrictMock<MockBackend> backendMock_{util::config::ClioConfigDefinition{}};
+    testing::StrictMock<MockLedgerCacheSaver> cacheSaverMock_;
     boost::asio::io_context ioContextToStop_;
 
     bool
@@ -86,10 +92,17 @@ TEST_F(StopperMakeCallbackTest, makeCallbackTest)
     std::thread t{[this]() { ioContextToStop_.run(); }};
 
     auto callback = Stopper::makeOnStopCallback(
-        serverMock_, loadBalancerMock_, etlServiceMock_, subscriptionManagerMock_, backendMock_, ioContextToStop_
+        serverMock_,
+        loadBalancerMock_,
+        etlServiceMock_,
+        subscriptionManagerMock_,
+        backendMock_,
+        cacheSaverMock_,
+        ioContextToStop_
     );
 
     testing::Sequence const s1, s2;
+    EXPECT_CALL(cacheSaverMock_, save).InSequence(s1).WillOnce([this]() { EXPECT_FALSE(isContextStopped()); });
     EXPECT_CALL(serverMock_, stop).InSequence(s1).WillOnce([this]() { EXPECT_FALSE(isContextStopped()); });
     EXPECT_CALL(loadBalancerMock_, stop).InSequence(s2).WillOnce([this]() { EXPECT_FALSE(isContextStopped()); });
     EXPECT_CALL(etlServiceMock_, stop).InSequence(s1, s2).WillOnce([this]() { EXPECT_FALSE(isContextStopped()); });
@@ -99,6 +112,7 @@ TEST_F(StopperMakeCallbackTest, makeCallbackTest)
     EXPECT_CALL(backendMock_, waitForWritesToFinish).InSequence(s1, s2).WillOnce([this]() {
         EXPECT_FALSE(isContextStopped());
     });
+    EXPECT_CALL(cacheSaverMock_, waitToFinish).InSequence(s1).WillOnce([this]() { EXPECT_FALSE(isContextStopped()); });
 
     runSpawn([&](boost::asio::yield_context yield) {
         callback(yield);

tests/unit/data/LedgerCacheSaverTests.cpp

@@ -0,0 +1,156 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/LedgerCacheSaver.hpp"
+#include "util/MockAssert.hpp"
+#include "util/MockLedgerCache.hpp"
+#include "util/config/ConfigDefinition.hpp"
+#include "util/config/ConfigFileJson.hpp"
+#include "util/config/ConfigValue.hpp"
+#include "util/config/Types.hpp"
+
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/value.hpp>
+#include <fmt/format.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <chrono>
+#include <expected>
+#include <semaphore>
+#include <string>
+#include <thread>
+
+using namespace data;
+using namespace util::config;
+
+struct LedgerCacheSaverTest : virtual testing::Test {
+    testing::StrictMock<MockLedgerCache> cache;
+    constexpr static auto kFILE_PATH = "./cache.bin";
+
+    static ClioConfigDefinition
+    generateConfig(bool cacheFilePathHasValue)
+    {
+        auto config = ClioConfigDefinition{{
+            {"cache.file.path", ConfigValue{ConfigType::String}.optional()},
+        }};
+
+        ConfigFileJson jsonFile{boost::json::object{}};
+        if (cacheFilePathHasValue) {
+            auto const jsonObject =
+                boost::json::parse(fmt::format(R"JSON({{"cache": {{"file": {{"path": "{}"}}}}}})JSON", kFILE_PATH))
+                    .as_object();
+            jsonFile = ConfigFileJson{jsonObject};
+        }
+        auto const errors = config.parse(jsonFile);
+        EXPECT_FALSE(errors.has_value());
+        return config;
+    }
+};
+
+TEST_F(LedgerCacheSaverTest, SaveSuccessfully)
+{
+    auto const config = generateConfig(true);
+    LedgerCacheSaver saver{config, cache};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce(testing::Return(std::expected<void, std::string>{}));
+
+    saver.save();
+    saver.waitToFinish();
+}
+
+TEST_F(LedgerCacheSaverTest, SaveWithError)
+{
+    auto const config = generateConfig(true);
+    LedgerCacheSaver saver{config, cache};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH))
+        .WillOnce(testing::Return(std::expected<void, std::string>(std::unexpected("Failed to save"))));
+
+    saver.save();
+    saver.waitToFinish();
+}
+
+TEST_F(LedgerCacheSaverTest, NoSaveWhenPathNotConfigured)
+{
+    auto const config = generateConfig(false);
+
+    LedgerCacheSaver saver{config, cache};
+    saver.save();
+    saver.waitToFinish();
+}
+
+TEST_F(LedgerCacheSaverTest, DestructorWaitsForCompletion)
+{
+    auto const config = generateConfig(true);
+
+    std::binary_semaphore semaphore{1};
+    std::atomic_bool saveCompleted{false};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce([&]() {
+        semaphore.release();
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        saveCompleted = true;
+        return std::expected<void, std::string>{};
+    });
+
+    {
+        LedgerCacheSaver saver{config, cache};
+        saver.save();
+        EXPECT_TRUE(semaphore.try_acquire_for(std::chrono::seconds{5}));
+    }
+
+    EXPECT_TRUE(saveCompleted);
+}
+
+TEST_F(LedgerCacheSaverTest, WaitToFinishCanBeCalledMultipleTimes)
+{
+    auto const config = generateConfig(true);
+    LedgerCacheSaver saver{config, cache};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH));
+
+    saver.save();
+    saver.waitToFinish();
+    EXPECT_NO_THROW(saver.waitToFinish());
+}
+
+TEST_F(LedgerCacheSaverTest, WaitToFinishWithoutSaveIsSafe)
+{
+    auto const config = generateConfig(true);
+    LedgerCacheSaver saver{config, cache};
+    EXPECT_NO_THROW(saver.waitToFinish());
+}
+
+struct LedgerCacheSaverAssertTest : LedgerCacheSaverTest, common::util::WithMockAssert {};
+
+TEST_F(LedgerCacheSaverAssertTest, MultipleSavesNotAllowed)
+{
+    auto const config = generateConfig(true);
+
+    LedgerCacheSaver saver{config, cache};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH));
+    saver.save();
+    EXPECT_CLIO_ASSERT_FAIL({ saver.save(); });
+
+    saver.waitToFinish();
+}