feat: Option to save cache asyncronously (#2883)

This PR adds an option to save cache to file asynchronously in parallel
with shutting down the rest of Clio services.

docs/config-description.md

@@ -457,6 +457,14 @@ This document provides a list of all available Clio configuration properties in
 - **Constraints**: None
 - **Description**: Max allowed difference between the latest sequence in DB and in cache file. If the cache file is too old (contains too low latest sequence) Clio will reject using it.
 
+### cache.file.async_save
+
+- **Required**: True
+- **Type**: boolean
+- **Default value**: `False`
+- **Constraints**: None
+- **Description**: When false, Clio waits for cache saving to finish before shutting down. When true, cache saving runs in parallel with other shutdown operations.
+
 ### log.channels.[].channel
 
 - **Required**: False

src/data/LedgerCacheSaver.cpp

@@ -30,7 +30,9 @@
 namespace data {
 
 LedgerCacheSaver::LedgerCacheSaver(util::config::ClioConfigDefinition const& config, LedgerCacheInterface const& cache)
-    : cacheFilePath_(config.maybeValue<std::string>("cache.file.path")), cache_(cache)
+    : cacheFilePath_(config.maybeValue<std::string>("cache.file.path"))
+    , cache_(cache)
+    , isAsync_(config.get<bool>("cache.file.async_save"))
 {
 }
 
@@ -56,6 +58,9 @@ LedgerCacheSaver::save()
             LOG(util::LogService::error()) << "Error saving LedgerCache to file: " << success.error();
         }
     });
+    if (not isAsync_) {
+        waitToFinish();
+    }
 }
 
 void

src/data/LedgerCacheSaver.hpp

@@ -53,6 +53,7 @@ class LedgerCacheSaver {
     std::optional<std::string> cacheFilePath_;
     std::reference_wrapper<LedgerCacheInterface const> cache_;
     std::optional<std::thread> savingThread_;
+    bool isAsync_;
 
 public:
     /**

src/util/config/ConfigDefinition.cpp

@@ -361,6 +361,7 @@ getClioConfig()
          {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(gValidateLoadMode)},
          {"cache.file.path", ConfigValue{ConfigType::String}.optional()},
          {"cache.file.max_sequence_age", ConfigValue{ConfigType::Integer}.defaultValue(5000)},
+         {"cache.file.async_save", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
 
          {"log.channels.[].channel",
           Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateChannelName)}},

src/util/config/ConfigDescription.hpp

@@ -282,6 +282,9 @@ This document provides a list of all available Clio configuration properties in
         KV{.key = "cache.file.max_sequence_age",
            .value = "Max allowed difference between the latest sequence in DB and in cache file. If the cache file is "
                     "too old (contains too low latest sequence) Clio will reject using it."},
+        KV{.key = "cache.file.async_save",
+           .value = "When false, Clio waits for cache saving to finish before shutting down. When true, "
+                    "cache saving runs in parallel with other shutdown operations."},
         KV{.key = "log.channels.[].channel", .value = "The name of the log channel."},
         KV{.key = "log.channels.[].level", .value = "The log level for the specific log channel."},
         KV{.key = "log.level",

tests/unit/data/LedgerCacheSaverTests.cpp

@@ -47,17 +47,23 @@ struct LedgerCacheSaverTest : virtual testing::Test {
     constexpr static auto kFILE_PATH = "./cache.bin";
 
     static ClioConfigDefinition
-    generateConfig(bool cacheFilePathHasValue)
+    generateConfig(bool cacheFilePathHasValue, bool asyncSave)
     {
         auto config = ClioConfigDefinition{{
             {"cache.file.path", ConfigValue{ConfigType::String}.optional()},
+            {"cache.file.async_save", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
         }};
 
         ConfigFileJson jsonFile{boost::json::object{}};
         if (cacheFilePathHasValue) {
-            auto const jsonObject =
-                boost::json::parse(fmt::format(R"JSON({{"cache": {{"file": {{"path": "{}"}}}}}})JSON", kFILE_PATH))
-                    .as_object();
+            auto const jsonObject = boost::json::parse(
+                                        fmt::format(
+                                            R"JSON({{"cache": {{"file": {{"path": "{}", "async_save": {} }} }} }})JSON",
+                                            kFILE_PATH,
+                                            asyncSave
+                                        )
+            )
+                                        .as_object();
             jsonFile = ConfigFileJson{jsonObject};
         }
         auto const errors = config.parse(jsonFile);
@@ -68,7 +74,7 @@ struct LedgerCacheSaverTest : virtual testing::Test {
 
 TEST_F(LedgerCacheSaverTest, SaveSuccessfully)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
     LedgerCacheSaver saver{config, cache};
 
     EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce(testing::Return(std::expected<void, std::string>{}));
@@ -79,7 +85,7 @@ TEST_F(LedgerCacheSaverTest, SaveSuccessfully)
 
 TEST_F(LedgerCacheSaverTest, SaveWithError)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
     LedgerCacheSaver saver{config, cache};
 
     EXPECT_CALL(cache, saveToFile(kFILE_PATH))
@@ -91,7 +97,7 @@ TEST_F(LedgerCacheSaverTest, SaveWithError)
 
 TEST_F(LedgerCacheSaverTest, NoSaveWhenPathNotConfigured)
 {
-    auto const config = generateConfig(false);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ false, /* asyncSave = */ true);
 
     LedgerCacheSaver saver{config, cache};
     saver.save();
@@ -100,7 +106,7 @@ TEST_F(LedgerCacheSaverTest, NoSaveWhenPathNotConfigured)
 
 TEST_F(LedgerCacheSaverTest, DestructorWaitsForCompletion)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
 
     std::binary_semaphore semaphore{1};
     std::atomic_bool saveCompleted{false};
@@ -123,7 +129,7 @@ TEST_F(LedgerCacheSaverTest, DestructorWaitsForCompletion)
 
 TEST_F(LedgerCacheSaverTest, WaitToFinishCanBeCalledMultipleTimes)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
     LedgerCacheSaver saver{config, cache};
 
     EXPECT_CALL(cache, saveToFile(kFILE_PATH));
@@ -135,7 +141,7 @@ TEST_F(LedgerCacheSaverTest, WaitToFinishCanBeCalledMultipleTimes)
 
 TEST_F(LedgerCacheSaverTest, WaitToFinishWithoutSaveIsSafe)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
     LedgerCacheSaver saver{config, cache};
     EXPECT_NO_THROW(saver.waitToFinish());
 }
@@ -144,13 +150,61 @@ struct LedgerCacheSaverAssertTest : LedgerCacheSaverTest, common::util::WithMock
 
 TEST_F(LedgerCacheSaverAssertTest, MultipleSavesNotAllowed)
 {
-    auto const config = generateConfig(true);
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
 
     LedgerCacheSaver saver{config, cache};
+    std::binary_semaphore semaphore{0};
 
-    EXPECT_CALL(cache, saveToFile(kFILE_PATH));
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce([&](auto&&) {
+        semaphore.acquire();
+        return std::expected<void, std::string>{};
+    });
     saver.save();
     EXPECT_CLIO_ASSERT_FAIL({ saver.save(); });
+    semaphore.release();
 
     saver.waitToFinish();
 }
+
+TEST_F(LedgerCacheSaverTest, SyncSaveWaitsForCompletion)
+{
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ false);
+
+    std::atomic_bool saveCompleted{false};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce([&]() {
+        std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        saveCompleted = true;
+        return std::expected<void, std::string>{};
+    });
+
+    LedgerCacheSaver saver{config, cache};
+    saver.save();
+    EXPECT_TRUE(saveCompleted);
+}
+
+TEST_F(LedgerCacheSaverTest, AsyncSaveDoesNotWaitForCompletion)
+{
+    auto const config = generateConfig(/* cacheFilePathHasValue = */ true, /* asyncSave = */ true);
+
+    std::binary_semaphore saveStarted{0};
+    std::binary_semaphore continueExecution{0};
+    std::atomic_bool saveCompleted{false};
+
+    EXPECT_CALL(cache, saveToFile(kFILE_PATH)).WillOnce([&]() {
+        saveStarted.release();
+        continueExecution.acquire();
+        saveCompleted = true;
+        return std::expected<void, std::string>{};
+    });
+
+    LedgerCacheSaver saver{config, cache};
+    saver.save();
+
+    EXPECT_TRUE(saveStarted.try_acquire_for(std::chrono::seconds{5}));
+    EXPECT_FALSE(saveCompleted);
+
+    continueExecution.release();
+    saver.waitToFinish();
+    EXPECT_TRUE(saveCompleted);
+}