fix: Introduce retry for transient errors from DB (#3167)

src/data/BackendInterface.hpp

@@ -4,6 +4,7 @@
 #include "data/LedgerCacheInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/CorruptionDetector.hpp"
+#include "util/Retry.hpp"
 #include "util/Spawn.hpp"
 #include "util/log/Logger.hpp"
 
@@ -35,41 +36,118 @@
 namespace data {
 
 /**
- * @brief Represents a database timeout error.
+ * @brief Represents a transient database error that the caller should retry.
  */
-class DatabaseTimeout : public std::exception {
+class DatabaseError : public std::exception {
+    std::string message_{"Transient database error. Please retry the request"};
+
 public:
+    DatabaseError() = default;
+
+    /**
+     * @brief Construct with a description of the underlying failure.
+     *
+     * @param message What actually went wrong.
+     */
+    explicit DatabaseError(std::string message) : message_{std::move(message)}
+    {
+    }
+
     /**
      * @return The error message as a C string
      */
     [[nodiscard]] char const*
-    what() const throw() override
+    what() const noexcept override
     {
-        return "Database read timed out. Please retry the request";
+        return message_.c_str();
     }
 };
 
-static constexpr std::size_t kDefaultWaitBetweenRetry = 500;
 /**
- * @brief A helper function that catches DatabaseTimeout exceptions and retries indefinitely.
+ * @brief Delay before the first retry in @ref retryOnTimeout().
+ */
+static constexpr std::chrono::milliseconds kDefaultWaitBetweenRetry{500};
+
+/**
+ * @brief Default upper bound for the exponential backoff in @ref retryOnTimeout().
+ */
+static constexpr std::chrono::milliseconds kMaxWaitBetweenRetry{5'000};
+
+/**
+ * @brief Default delays for @ref retryOnTimeout().
+ */
+static constexpr util::Retry::Delays kDefaultRetryDelays{
+    .initial = kDefaultWaitBetweenRetry,
+    .max = kDefaultWaitBetweenRetry
+};
+
+/**
+ * @brief Retry `func` while it throws DatabaseError, suspending the calling coroutine in between.
  *
  * @tparam FnType The type of function object to execute
  * @param func The function object to execute
- * @param waitMs Delay between retry attempts
+ * @param yield The coroutine to suspend between attempts
+ * @param delays The delays to use between attempts
  * @return The same as the return type of func
  */
 template <typename FnType>
 auto
-retryOnTimeout(FnType func, size_t waitMs = kDefaultWaitBetweenRetry)
+retryOnTimeout(
+    FnType func,
+    boost::asio::yield_context yield,
+    util::Retry::Delays delays = kDefaultRetryDelays
+)
 {
     static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
 
+    auto retry = util::makeRetryExponentialBackoff(delays, yield.get_executor());
+
     while (true) {
         try {
             return func();
-        } catch (DatabaseTimeout const&) {
-            LOG(log.error()) << "Database request timed out. Sleeping and retrying ... ";
-            std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
+        } catch (DatabaseError const& e) {
+            auto const delayMs =
+                std::chrono::duration_cast<std::chrono::milliseconds>(retry.delayValue()).count();
+            LOG(log.error()) << e.what() << " (attempt " << retry.attemptNumber() + 1
+                             << "). Retrying in " << delayMs << "ms ...";
+
+            retry.wait(yield);
+        }
+    }
+}
+
+/**
+ * @brief Retry `func` while it throws DatabaseError, blocking the calling thread in between.
+ *
+ * @warning Blocks the calling thread; from a coroutine use the `yield_context` overload instead.
+ *
+ * @tparam FnType The type of function object to execute
+ * @param func The function object to execute
+ * @param delays The delays to use between attempts
+ * @return The same as the return type of func
+ */
+template <typename FnType>
+auto
+retryOnTimeout(FnType func, util::Retry::Delays delays = kDefaultRetryDelays)
+{
+    static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
+
+    util::ExponentialBackoffStrategy backoff{delays};
+    std::size_t attempt = 1;
+
+    while (true) {
+        try {
+            return func();
+        } catch (DatabaseError const& e) {
+            auto const delay = backoff.getDelay();
+            LOG(log.error()) << e.what() << " (attempt " << attempt << "). Retrying in "
+                             << std::chrono::duration_cast<std::chrono::milliseconds>(delay).count()
+                             << "ms ...";
+
+            ++attempt;
+
+            std::this_thread::sleep_for(delay);
+            backoff.increaseDelay();
         }
     }
 }
@@ -105,18 +183,21 @@ synchronous(FnType&& func)
 }
 
 /**
- * @brief Synchronously execute the given function object and retry until no DatabaseTimeout is
+ * @brief Synchronously execute the given function object and retry until no DatabaseError is
  * thrown.
  *
+ * @warning Blocks the calling thread while backing off.
+ *
  * @tparam FnType The type of function object to execute
  * @param func The function object to execute
+ * @param delays The delays to use between attempts
  * @return The same as the return type of func
  */
 template <typename FnType>
 auto
-synchronousAndRetryOnTimeout(FnType&& func)
+synchronousAndRetryOnTimeout(FnType&& func, util::Retry::Delays delays = kDefaultRetryDelays)
 {
-    return retryOnTimeout([&]() { return synchronous(func); });
+    return retryOnTimeout([&]() { return synchronous(func); }, delays);
 }
 
 /**
@@ -139,8 +220,27 @@ public:
     BackendInterface(LedgerCacheInterface& cache) : cache_{cache}
     {
     }
+
     virtual ~BackendInterface() = default;
 
+    /**
+     * @return Delay before the first retry of a request against this backend
+     */
+    [[nodiscard]] virtual std::chrono::milliseconds
+    initialRetryDelay() const
+    {
+        return kDefaultWaitBetweenRetry;
+    }
+
+    /**
+     * @return Upper bound for the retry backoff; equal to @ref initialRetryDelay() means flat
+     */
+    [[nodiscard]] virtual std::chrono::milliseconds
+    maxRetryDelay() const
+    {
+        return kMaxWaitBetweenRetry;
+    }
+
     // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
     // Cache should not be exposed thru BackendInterface
 
@@ -705,7 +805,7 @@ public:
     hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;
 
     /**
-     * @brief Fetches the ledger range from DB retrying until no DatabaseTimeout is thrown.
+     * @brief Fetches the ledger range from DB retrying until no DatabaseError is thrown.
      *
      * @return The ledger range if available; nullopt otherwise
      */

src/data/cassandra/CassandraBackendFamily.hpp

@@ -139,6 +139,24 @@ public:
      */
     CassandraBackendFamily(CassandraBackendFamily&&) = delete;
 
+    /**
+     * @return The configured delay before the first retry
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    initialRetryDelay() const override
+    {
+        return settingsProvider_.getInitialRetryDelay();
+    }
+
+    /**
+     * @return The configured upper bound for the retry backoff
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    maxRetryDelay() const override
+    {
+        return settingsProvider_.getMaxRetryDelay();
+    }
+
     TransactionsAndCursor
     fetchAccountTransactions(
         xrpl::AccountID const& account,

src/data/cassandra/Error.hpp

@@ -80,17 +80,6 @@ public:
         return code_;
     }
 
-    /**
-     * @return true if the wrapped error is considered a timeout; false otherwise
-     */
-    [[nodiscard]] bool
-    isTimeout() const
-    {
-        return code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or
-            code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or code_ == CASS_ERROR_SERVER_UNAVAILABLE or
-            code_ == CASS_ERROR_SERVER_OVERLOADED or code_ == CASS_ERROR_SERVER_READ_TIMEOUT;
-    }
-
     /**
      * @return true if the wrapped error is an invalid query; false otherwise
      */

src/data/cassandra/SettingsProvider.cpp

@@ -3,6 +3,7 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/Constants.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/config/ObjectView.hpp"
 
 #include <cerrno>
@@ -25,6 +26,12 @@ SettingsProvider::SettingsProvider(util::config::ObjectView const& cfg)
     , keyspace_{cfg.get<std::string>("keyspace")}
     , tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
     , replicationFactor_{cfg.get<uint16_t>("replication_factor")}
+    , initialRetryDelay_{util::config::ClioConfigDefinition::toMilliseconds(
+          cfg.get<float>("initial_request_retry_delay")
+      )}
+    , maxRetryDelay_{util::config::ClioConfigDefinition::toMilliseconds(
+          cfg.get<float>("max_request_retry_delay")
+      )}
     , settings_{parseSettings()}
 {
 }
@@ -94,9 +101,9 @@ SettingsProvider::parseSettings() const
     }
 
     if (config_.getValueView("request_timeout").hasValue()) {
-        auto const requestTimeoutSecond = config_.get<uint32_t>("request_timeout");
-        settings.requestTimeout =
-            std::chrono::milliseconds{requestTimeoutSecond * util::kMillisecondsPerSecond};
+        settings.requestTimeout = util::config::ClioConfigDefinition::toMilliseconds(
+            config_.get<float>("request_timeout")
+        );
     }
 
     settings.certificate = parseOptionalCertificate();

src/data/cassandra/SettingsProvider.hpp

@@ -4,6 +4,7 @@
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/config/ObjectView.hpp"
 
+#include <chrono>
 #include <cstdint>
 #include <optional>
 #include <string>
@@ -19,6 +20,8 @@ class SettingsProvider {
     std::string keyspace_;
     std::optional<std::string> tablePrefix_;
     uint16_t replicationFactor_;
+    std::chrono::milliseconds initialRetryDelay_;
+    std::chrono::milliseconds maxRetryDelay_;
     Settings settings_;
 
 public:
@@ -62,6 +65,24 @@ public:
         return replicationFactor_;
     }
 
+    /**
+     * @return Delay before the first retry of a failed request
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    getInitialRetryDelay() const
+    {
+        return initialRetryDelay_;
+    }
+
+    /**
+     * @return Upper bound for the retry backoff
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    getMaxRetryDelay() const
+    {
+        return maxRetryDelay_;
+    }
+
 private:
     [[nodiscard]] std::optional<std::string>
     parseOptionalCertificate() const;

src/data/cassandra/impl/ExecutionStrategy.hpp

@@ -15,6 +15,7 @@
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>
+#include <fmt/format.h>
 
 #include <algorithm>
 #include <atomic>
@@ -27,6 +28,7 @@
 #include <mutex>
 #include <optional>
 #include <stdexcept>
+#include <string>
 #include <thread>
 #include <type_traits>
 #include <vector>
@@ -169,7 +171,7 @@ public:
      *
      * @param preparedStatement Statement to prepare and execute
      * @param args Args to bind to the prepared statement
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     template <typename... Args>
     void
@@ -185,7 +187,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor
      *
      * @param statement Statement to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     write(StatementType&& statement)
@@ -215,7 +217,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor.
      *
      * @param statements Vector of statements to execute as a batch
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     write(std::vector<StatementType>&& statements)
@@ -254,7 +256,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor.
      *
      * @param statements Vector of statements to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     writeEach(std::vector<StatementType>&& statements)
@@ -272,7 +274,7 @@ public:
      * @param token Completion token (yield_context)
      * @param preparedStatement Statement to prepare and execute
      * @param args Args to bind to the prepared statement
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     template <typename... Args>
@@ -289,7 +291,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statements Statements to execute in a batch
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     [[maybe_unused]] ResultOrErrorType
@@ -346,7 +348,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statement Statement to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     [[maybe_unused]] ResultOrErrorType
@@ -402,7 +404,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statements Statements to execute
-     * @throws DatabaseTimeout on db error
+     * @throws DatabaseError on a database error
      * @return Vector of results
      */
     std::vector<ResultType>
@@ -457,7 +459,7 @@ public:
             );
             counters_->registerReadError(errorsCount);
             counters_->registerReadFinished(startTime, statements.size() - errorsCount);
-            throw DatabaseTimeout{};
+            throw DatabaseError{};
         }
         counters_->registerReadFinished(startTime, statements.size());
 
@@ -551,11 +553,13 @@ private:
     void
     throwErrorIfNeeded(CassandraError err) const
     {
-        if (err.isTimeout())
-            throw DatabaseTimeout();
-
+        // NOTE: etl::impl::Loader and etl::impl::Extractor treat std::runtime_error as
+        // "amendment blocked", so only genuinely permanent failures may be thrown as one.
         if (err.isInvalidQuery())
             throw std::runtime_error("Invalid query");
+
+        // anything else, including unclassified codes, is transient and gets retried
+        throw DatabaseError{fmt::format("Database error [{}]: {}", err.code(), err.message())};
     }
 };
 

src/data/cassandra/impl/RetryPolicy.hpp

@@ -28,8 +28,7 @@ public:
     ExponentialBackoffRetryPolicy(boost::asio::io_context& ioc)
         : retry_(
               util::makeRetryExponentialBackoff(
-                  std::chrono::milliseconds(1),
-                  std::chrono::seconds(1),
+                  {.initial = std::chrono::milliseconds(1), .max = std::chrono::seconds(1)},
                   boost::asio::make_strand(ioc)
               )
           )