fix: Introduce retry for transient errors from DB (#3167)

src/data/BackendInterface.hpp

@@ -4,6 +4,7 @@
 #include "data/LedgerCacheInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/CorruptionDetector.hpp"
+#include "util/Retry.hpp"
 #include "util/Spawn.hpp"
 #include "util/log/Logger.hpp"
 
@@ -35,41 +36,118 @@
 namespace data {
 
 /**
- * @brief Represents a database timeout error.
+ * @brief Represents a transient database error that the caller should retry.
  */
-class DatabaseTimeout : public std::exception {
+class DatabaseError : public std::exception {
+    std::string message_{"Transient database error. Please retry the request"};
+
 public:
+    DatabaseError() = default;
+
+    /**
+     * @brief Construct with a description of the underlying failure.
+     *
+     * @param message What actually went wrong.
+     */
+    explicit DatabaseError(std::string message) : message_{std::move(message)}
+    {
+    }
+
     /**
      * @return The error message as a C string
      */
     [[nodiscard]] char const*
-    what() const throw() override
+    what() const noexcept override
     {
-        return "Database read timed out. Please retry the request";
+        return message_.c_str();
     }
 };
 
-static constexpr std::size_t kDefaultWaitBetweenRetry = 500;
 /**
- * @brief A helper function that catches DatabaseTimeout exceptions and retries indefinitely.
+ * @brief Delay before the first retry in @ref retryOnTimeout().
+ */
+static constexpr std::chrono::milliseconds kDefaultWaitBetweenRetry{500};
+
+/**
+ * @brief Default upper bound for the exponential backoff in @ref retryOnTimeout().
+ */
+static constexpr std::chrono::milliseconds kMaxWaitBetweenRetry{5'000};
+
+/**
+ * @brief Default delays for @ref retryOnTimeout().
+ */
+static constexpr util::Retry::Delays kDefaultRetryDelays{
+    .initial = kDefaultWaitBetweenRetry,
+    .max = kDefaultWaitBetweenRetry
+};
+
+/**
+ * @brief Retry `func` while it throws DatabaseError, suspending the calling coroutine in between.
  *
  * @tparam FnType The type of function object to execute
  * @param func The function object to execute
- * @param waitMs Delay between retry attempts
+ * @param yield The coroutine to suspend between attempts
+ * @param delays The delays to use between attempts
  * @return The same as the return type of func
  */
 template <typename FnType>
 auto
-retryOnTimeout(FnType func, size_t waitMs = kDefaultWaitBetweenRetry)
+retryOnTimeout(
+    FnType func,
+    boost::asio::yield_context yield,
+    util::Retry::Delays delays = kDefaultRetryDelays
+)
 {
     static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
 
+    auto retry = util::makeRetryExponentialBackoff(delays, yield.get_executor());
+
     while (true) {
         try {
             return func();
-        } catch (DatabaseTimeout const&) {
-            LOG(log.error()) << "Database request timed out. Sleeping and retrying ... ";
-            std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
+        } catch (DatabaseError const& e) {
+            auto const delayMs =
+                std::chrono::duration_cast<std::chrono::milliseconds>(retry.delayValue()).count();
+            LOG(log.error()) << e.what() << " (attempt " << retry.attemptNumber() + 1
+                             << "). Retrying in " << delayMs << "ms ...";
+
+            retry.wait(yield);
+        }
+    }
+}
+
+/**
+ * @brief Retry `func` while it throws DatabaseError, blocking the calling thread in between.
+ *
+ * @warning Blocks the calling thread; from a coroutine use the `yield_context` overload instead.
+ *
+ * @tparam FnType The type of function object to execute
+ * @param func The function object to execute
+ * @param delays The delays to use between attempts
+ * @return The same as the return type of func
+ */
+template <typename FnType>
+auto
+retryOnTimeout(FnType func, util::Retry::Delays delays = kDefaultRetryDelays)
+{
+    static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
+
+    util::ExponentialBackoffStrategy backoff{delays};
+    std::size_t attempt = 1;
+
+    while (true) {
+        try {
+            return func();
+        } catch (DatabaseError const& e) {
+            auto const delay = backoff.getDelay();
+            LOG(log.error()) << e.what() << " (attempt " << attempt << "). Retrying in "
+                             << std::chrono::duration_cast<std::chrono::milliseconds>(delay).count()
+                             << "ms ...";
+
+            ++attempt;
+
+            std::this_thread::sleep_for(delay);
+            backoff.increaseDelay();
         }
     }
 }
@@ -105,18 +183,21 @@ synchronous(FnType&& func)
 }
 
 /**
- * @brief Synchronously execute the given function object and retry until no DatabaseTimeout is
+ * @brief Synchronously execute the given function object and retry until no DatabaseError is
  * thrown.
  *
+ * @warning Blocks the calling thread while backing off.
+ *
  * @tparam FnType The type of function object to execute
  * @param func The function object to execute
+ * @param delays The delays to use between attempts
  * @return The same as the return type of func
  */
 template <typename FnType>
 auto
-synchronousAndRetryOnTimeout(FnType&& func)
+synchronousAndRetryOnTimeout(FnType&& func, util::Retry::Delays delays = kDefaultRetryDelays)
 {
-    return retryOnTimeout([&]() { return synchronous(func); });
+    return retryOnTimeout([&]() { return synchronous(func); }, delays);
 }
 
 /**
@@ -139,8 +220,27 @@ public:
     BackendInterface(LedgerCacheInterface& cache) : cache_{cache}
     {
     }
+
     virtual ~BackendInterface() = default;
 
+    /**
+     * @return Delay before the first retry of a request against this backend
+     */
+    [[nodiscard]] virtual std::chrono::milliseconds
+    initialRetryDelay() const
+    {
+        return kDefaultWaitBetweenRetry;
+    }
+
+    /**
+     * @return Upper bound for the retry backoff; equal to @ref initialRetryDelay() means flat
+     */
+    [[nodiscard]] virtual std::chrono::milliseconds
+    maxRetryDelay() const
+    {
+        return kMaxWaitBetweenRetry;
+    }
+
     // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
     // Cache should not be exposed thru BackendInterface
 
@@ -705,7 +805,7 @@ public:
     hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;
 
     /**
-     * @brief Fetches the ledger range from DB retrying until no DatabaseTimeout is thrown.
+     * @brief Fetches the ledger range from DB retrying until no DatabaseError is thrown.
      *
      * @return The ledger range if available; nullopt otherwise
      */

src/data/cassandra/CassandraBackendFamily.hpp

@@ -139,6 +139,24 @@ public:
      */
     CassandraBackendFamily(CassandraBackendFamily&&) = delete;
 
+    /**
+     * @return The configured delay before the first retry
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    initialRetryDelay() const override
+    {
+        return settingsProvider_.getInitialRetryDelay();
+    }
+
+    /**
+     * @return The configured upper bound for the retry backoff
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    maxRetryDelay() const override
+    {
+        return settingsProvider_.getMaxRetryDelay();
+    }
+
     TransactionsAndCursor
     fetchAccountTransactions(
         xrpl::AccountID const& account,

src/data/cassandra/Error.hpp

@@ -80,17 +80,6 @@ public:
         return code_;
     }
 
-    /**
-     * @return true if the wrapped error is considered a timeout; false otherwise
-     */
-    [[nodiscard]] bool
-    isTimeout() const
-    {
-        return code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or
-            code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or code_ == CASS_ERROR_SERVER_UNAVAILABLE or
-            code_ == CASS_ERROR_SERVER_OVERLOADED or code_ == CASS_ERROR_SERVER_READ_TIMEOUT;
-    }
-
     /**
      * @return true if the wrapped error is an invalid query; false otherwise
      */

src/data/cassandra/SettingsProvider.cpp

@@ -3,6 +3,7 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/Constants.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/config/ObjectView.hpp"
 
 #include <cerrno>
@@ -25,6 +26,12 @@ SettingsProvider::SettingsProvider(util::config::ObjectView const& cfg)
     , keyspace_{cfg.get<std::string>("keyspace")}
     , tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
     , replicationFactor_{cfg.get<uint16_t>("replication_factor")}
+    , initialRetryDelay_{util::config::ClioConfigDefinition::toMilliseconds(
+          cfg.get<float>("initial_request_retry_delay")
+      )}
+    , maxRetryDelay_{util::config::ClioConfigDefinition::toMilliseconds(
+          cfg.get<float>("max_request_retry_delay")
+      )}
     , settings_{parseSettings()}
 {
 }
@@ -94,9 +101,9 @@ SettingsProvider::parseSettings() const
     }
 
     if (config_.getValueView("request_timeout").hasValue()) {
-        auto const requestTimeoutSecond = config_.get<uint32_t>("request_timeout");
-        settings.requestTimeout =
-            std::chrono::milliseconds{requestTimeoutSecond * util::kMillisecondsPerSecond};
+        settings.requestTimeout = util::config::ClioConfigDefinition::toMilliseconds(
+            config_.get<float>("request_timeout")
+        );
     }
 
     settings.certificate = parseOptionalCertificate();

src/data/cassandra/SettingsProvider.hpp

@@ -4,6 +4,7 @@
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/config/ObjectView.hpp"
 
+#include <chrono>
 #include <cstdint>
 #include <optional>
 #include <string>
@@ -19,6 +20,8 @@ class SettingsProvider {
     std::string keyspace_;
     std::optional<std::string> tablePrefix_;
     uint16_t replicationFactor_;
+    std::chrono::milliseconds initialRetryDelay_;
+    std::chrono::milliseconds maxRetryDelay_;
     Settings settings_;
 
 public:
@@ -62,6 +65,24 @@ public:
         return replicationFactor_;
     }
 
+    /**
+     * @return Delay before the first retry of a failed request
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    getInitialRetryDelay() const
+    {
+        return initialRetryDelay_;
+    }
+
+    /**
+     * @return Upper bound for the retry backoff
+     */
+    [[nodiscard]] std::chrono::milliseconds
+    getMaxRetryDelay() const
+    {
+        return maxRetryDelay_;
+    }
+
 private:
     [[nodiscard]] std::optional<std::string>
     parseOptionalCertificate() const;

src/data/cassandra/impl/ExecutionStrategy.hpp

@@ -15,6 +15,7 @@
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>
+#include <fmt/format.h>
 
 #include <algorithm>
 #include <atomic>
@@ -27,6 +28,7 @@
 #include <mutex>
 #include <optional>
 #include <stdexcept>
+#include <string>
 #include <thread>
 #include <type_traits>
 #include <vector>
@@ -169,7 +171,7 @@ public:
      *
      * @param preparedStatement Statement to prepare and execute
      * @param args Args to bind to the prepared statement
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     template <typename... Args>
     void
@@ -185,7 +187,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor
      *
      * @param statement Statement to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     write(StatementType&& statement)
@@ -215,7 +217,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor.
      *
      * @param statements Vector of statements to execute as a batch
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     write(std::vector<StatementType>&& statements)
@@ -254,7 +256,7 @@ public:
      * Retries forever with retry policy specified by @ref AsyncExecutor.
      *
      * @param statements Vector of statements to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      */
     void
     writeEach(std::vector<StatementType>&& statements)
@@ -272,7 +274,7 @@ public:
      * @param token Completion token (yield_context)
      * @param preparedStatement Statement to prepare and execute
      * @param args Args to bind to the prepared statement
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     template <typename... Args>
@@ -289,7 +291,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statements Statements to execute in a batch
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     [[maybe_unused]] ResultOrErrorType
@@ -346,7 +348,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statement Statement to execute
-     * @throws DatabaseTimeout on timeout
+     * @throws DatabaseError on a database error
      * @return ResultType or error wrapped in Expected
      */
     [[maybe_unused]] ResultOrErrorType
@@ -402,7 +404,7 @@ public:
      *
      * @param token Completion token (yield_context)
      * @param statements Statements to execute
-     * @throws DatabaseTimeout on db error
+     * @throws DatabaseError on a database error
      * @return Vector of results
      */
     std::vector<ResultType>
@@ -457,7 +459,7 @@ public:
             );
             counters_->registerReadError(errorsCount);
             counters_->registerReadFinished(startTime, statements.size() - errorsCount);
-            throw DatabaseTimeout{};
+            throw DatabaseError{};
         }
         counters_->registerReadFinished(startTime, statements.size());
 
@@ -551,11 +553,13 @@ private:
     void
     throwErrorIfNeeded(CassandraError err) const
     {
-        if (err.isTimeout())
-            throw DatabaseTimeout();
-
+        // NOTE: etl::impl::Loader and etl::impl::Extractor treat std::runtime_error as
+        // "amendment blocked", so only genuinely permanent failures may be thrown as one.
         if (err.isInvalidQuery())
             throw std::runtime_error("Invalid query");
+
+        // anything else, including unclassified codes, is transient and gets retried
+        throw DatabaseError{fmt::format("Database error [{}]: {}", err.code(), err.message())};
     }
 };
 

src/data/cassandra/impl/RetryPolicy.hpp

@@ -28,8 +28,7 @@ public:
     ExponentialBackoffRetryPolicy(boost::asio::io_context& ioc)
         : retry_(
               util::makeRetryExponentialBackoff(
-                  std::chrono::milliseconds(1),
-                  std::chrono::seconds(1),
+                  {.initial = std::chrono::milliseconds(1), .max = std::chrono::seconds(1)},
                   boost::asio::make_strand(ioc)
               )
           )

src/etl/impl/CacheLoader.hpp

@@ -145,9 +145,17 @@ private:
             LOG(log_.debug()) << "Starting a cursor: " << xrpl::strHex(start);
 
             while (not token.isStopRequested() and not cache_.get().isDisabled()) {
-                auto res = data::retryOnTimeout([this, seq, cachePageFetchSize, &start, token]() {
-                    return backend_->fetchLedgerPage(start, seq, cachePageFetchSize, false, token);
-                });
+                auto res = data::retryOnTimeout(
+                    [this, seq, cachePageFetchSize, &start, token]() {
+                        return backend_->fetchLedgerPage(
+                            start, seq, cachePageFetchSize, false, token
+                        );
+                    },
+                    token,
+                    util::Retry::Delays{
+                        .initial = backend_->initialRetryDelay(), .max = backend_->maxRetryDelay()
+                    }
+                );
 
                 cache_.get().update(res.objects, seq, true);
 

src/etl/impl/SubscriptionSource.cpp

@@ -58,7 +58,9 @@ SubscriptionSource::SubscriptionSource(
     , subscriptions_(std::move(subscriptions))
     , strand_(boost::asio::make_strand(ioContext))
     , wsTimeout_(wsTimeout)
-    , retry_(util::makeRetryExponentialBackoff(retryDelay, kRetryMaxDelay, strand_))
+    , retry_(
+          util::makeRetryExponentialBackoff({.initial = retryDelay, .max = kMaxRetryDelay}, strand_)
+      )
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))

src/etl/impl/SubscriptionSource.hpp

@@ -75,7 +75,7 @@ private:
     util::StopHelper stopHelper_;
 
     static constexpr std::chrono::seconds kWsTimeout{30};
-    static constexpr std::chrono::seconds kRetryMaxDelay{30};
+    static constexpr std::chrono::seconds kMaxRetryDelay{30};
     static constexpr std::chrono::seconds kRetryDelay{1};
 
 public:

src/rpc/Errors.hpp

@@ -239,7 +239,7 @@ public:
      * @return The error message
      */
     [[nodiscard]] char const*
-    what() const throw() override
+    what() const noexcept override
     {
         return msg_.c_str();
     }
@@ -267,7 +267,7 @@ public:
      * @return The error message
      */
     [[nodiscard]] char const*
-    what() const throw() override
+    what() const noexcept override
     {
         return account_.c_str();
     }

src/rpc/RPCEngine.hpp

@@ -178,8 +178,8 @@ public:
             }
 
             return Result{std::move(v)};
-        } catch (data::DatabaseTimeout const& t) {
-            LOG(log_.error()) << "Database timeout";
+        } catch (data::DatabaseError const& t) {
+            LOG(log_.error()) << "Database error: " << t.what();
             notifyTooBusy();
 
             return Result{Status{RippledError::RpcTooBusy}};
@@ -361,8 +361,8 @@ private:
             }
 
             return Result{std::move(v)};
-        } catch (data::DatabaseTimeout const& t) {
-            LOG(log_.error()) << "Database timeout";
+        } catch (data::DatabaseError const& t) {
+            LOG(log_.error()) << "Database error: " << t.what();
             notifyTooBusy();
 
             return Result{Status{RippledError::RpcTooBusy}};

src/util/Retry.cpp

@@ -1,6 +1,8 @@
 #include "util/Retry.hpp"
 
+#include <boost/asio/any_io_executor.hpp>
 #include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
 #include <boost/asio/strand.hpp>
 
 #include <algorithm>
@@ -42,6 +44,11 @@ Retry::Retry(
 {
 }
 
+Retry::Retry(RetryStrategyPtr strategy, boost::asio::any_io_executor executor)
+    : strategy_(std::move(strategy)), timer_(executor)
+{
+}
+
 Retry::~Retry()
 {
     *canceled_ = true;
@@ -73,11 +80,8 @@ Retry::reset()
     (*strategy_).reset();
 }
 
-ExponentialBackoffStrategy::ExponentialBackoffStrategy(
-    std::chrono::steady_clock::duration delay,
-    std::chrono::steady_clock::duration maxDelay
-)
-    : RetryStrategy(delay), maxDelay_(maxDelay)
+ExponentialBackoffStrategy::ExponentialBackoffStrategy(Retry::Delays delays)
+    : RetryStrategy(delays.initial), maxDelay_(delays.max)
 {
 }
 
@@ -88,14 +92,32 @@ ExponentialBackoffStrategy::nextDelay() const
     return std::min(next, maxDelay_);
 }
 
+void
+Retry::wait(boost::asio::yield_context yield)
+{
+    *canceled_ = false;
+    timer_.expires_after(strategy_->getDelay());
+    strategy_->increaseDelay();
+    ++attemptNumber_;
+
+    // error ignored on purpose: a cancelled timer just means the caller retries sooner
+    boost::system::error_code ec;
+    timer_.async_wait(yield[ec]);
+}
+
 Retry
 makeRetryExponentialBackoff(
-    std::chrono::steady_clock::duration delay,
-    std::chrono::steady_clock::duration maxDelay,
+    Retry::Delays delays,
     boost::asio::strand<boost::asio::io_context::executor_type> strand
 )
 {
-    return Retry(std::make_unique<ExponentialBackoffStrategy>(delay, maxDelay), std::move(strand));
+    return Retry(std::make_unique<ExponentialBackoffStrategy>(delays), std::move(strand));
+}
+
+Retry
+makeRetryExponentialBackoff(Retry::Delays delays, boost::asio::any_io_executor executor)
+{
+    return Retry(std::make_unique<ExponentialBackoffStrategy>(delays), std::move(executor));
 }
 
 }  // namespace util

src/util/Retry.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <boost/asio/any_io_executor.hpp>
 #include <boost/asio/error.hpp>
 #include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <boost/asio/strand.hpp>
 
@@ -65,6 +67,16 @@ class Retry {
     std::shared_ptr<std::atomic_bool> canceled_{std::make_shared<std::atomic_bool>(false)};
 
 public:
+    /**
+     * @brief The delays to use between retry attempts
+     *
+     * Equal `initial` and `max` mean a flat delay with no backoff.
+     */
+    struct Delays {
+        std::chrono::steady_clock::duration initial;
+        std::chrono::steady_clock::duration max;
+    };
+
     /**
      * @brief Construct a new Retry object
      *
@@ -76,6 +88,16 @@ public:
         boost::asio::strand<boost::asio::io_context::executor_type> strand
     );
 
+    /**
+     * @brief Construct a new Retry object from any I/O executor
+     *
+     * For coroutines, pass `yield.get_executor()` and drive it with @ref wait().
+     *
+     * @param strategy The retry strategy to use
+     * @param executor The executor to run the retry timer on
+     */
+    Retry(RetryStrategyPtr strategy, boost::asio::any_io_executor executor);
+
     /**
      * @brief Destroy the Retry object
      */
@@ -105,6 +127,17 @@ public:
         });
     }
 
+    /**
+     * @brief Wait out the current delay by suspending the calling coroutine, then back off.
+     *
+     * Unlike @ref retry() this returns once the delay elapsed instead of scheduling a callback, so
+     * the caller can keep its own loop. Advances the delay and attempt number like @ref retry().
+     *
+     * @param yield The coroutine to suspend
+     */
+    void
+    wait(boost::asio::yield_context yield);
+
     /**
      * @brief Cancel scheduled retry if any
      */
@@ -140,13 +173,9 @@ public:
     /**
      * @brief Construct a new Exponential Backoff Strategy object
      *
-     * @param delay The initial delay value
-     * @param maxDelay The maximum delay value
+     * @param delays The delays to use between attempts
      */
-    ExponentialBackoffStrategy(
-        std::chrono::steady_clock::duration delay,
-        std::chrono::steady_clock::duration maxDelay
-    );
+    explicit ExponentialBackoffStrategy(Retry::Delays delays);
 
 private:
     [[nodiscard]] std::chrono::steady_clock::duration
@@ -156,16 +185,24 @@ private:
 /**
  * @brief Create a retry mechanism with exponential backoff strategy
  *
- * @param delay The initial delay value
- * @param maxDelay The maximum delay value
+ * @param delays The delays to use between attempts
  * @param strand The strand to use for async operations
  * @return The retry object
  */
 Retry
 makeRetryExponentialBackoff(
-    std::chrono::steady_clock::duration delay,
-    std::chrono::steady_clock::duration maxDelay,
+    Retry::Delays delays,
     boost::asio::strand<boost::asio::io_context::executor_type> strand
 );
 
+/**
+ * @brief Create a retry mechanism with exponential backoff strategy on any I/O executor
+ *
+ * @param delays The delays to use between attempts
+ * @param executor The executor to run the retry timer on
+ * @return The retry object
+ */
+Retry
+makeRetryExponentialBackoff(Retry::Delays delays, boost::asio::any_io_executor executor);
+
 }  // namespace util

src/util/config/ConfigDefinition.cpp

@@ -278,7 +278,15 @@ getClioConfig()
          {"database.cassandra.connect_timeout",
           ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateUint32)},
          {"database.cassandra.request_timeout",
-          ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateUint32)},
+          ConfigValue{ConfigType::Double}.optional().withConstraint(gValidatePositiveDouble)},
+         {"database.cassandra.initial_request_retry_delay",
+          ConfigValue{ConfigType::Double}.defaultValue(0.5).withConstraint(
+              gValidatePositiveDouble
+          )},
+         {"database.cassandra.max_request_retry_delay",
+          ConfigValue{ConfigType::Double}.defaultValue(5.0).withConstraint(
+              gValidatePositiveDouble
+          )},
          {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
          {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
          {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},

src/util/config/ConfigDescription.hpp

@@ -182,8 +182,14 @@ This document provides a list of all available Clio configuration properties in
                     "established."},
         KV{.key = "database.cassandra.request_timeout",
            .value = "The maximum amount of time in seconds that the system waits for a request to "
-                    "be fetched from the "
-                    "database."},
+                    "be fetched from the database. Should be set higher than the server side read "
+                    "timeout. If omitted, no request timeout is applied."},
+        KV{.key = "database.cassandra.initial_request_retry_delay",
+           .value = "How long in seconds to wait before the first retry of a database request "
+                    "that failed with a transient error."},
+        KV{.key = "database.cassandra.max_request_retry_delay",
+           .value = "Upper bound in seconds for the exponential backoff between retries of a "
+                    "database request."},
         KV{.key = "database.cassandra.username",
            .value = "The username used for authenticating with the database."},
         KV{.key = "database.cassandra.password",