fix: Introduce retry for transient errors from DB (#3167)

tests/unit/data/BackendInterfaceTests.cpp

@@ -1,10 +1,13 @@
+#include "data/BackendInterface.hpp"
 #include "etl/CorruptionDetector.hpp"
 #include "etl/SystemState.hpp"
 #include "util/AsioContextTestFixture.hpp"
 #include "util/MockBackendTestFixture.hpp"
 #include "util/MockPrometheus.hpp"
+#include "util/Retry.hpp"
 #include "util/TestObject.hpp"
 
+#include <boost/asio/post.hpp>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <xrpl/basics/Blob.h>
@@ -12,7 +15,12 @@
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/XRPAmount.h>
 
+#include <chrono>
+#include <cstddef>
+#include <exception>
 #include <optional>
+#include <stdexcept>
+#include <type_traits>
 #include <vector>
 
 using namespace data;
@@ -160,3 +168,122 @@ TEST_F(
     });
     EXPECT_FALSE(backend_->cache().isDisabled());
 }
+
+// Loader and Extractor catch std::runtime_error to decide the server must amendment-block, so
+// DatabaseError has to stay outside that hierarchy.
+TEST(BackendInterfaceRetryTest, DatabaseErrorIsNotARuntimeError)
+{
+    static_assert(std::is_base_of_v<std::exception, DatabaseError>);
+    static_assert(not std::is_base_of_v<std::runtime_error, DatabaseError>);
+
+    try {
+        throw DatabaseError{"transient"};
+    } catch (std::runtime_error const&) {
+        FAIL() << "DatabaseError must not be caught as std::runtime_error - doing so would "
+                  "amendment-block the server on a transient database error";
+    } catch (std::exception const& e) {
+        EXPECT_STREQ(e.what(), "transient");
+    }
+}
+
+TEST(BackendInterfaceRetryTest, DatabaseErrorKeepsDefaultMessage)
+{
+    EXPECT_STREQ(DatabaseError{}.what(), "Transient database error. Please retry the request");
+}
+
+TEST(BackendInterfaceRetryTest, RetryOnTimeoutBlockingRetriesUntilSuccess)
+{
+    std::size_t calls = 0;
+
+    auto const result = retryOnTimeout(
+        [&calls]() -> int {
+            if (++calls < 3)
+                throw DatabaseError{};
+            return 42;
+        },
+        util::Retry::Delays{
+            .initial = std::chrono::milliseconds{1}, .max = std::chrono::milliseconds{2}
+        }
+    );
+
+    EXPECT_EQ(result, 42);
+    EXPECT_EQ(calls, 3);
+}
+
+TEST(BackendInterfaceRetryTest, RetryOnTimeoutBlockingDoesNotSwallowOtherExceptions)
+{
+    EXPECT_THROW(
+        retryOnTimeout(
+            []() -> int { throw std::runtime_error{"permanent"}; },
+            util::Retry::Delays{
+                .initial = std::chrono::milliseconds{1}, .max = std::chrono::milliseconds{1}
+            }
+        ),
+        std::runtime_error
+    );
+}
+
+struct BackendInterfaceRetryCoroTest : SyncAsioContextTest {};
+
+TEST_F(BackendInterfaceRetryCoroTest, RetryOnTimeoutCoroRetriesUntilSuccess)
+{
+    std::size_t calls = 0;
+
+    runSpawn([&calls](auto yield) {
+        auto const result = retryOnTimeout(
+            [&calls]() -> int {
+                if (++calls < 3)
+                    throw DatabaseError{};
+                return 42;
+            },
+            yield,
+            util::Retry::Delays{
+                .initial = std::chrono::milliseconds{1}, .max = std::chrono::milliseconds{2}
+            }
+        );
+
+        EXPECT_EQ(result, 42);
+    });
+
+    EXPECT_EQ(calls, 3);
+}
+
+TEST_F(BackendInterfaceRetryCoroTest, RetryOnTimeoutCoroDoesNotSwallowOtherExceptions)
+{
+    runSpawn([](auto yield) {
+        EXPECT_THROW(
+            retryOnTimeout(
+                []() -> int { throw std::runtime_error{"permanent"}; },
+                yield,
+                util::Retry::Delays{
+                    .initial = std::chrono::milliseconds{1}, .max = std::chrono::milliseconds{1}
+                }
+            ),
+            std::runtime_error
+        );
+    });
+}
+
+TEST_F(BackendInterfaceRetryCoroTest, RetryOnTimeoutCoroDoesNotBlockItsThread)
+{
+    bool ran = false;
+
+    runSpawn([&ran, this](auto yield) {
+        boost::asio::post(ctx_, [&ran]() { ran = true; });
+
+        std::size_t calls = 0;
+        retryOnTimeout(
+            [&calls]() -> int {
+                if (++calls < 2)
+                    throw DatabaseError{};
+                return 0;
+            },
+            yield,
+            util::Retry::Delays{
+                .initial = std::chrono::milliseconds{20}, .max = std::chrono::milliseconds{20}
+            }
+        );
+
+        EXPECT_TRUE(ran);
+    });
+}