fix: Introduce retry for transient errors from DB (#3167)

This commit is contained in:
Alex Kremer
2026-08-10 14:39:15 +01:00
committed by GitHub
parent f490117cfb
commit dbdbe4ea1f
27 changed files with 665 additions and 88 deletions

View File

@@ -19,6 +19,7 @@
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
@@ -154,7 +155,45 @@ TEST_F(BackendCassandraExecutionStrategyTest, ReadOneInCoroutineThrowsOnTimeoutF
runSpawn([&strat](boost::asio::yield_context yield) {
auto statement = FakeStatement{};
EXPECT_THROW(strat.read(yield, statement), data::DatabaseTimeout);
EXPECT_THROW(strat.read(yield, statement), data::DatabaseError);
});
}
// A CL=QUORUM read failure is CASS_ERROR_SERVER_READ_FAILURE, which used to not throw at all,
// leaving read() spinning; Times(1) pins that down.
TEST_F(BackendCassandraExecutionStrategyTest, ReadOneInCoroutineThrowsOnQuorumReadFailure)
{
auto strat = makeStrategy();
ON_CALL(
handle_,
asyncExecute(A<FakeStatement const&>(), A<std::function<void(FakeResultOrError)>&&>())
)
.WillByDefault([](auto const&, auto&& cb) {
auto res = FakeResultOrError{CassandraError{
"received 1 responses and 1 failures", CASS_ERROR_SERVER_READ_FAILURE
}};
cb(res); // notify that item is ready
return FakeFutureWithCallback{res};
});
EXPECT_CALL(
handle_,
asyncExecute(A<FakeStatement const&>(), A<std::function<void(FakeResultOrError)>&&>())
)
.Times(1);
EXPECT_CALL(*counters_, registerReadStartedImpl(1));
EXPECT_CALL(*counters_, registerReadErrorImpl(1));
runSpawn([&strat](boost::asio::yield_context yield) {
auto statement = FakeStatement{};
try {
strat.read(yield, statement);
FAIL() << "expected DatabaseError";
} catch (data::DatabaseError const& e) {
EXPECT_THAT(
std::string{e.what()}, testing::HasSubstr("received 1 responses and 1 failures")
);
}
});
}
@@ -246,7 +285,7 @@ TEST_F(BackendCassandraExecutionStrategyTest, ReadBatchInCoroutineThrowsOnTimeou
runSpawn([&strat](boost::asio::yield_context yield) {
auto statements = std::vector<FakeStatement>(kNumStatements);
EXPECT_THROW(strat.read(yield, statements), data::DatabaseTimeout);
EXPECT_THROW(strat.read(yield, statements), data::DatabaseError);
});
}
@@ -384,7 +423,7 @@ TEST_F(BackendCassandraExecutionStrategyTest, ReadEachInCoroutineThrowsOnFailure
runSpawn([&strat](boost::asio::yield_context yield) {
auto statements = std::vector<FakeStatement>(kNumStatements);
EXPECT_THROW(strat.readEach(yield, statements), data::DatabaseTimeout);
EXPECT_THROW(strat.readEach(yield, statements), data::DatabaseError);
});
}