mirror of
https://github.com/XRPLF/rippled.git
synced 2026-04-29 15:37:57 +00:00
refactore
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/contract.h>
|
||||
#include <xrpl/server/FDGuard.h>
|
||||
#include <xrpl/server/detail/ExponentialBackoff.h>
|
||||
#include <xrpl/server/detail/PlainHTTPPeer.h>
|
||||
#include <xrpl/server/detail/SSLHTTPPeer.h>
|
||||
#include <xrpl/server/detail/io_list.h>
|
||||
@@ -83,9 +84,7 @@ private:
|
||||
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
|
||||
bool ssl_;
|
||||
bool plain_;
|
||||
static constexpr std::chrono::milliseconds INITIAL_ACCEPT_DELAY{50};
|
||||
static constexpr std::chrono::milliseconds MAX_ACCEPT_DELAY{2000};
|
||||
std::chrono::milliseconds accept_delay_{INITIAL_ACCEPT_DELAY};
|
||||
ExponentialBackoff backoff_;
|
||||
boost::asio::steady_timer backoff_timer_;
|
||||
|
||||
void
|
||||
@@ -317,11 +316,11 @@ Door<Handler>::do_accept(boost::asio::yield_context do_yield)
|
||||
{
|
||||
if (FDGuard::should_throttle(0.70))
|
||||
{
|
||||
backoff_timer_.expires_after(accept_delay_);
|
||||
backoff_timer_.expires_after(backoff_.current());
|
||||
boost::system::error_code tec;
|
||||
backoff_timer_.async_wait(do_yield[tec]);
|
||||
accept_delay_ = std::min(accept_delay_ * 2, MAX_ACCEPT_DELAY);
|
||||
JLOG(j_.warn()) << "Throttling do_accept for " << accept_delay_.count() << "ms.";
|
||||
auto const delay = backoff_.increase();
|
||||
JLOG(j_.warn()) << "Throttling do_accept for " << delay.count() << "ms.";
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -338,14 +337,15 @@ Door<Handler>::do_accept(boost::asio::yield_context do_yield)
|
||||
if (ec == boost::asio::error::no_descriptors ||
|
||||
ec == boost::asio::error::no_buffer_space)
|
||||
{
|
||||
auto const delay = backoff_.current();
|
||||
JLOG(j_.warn()) << "accept: Too many open files. Pausing for "
|
||||
<< accept_delay_.count() << "ms.";
|
||||
<< delay.count() << "ms.";
|
||||
|
||||
backoff_timer_.expires_after(accept_delay_);
|
||||
backoff_timer_.expires_after(delay);
|
||||
boost::system::error_code tec;
|
||||
backoff_timer_.async_wait(do_yield[tec]);
|
||||
|
||||
accept_delay_ = std::min(accept_delay_ * 2, MAX_ACCEPT_DELAY);
|
||||
backoff_.increase();
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -354,7 +354,7 @@ Door<Handler>::do_accept(boost::asio::yield_context do_yield)
|
||||
continue;
|
||||
}
|
||||
|
||||
accept_delay_ = INITIAL_ACCEPT_DELAY;
|
||||
backoff_.reset();
|
||||
|
||||
if (ssl_ && plain_)
|
||||
{
|
||||
|
||||
93
include/xrpl/server/detail/ExponentialBackoff.h
Normal file
93
include/xrpl/server/detail/ExponentialBackoff.h
Normal file
@@ -0,0 +1,93 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <algorithm>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* @brief Exponential backoff delay manager with configurable limits.
|
||||
*
|
||||
* Manages delay values that double on each increase (capped at maximum)
|
||||
* and can be reset to initial value. Used for throttling accept() calls
|
||||
* when file descriptor pressure is detected.
|
||||
*/
|
||||
class ExponentialBackoff
|
||||
{
|
||||
public:
|
||||
using duration_type = std::chrono::milliseconds;
|
||||
|
||||
static constexpr duration_type DEFAULT_INITIAL_DELAY{50};
|
||||
static constexpr duration_type DEFAULT_MAX_DELAY{2000};
|
||||
|
||||
/**
|
||||
* @brief Construct with custom or default delay parameters.
|
||||
*
|
||||
* @param initial Initial delay value (default: 50ms)
|
||||
* @param maximum Maximum delay cap (default: 2000ms)
|
||||
*/
|
||||
explicit ExponentialBackoff(
|
||||
duration_type initial = DEFAULT_INITIAL_DELAY,
|
||||
duration_type maximum = DEFAULT_MAX_DELAY)
|
||||
: initial_(initial), maximum_(maximum), current_(initial)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get current delay value.
|
||||
*/
|
||||
[[nodiscard]] duration_type
|
||||
current() const noexcept
|
||||
{
|
||||
return current_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Double the current delay, capped at maximum.
|
||||
*
|
||||
* @return The new current delay value after increase.
|
||||
*/
|
||||
duration_type
|
||||
increase() noexcept
|
||||
{
|
||||
current_ = std::min(current_ * 2, maximum_);
|
||||
return current_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reset delay to initial value.
|
||||
*
|
||||
* @return The initial delay value.
|
||||
*/
|
||||
duration_type
|
||||
reset() noexcept
|
||||
{
|
||||
current_ = initial_;
|
||||
return current_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get initial delay value.
|
||||
*/
|
||||
[[nodiscard]] duration_type
|
||||
initial() const noexcept
|
||||
{
|
||||
return initial_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get maximum delay value.
|
||||
*/
|
||||
[[nodiscard]] duration_type
|
||||
maximum() const noexcept
|
||||
{
|
||||
return maximum_;
|
||||
}
|
||||
|
||||
private:
|
||||
duration_type const initial_;
|
||||
duration_type const maximum_;
|
||||
duration_type current_;
|
||||
};
|
||||
|
||||
} // namespace xrpl
|
||||
162
src/test/server/ExponentialBackoff_test.cpp
Normal file
162
src/test/server/ExponentialBackoff_test.cpp
Normal file
@@ -0,0 +1,162 @@
|
||||
#include <xrpl/beast/unit_test.h>
|
||||
#include <xrpl/server/detail/ExponentialBackoff.h>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
class ExponentialBackoff_test : public beast::unit_test::suite
|
||||
{
|
||||
public:
|
||||
void
|
||||
testDefaultConstruction()
|
||||
{
|
||||
testcase("default construction");
|
||||
|
||||
ExponentialBackoff backoff;
|
||||
|
||||
BEAST_EXPECT(backoff.initial() == ExponentialBackoff::DEFAULT_INITIAL_DELAY);
|
||||
BEAST_EXPECT(backoff.maximum() == ExponentialBackoff::DEFAULT_MAX_DELAY);
|
||||
BEAST_EXPECT(backoff.current() == ExponentialBackoff::DEFAULT_INITIAL_DELAY);
|
||||
}
|
||||
|
||||
void
|
||||
testCustomConstruction()
|
||||
{
|
||||
testcase("custom construction");
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
ExponentialBackoff backoff{100ms, 5000ms};
|
||||
|
||||
BEAST_EXPECT(backoff.initial() == 100ms);
|
||||
BEAST_EXPECT(backoff.maximum() == 5000ms);
|
||||
BEAST_EXPECT(backoff.current() == 100ms);
|
||||
}
|
||||
|
||||
void
|
||||
testIncreaseDoublesDelay()
|
||||
{
|
||||
testcase("increase doubles delay");
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
ExponentialBackoff backoff{50ms, 2000ms};
|
||||
|
||||
BEAST_EXPECT(backoff.current() == 50ms);
|
||||
|
||||
auto delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 100ms);
|
||||
BEAST_EXPECT(backoff.current() == 100ms);
|
||||
|
||||
delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 200ms);
|
||||
BEAST_EXPECT(backoff.current() == 200ms);
|
||||
|
||||
delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 400ms);
|
||||
BEAST_EXPECT(backoff.current() == 400ms);
|
||||
|
||||
delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 800ms);
|
||||
BEAST_EXPECT(backoff.current() == 800ms);
|
||||
|
||||
delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 1600ms);
|
||||
BEAST_EXPECT(backoff.current() == 1600ms);
|
||||
}
|
||||
|
||||
void
|
||||
testIncreaseCapsAtMaximum()
|
||||
{
|
||||
testcase("increase caps at maximum");
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
ExponentialBackoff backoff{50ms, 2000ms};
|
||||
|
||||
// Increase until we hit the cap
|
||||
for (int i = 0; i < 10; ++i)
|
||||
{
|
||||
backoff.increase();
|
||||
}
|
||||
|
||||
// Should be capped at maximum
|
||||
BEAST_EXPECT(backoff.current() == 2000ms);
|
||||
|
||||
// Further increases should not exceed maximum
|
||||
auto delay = backoff.increase();
|
||||
BEAST_EXPECT(delay == 2000ms);
|
||||
BEAST_EXPECT(backoff.current() == 2000ms);
|
||||
}
|
||||
|
||||
void
|
||||
testResetReturnsToInitial()
|
||||
{
|
||||
testcase("reset returns to initial");
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
ExponentialBackoff backoff{50ms, 2000ms};
|
||||
|
||||
// Increase several times
|
||||
backoff.increase();
|
||||
backoff.increase();
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 400ms);
|
||||
|
||||
// Reset should return to initial
|
||||
auto delay = backoff.reset();
|
||||
BEAST_EXPECT(delay == 50ms);
|
||||
BEAST_EXPECT(backoff.current() == 50ms);
|
||||
}
|
||||
|
||||
void
|
||||
testTypicalDoorUsage()
|
||||
{
|
||||
testcase("typical door usage pattern");
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
// Simulates Door's usage pattern
|
||||
ExponentialBackoff backoff{50ms, 2000ms};
|
||||
|
||||
// First throttle
|
||||
BEAST_EXPECT(backoff.current() == 50ms);
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 100ms);
|
||||
|
||||
// Second throttle
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 200ms);
|
||||
|
||||
// Success - reset
|
||||
backoff.reset();
|
||||
BEAST_EXPECT(backoff.current() == 50ms);
|
||||
|
||||
// Another throttle sequence
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 100ms);
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 200ms);
|
||||
backoff.increase();
|
||||
BEAST_EXPECT(backoff.current() == 400ms);
|
||||
|
||||
// Success - reset
|
||||
backoff.reset();
|
||||
BEAST_EXPECT(backoff.current() == 50ms);
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
testDefaultConstruction();
|
||||
testCustomConstruction();
|
||||
testIncreaseDoublesDelay();
|
||||
testIncreaseCapsAtMaximum();
|
||||
testResetReturnsToInitial();
|
||||
testTypicalDoorUsage();
|
||||
}
|
||||
};
|
||||
|
||||
BEAST_DEFINE_TESTSUITE(ExponentialBackoff, server, xrpl);
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -457,14 +457,14 @@ GRPCServerImpl::handleRpcs()
|
||||
{
|
||||
backoffScheduled_ = true;
|
||||
|
||||
auto deadline = std::chrono::system_clock::now() + acceptDelay_;
|
||||
auto deadline = std::chrono::system_clock::now() + backoff_.current();
|
||||
|
||||
backoffAlarm_.Set(cq_.get(), deadline, static_cast<void*>(&backoffTag_));
|
||||
|
||||
acceptDelay_ = std::min(acceptDelay_ * 2, MAX_ACCEPT_DELAY);
|
||||
auto const delay = backoff_.increase();
|
||||
|
||||
JLOG(journal_.warn())
|
||||
<< "Scheduled backoff alarm for " << acceptDelay_.count() << "ms";
|
||||
<< "Scheduled backoff alarm for " << delay.count() << "ms";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -474,7 +474,7 @@ GRPCServerImpl::handleRpcs()
|
||||
else
|
||||
{
|
||||
// Not throttled - reset delay and clone immediately
|
||||
acceptDelay_ = INITIAL_ACCEPT_DELAY;
|
||||
backoff_.reset();
|
||||
|
||||
// ptr is now processing a request, so create a new CallData
|
||||
// object to handle additional requests
|
||||
@@ -520,13 +520,13 @@ GRPCServerImpl::onBackoffFired()
|
||||
{
|
||||
backoffScheduled_ = true;
|
||||
|
||||
auto deadline = std::chrono::system_clock::now() + acceptDelay_;
|
||||
auto deadline = std::chrono::system_clock::now() + backoff_.current();
|
||||
|
||||
backoffAlarm_.Set(cq_.get(), deadline, static_cast<void*>(&backoffTag_));
|
||||
|
||||
acceptDelay_ = std::min(acceptDelay_ * 2, MAX_ACCEPT_DELAY);
|
||||
auto const delay = backoff_.increase();
|
||||
|
||||
JLOG(journal_.warn()) << "Rescheduled backoff alarm for " << acceptDelay_.count() << "ms";
|
||||
JLOG(journal_.warn()) << "Rescheduled backoff alarm for " << delay.count() << "ms";
|
||||
}
|
||||
|
||||
return;
|
||||
@@ -534,7 +534,7 @@ GRPCServerImpl::onBackoffFired()
|
||||
|
||||
// Recovery - FD pressure relieved
|
||||
JLOG(journal_.info()) << "FD pressure relieved - resuming normal operation";
|
||||
acceptDelay_ = INITIAL_ACCEPT_DELAY;
|
||||
backoff_.reset();
|
||||
|
||||
for (auto const& ptr : toRepost)
|
||||
{
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <xrpl/resource/Charge.h>
|
||||
#include <xrpl/server/FDGuard.h>
|
||||
#include <xrpl/server/InfoSub.h>
|
||||
#include <xrpl/server/detail/ExponentialBackoff.h>
|
||||
|
||||
#include <grpcpp/alarm.h>
|
||||
#include <grpcpp/grpcpp.h>
|
||||
@@ -95,9 +96,7 @@ private:
|
||||
// FD throttling and backoff state
|
||||
std::mutex backoffMutex_;
|
||||
bool backoffScheduled_{false};
|
||||
std::chrono::milliseconds acceptDelay_{std::chrono::milliseconds{50}};
|
||||
static constexpr std::chrono::milliseconds INITIAL_ACCEPT_DELAY{50};
|
||||
static constexpr std::chrono::milliseconds MAX_ACCEPT_DELAY{2000};
|
||||
ExponentialBackoff backoff_;
|
||||
BackoffTag backoffTag_;
|
||||
grpc::Alarm backoffAlarm_;
|
||||
std::vector<std::shared_ptr<Processor>> pendingReposts_;
|
||||
|
||||
Reference in New Issue
Block a user