diff --git a/include/xrpl/server/detail/Door.h b/include/xrpl/server/detail/Door.h index 7906af2a52..65bc310d4c 100644 --- a/include/xrpl/server/detail/Door.h +++ b/include/xrpl/server/detail/Door.h @@ -30,15 +30,29 @@ #include #include #include +#include #include +#include #include #include #include #include +#include +#if !BOOST_OS_WINDOWS +#include + +#include +#include +#endif + +#include #include +#include #include #include +#include +#include namespace ripple { @@ -98,10 +112,27 @@ private: boost::asio::strand strand_; bool ssl_; bool plain_; + static constexpr std::chrono::milliseconds INITIAL_ACCEPT_DELAY{50}; + static constexpr std::chrono::milliseconds MAX_ACCEPT_DELAY{2000}; + std::chrono::milliseconds accept_delay_{INITIAL_ACCEPT_DELAY}; + boost::asio::steady_timer backoff_timer_; + static constexpr double FREE_FD_THRESHOLD = 0.70; + + struct FDStats + { + std::uint64_t used{0}; + std::uint64_t limit{0}; + }; void reOpen(); + std::optional + query_fd_stats() const; + + bool + should_throttle_for_fds(); + public: Door( Handler& handler, @@ -299,6 +330,7 @@ Door::Door( , plain_( port_.protocol.count("http") > 0 || port_.protocol.count("ws") > 0 || port_.protocol.count("ws2")) + , backoff_timer_(io_context) { reOpen(); } @@ -323,6 +355,7 @@ Door::close() return boost::asio::post( strand_, std::bind(&Door::close, this->shared_from_this())); + backoff_timer_.cancel(); error_code ec; acceptor_.close(ec); } @@ -368,6 +401,17 @@ Door::do_accept(boost::asio::yield_context do_yield) { while (acceptor_.is_open()) { + if (should_throttle_for_fds()) + { + backoff_timer_.expires_after(accept_delay_); + boost::system::error_code tec; + backoff_timer_.async_wait(do_yield[tec]); + accept_delay_ = std::min(accept_delay_ * 2, MAX_ACCEPT_DELAY); + JLOG(j_.warn()) << "Throttling do_accept for " + << accept_delay_.count() << "ms."; + continue; + } + error_code ec; endpoint_type remote_address; stream_type stream(ioc_); @@ -377,15 +421,28 @@ Door::do_accept(boost::asio::yield_context do_yield) { if (ec == boost::asio::error::operation_aborted) break; - JLOG(j_.error()) << "accept: " << ec.message(); - if (ec == boost::asio::error::no_descriptors) + + if (ec == boost::asio::error::no_descriptors || + ec == boost::asio::error::no_buffer_space) { - JLOG(j_.info()) << "re-opening acceptor"; - reOpen(); + JLOG(j_.warn()) << "accept: Too many open files. Pausing for " + << accept_delay_.count() << "ms."; + + backoff_timer_.expires_after(accept_delay_); + boost::system::error_code tec; + backoff_timer_.async_wait(do_yield[tec]); + + accept_delay_ = std::min(accept_delay_ * 2, MAX_ACCEPT_DELAY); + } + else + { + JLOG(j_.error()) << "accept error: " << ec.message(); } continue; } + accept_delay_ = INITIAL_ACCEPT_DELAY; + if (ssl_ && plain_) { if (auto sp = ios().template emplace( @@ -408,6 +465,60 @@ Door::do_accept(boost::asio::yield_context do_yield) } } +template +std::optional::FDStats> +Door::query_fd_stats() const +{ +#if BOOST_OS_WINDOWS + return std::nullopt; +#else + FDStats s; + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) != 0 || rl.rlim_cur == RLIM_INFINITY) + return std::nullopt; + s.limit = static_cast(rl.rlim_cur); +#if BOOST_OS_LINUX + constexpr char const* kFdDir = "/proc/self/fd"; +#else + constexpr char const* kFdDir = "/dev/fd"; +#endif + if (DIR* d = ::opendir(kFdDir)) + { + std::uint64_t cnt = 0; + while (::readdir(d) != nullptr) + ++cnt; + ::closedir(d); + // readdir counts '.', '..', and the DIR* itself shows in the list + s.used = (cnt >= 3) ? (cnt - 3) : 0; + return s; + } + return std::nullopt; +#endif +} + +template +bool +Door::should_throttle_for_fds() +{ +#if BOOST_OS_WINDOWS + return false; +#else + auto const stats = query_fd_stats(); + if (!stats || stats->limit == 0) + return false; + + auto const& s = *stats; + auto const free = (s.limit > s.used) ? (s.limit - s.used) : 0ull; + double const free_ratio = + static_cast(free) / static_cast(s.limit); + if (free_ratio < FREE_FD_THRESHOLD) + { + return true; + } + return false; +#endif +} + } // namespace ripple #endif