Compare commits

...

6 Commits

Author SHA1 Message Date
cyan317
8b0e68f48e chore: Add counter for total messages waiting to be sent (#1691) 2024-10-18 16:11:38 +01:00
Alex Kremer
189098d092 fix: Static linkage (#1551)
Fixes #1507
2024-10-18 16:11:38 +01:00
Alex Kremer
854749a05e fix: Add upper bound to limit 2024-09-11 15:15:17 +01:00
cyan317
f57706be3d fix: no restriction on type field (#1644)
'type' should not matter if 'full' or 'accounts' is false. Relax the
restriction for 'type'
2024-09-11 14:44:20 +01:00
cyan317
bb0d912f2b fix: Add more restrictions to admin fields (#1643) 2024-09-10 15:05:23 +01:00
cyan317
d02d6affdb fix: Don't forward ledger API if 'full' is a string (#1640)
Fix #1635
2024-09-09 11:21:42 +01:00
10 changed files with 203 additions and 45 deletions

View File

@@ -13,12 +13,15 @@ jobs:
include:
- os: macos14
build_type: Release
static: false
- os: heavy
build_type: Release
static: true
container:
image: rippleci/clio_ci:latest
- os: heavy
build_type: Debug
static: true
container:
image: rippleci/clio_ci:latest
runs-on: [self-hosted, "${{ matrix.os }}"]
@@ -50,6 +53,7 @@ jobs:
conan_profile: ${{ steps.conan.outputs.conan_profile }}
conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
build_type: ${{ matrix.build_type }}
static: ${{ matrix.static }}
- name: Build Clio
uses: ./.github/actions/build_clio

View File

@@ -11,18 +11,16 @@ target_sources(clio_server PRIVATE Main.cpp)
target_link_libraries(clio_server PRIVATE clio)
if (static)
target_link_options(clio_server PRIVATE -static)
if (is_gcc AND NOT san)
if (san)
message(FATAL_ERROR "Static linkage not allowed when using sanitizers")
elseif (is_appleclang)
message(FATAL_ERROR "Static linkage not supported on AppleClang")
else ()
target_link_options(
# For now let's assume that we only using libstdc++ under gcc.
# Note: -static-libstdc++ can statically link both libstdc++ and libc++
clio_server PRIVATE -static-libstdc++ -static-libgcc
)
endif ()
if (is_appleclang)
message(FATAL_ERROR "Static linkage not supported on AppleClang")
endif ()
endif ()
set_target_properties(clio_server PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

View File

@@ -36,6 +36,7 @@
#include <boost/json/array.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/serialize.hpp>
#include <boost/json/string.hpp>
#include <boost/json/value.hpp>
#include <boost/json/value_to.hpp>
@@ -49,6 +50,7 @@
#include <ripple/basics/chrono.h>
#include <ripple/basics/strHex.h>
#include <ripple/beast/utility/Zero.h>
#include <ripple/json/json_reader.h>
#include <ripple/json/json_value.h>
#include <ripple/protocol/AccountID.h>
#include <ripple/protocol/Book.h>
@@ -1276,12 +1278,20 @@ specifiesCurrentOrClosedLedger(boost::json::object const& request)
bool
isAdminCmd(std::string const& method, boost::json::object const& request)
{
auto const isFieldSet = [&request](auto const field) {
return request.contains(field) and request.at(field).is_bool() and request.at(field).as_bool();
};
if (method == JS(ledger)) {
if (isFieldSet(JS(full)) or isFieldSet(JS(accounts)) or isFieldSet(JS(type)))
auto const requestStr = boost::json::serialize(request);
Json::Value jv;
Json::Reader{}.parse(requestStr, jv);
// rippled considers string/non-zero int/non-empty array/ non-empty json as true.
// Use rippled's API asBool to get the same result.
// https://github.com/XRPLF/rippled/issues/5119
auto const isFieldSet = [&jv](auto const field) { return jv.isMember(field) and jv[field].asBool(); };
// According to doc
// https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/ledger-methods/ledger,
// full/accounts/type are admin only, but type only works when full/accounts are set, so we don't need to check
// type.
if (isFieldSet(JS(full)) or isFieldSet(JS(accounts)))
return true;
}

View File

@@ -39,7 +39,6 @@
#include <ripple/protocol/jss.h>
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <string>
@@ -57,8 +56,8 @@ class AccountTxHandler {
std::shared_ptr<BackendInterface> sharedPtrBackend_;
public:
// no max limit
static auto constexpr LIMIT_MIN = 1;
static auto constexpr LIMIT_MAX = 1000;
static auto constexpr LIMIT_DEFAULT = 200;
/**
@@ -133,7 +132,7 @@ public:
{JS(limit),
validation::Type<uint32_t>{},
validation::Min(1u),
modifiers::Clamp<int32_t>{LIMIT_MIN, std::numeric_limits<int32_t>::max()}},
modifiers::Clamp<int32_t>{LIMIT_MIN, LIMIT_MAX}},
{JS(marker),
meta::WithCustomError{
validation::Type<boost::json::object>{},

View File

@@ -134,7 +134,7 @@ private:
// The timer below can be called with no error code even if the operation is completed before the timeout, so we
// need an additional flag here
timer.async_wait([&cancellationSignal, isCompleted](boost::system::error_code errorCode) {
if (!errorCode and not *isCompleted)
if (!errorCode and not*isCompleted)
cancellationSignal.emit(boost::asio::cancellation_type::terminal);
});
operation(cyield);

View File

@@ -69,10 +69,8 @@ namespace web {
* @tparam HandlerType The executor to handle the requests
*/
template <
template <typename>
class PlainSessionType,
template <typename>
class SslSessionType,
template <typename> class PlainSessionType,
template <typename> class SslSessionType,
SomeServerHandler HandlerType>
class Detector : public std::enable_shared_from_this<Detector<PlainSessionType, SslSessionType, HandlerType>> {
using std::enable_shared_from_this<Detector<PlainSessionType, SslSessionType, HandlerType>>::shared_from_this;
@@ -191,10 +189,8 @@ public:
* @tparam HandlerType The handler to process the request and return response.
*/
template <
template <typename>
class PlainSessionType,
template <typename>
class SslSessionType,
template <typename> class PlainSessionType,
template <typename> class SslSessionType,
SomeServerHandler HandlerType>
class Server : public std::enable_shared_from_this<Server<PlainSessionType, SslSessionType, HandlerType>> {
using std::enable_shared_from_this<Server<PlainSessionType, SslSessionType, HandlerType>>::shared_from_this;

View File

@@ -23,6 +23,9 @@
#include "rpc/common/Types.hpp"
#include "util/Taggable.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Gauge.hpp"
#include "util/prometheus/Label.hpp"
#include "util/prometheus/Prometheus.hpp"
#include "web/DOSGuard.hpp"
#include "web/interface/Concepts.hpp"
#include "web/interface/ConnectionBase.hpp"
@@ -71,6 +74,8 @@ template <template <typename> typename Derived, SomeServerHandler HandlerType>
class WsBase : public ConnectionBase, public std::enable_shared_from_this<WsBase<Derived, HandlerType>> {
using std::enable_shared_from_this<WsBase<Derived, HandlerType>>::shared_from_this;
std::reference_wrapper<util::prometheus::GaugeInt> messagesLength_;
boost::beast::flat_buffer buffer_;
std::reference_wrapper<web::DOSGuard> dosGuard_;
bool sending_ = false;
@@ -103,15 +108,26 @@ public:
std::shared_ptr<HandlerType> const& handler,
boost::beast::flat_buffer&& buffer
)
: ConnectionBase(tagFactory, ip), buffer_(std::move(buffer)), dosGuard_(dosGuard), handler_(handler)
: ConnectionBase(tagFactory, ip)
, messagesLength_(PrometheusService::gaugeInt(
"ws_messages_length",
util::prometheus::Labels(),
"The total length of messages in the queue"
))
, buffer_(std::move(buffer))
, dosGuard_(dosGuard)
, handler_(handler)
{
upgraded = true; // NOLINT (cppcoreguidelines-pro-type-member-init)
LOG(perfLog_.debug()) << tag() << "session created";
}
~WsBase() override
{
LOG(perfLog_.debug()) << tag() << "session closed";
if (!messages_.empty())
messagesLength_.get() -= messages_.size();
dosGuard_.get().decrement(clientIp);
}
@@ -135,6 +151,7 @@ public:
onWrite(boost::system::error_code ec, std::size_t)
{
messages_.pop();
--messagesLength_.get();
sending_ = false;
if (ec) {
wsFail(ec, "Failed to write");
@@ -165,6 +182,7 @@ public:
derived().ws().get_executor(),
[this, self = derived().shared_from_this(), msg = std::move(msg)]() {
messages_.push(msg);
++messagesLength_.get();
maybeSendNext();
}
);

View File

@@ -553,15 +553,41 @@ static auto
generateTestValuesForParametersTest()
{
return std::vector<IsAdminCmdParamTestCaseBundle>{
{"ledgerEntry", "ledger_entry", R"({"type": false})", false},
{"featureVetoedTrue", "feature", R"({"vetoed": true, "feature": "foo"})", true},
{"featureVetoedFalse", "feature", R"({"vetoed": false, "feature": "foo"})", true},
{"featureVetoedIsStr", "feature", R"({"vetoed": "String"})", true},
{"ledger", "ledger", R"({})", false},
{"ledgerWithType", "ledger", R"({"type": "fee"})", false},
{"ledgerFullTrue", "ledger", R"({"full": true})", true},
{"ledgerAccountsTrue", "ledger", R"({"accounts": true})", true},
{"ledgerTypeTrue", "ledger", R"({"type": true})", true},
{"ledgerFullFalse", "ledger", R"({"full": false})", false},
{"ledgerFullIsStr", "ledger", R"({"full": "String"})", true},
{"ledgerFullIsEmptyStr", "ledger", R"({"full": ""})", false},
{"ledgerFullIsNumber1", "ledger", R"({"full": 1})", true},
{"ledgerFullIsNumber0", "ledger", R"({"full": 0})", false},
{"ledgerFullIsNull", "ledger", R"({"full": null})", false},
{"ledgerFullIsFloat0", "ledger", R"({"full": 0.0})", false},
{"ledgerFullIsFloat1", "ledger", R"({"full": 0.1})", true},
{"ledgerFullIsArray", "ledger", R"({"full": [1]})", true},
{"ledgerFullIsEmptyArray", "ledger", R"({"full": []})", false},
{"ledgerFullIsObject", "ledger", R"({"full": {"key": 1}})", true},
{"ledgerFullIsEmptyObject", "ledger", R"({"full": {}})", false},
{"ledgerAccountsTrue", "ledger", R"({"accounts": true})", true},
{"ledgerAccountsFalse", "ledger", R"({"accounts": false})", false},
{"ledgerTypeFalse", "ledger", R"({"type": false})", false},
{"ledgerEntry", "ledger_entry", R"({"type": false})", false}
{"ledgerAccountsIsStr", "ledger", R"({"accounts": "String"})", true},
{"ledgerAccountsIsEmptyStr", "ledger", R"({"accounts": ""})", false},
{"ledgerAccountsIsNumber1", "ledger", R"({"accounts": 1})", true},
{"ledgerAccountsIsNumber0", "ledger", R"({"accounts": 0})", false},
{"ledgerAccountsIsNull", "ledger", R"({"accounts": null})", false},
{"ledgerAccountsIsFloat0", "ledger", R"({"accounts": 0.0})", false},
{"ledgerAccountsIsFloat1", "ledger", R"({"accounts": 0.1})", true},
{"ledgerAccountsIsArray", "ledger", R"({"accounts": [1]})", true},
{"ledgerAccountsIsEmptyArray", "ledger", R"({"accounts": []})", false},
{"ledgerAccountsIsObject", "ledger", R"({"accounts": {"key": 1}})", true},
{"ledgerAccountsIsEmptyObject", "ledger", R"({"accounts": {}})", false},
};
}

View File

@@ -769,14 +769,13 @@ TEST_F(RPCAccountTxHandlerTest, LimitAndMarker)
auto const transactions = genTransactions(MINSEQ + 1, MAXSEQ - 1);
auto const transCursor = TransactionsAndCursor{transactions, TransactionsCursor{12, 34}};
ON_CALL(*backend, fetchAccountTransactions).WillByDefault(Return(transCursor));
EXPECT_CALL(
*backend,
fetchAccountTransactions(
testing::_, testing::_, false, testing::Optional(testing::Eq(TransactionsCursor{10, 11})), testing::_
)
)
.Times(1);
.WillOnce(Return(transCursor));
runSpawn([&, this](auto yield) {
auto const handler = AnyHandler{AccountTxHandler{backend}};
@@ -804,6 +803,73 @@ TEST_F(RPCAccountTxHandlerTest, LimitAndMarker)
});
}
TEST_F(RPCAccountTxHandlerTest, LimitIsCapped)
{
backend->setRange(MINSEQ, MAXSEQ);
auto const transactions = genTransactions(MINSEQ + 1, MAXSEQ - 1);
auto const transCursor = TransactionsAndCursor{transactions, TransactionsCursor{12, 34}};
EXPECT_CALL(*backend, fetchAccountTransactions(testing::_, testing::_, false, testing::_, testing::_))
.WillOnce(Return(transCursor));
runSpawn([&, this](auto yield) {
auto const handler = AnyHandler{AccountTxHandler{backend}};
auto static const input = json::parse(fmt::format(
R"({{
"account": "{}",
"ledger_index_min": {},
"ledger_index_max": {},
"limit": 100000,
"forward": false
}})",
ACCOUNT,
-1,
-1
));
auto const output = handler.process(input, Context{yield});
ASSERT_TRUE(output);
EXPECT_EQ(output.result->at("account").as_string(), ACCOUNT);
EXPECT_EQ(output.result->at("ledger_index_min").as_uint64(), MINSEQ);
EXPECT_EQ(output.result->at("ledger_index_max").as_uint64(), MAXSEQ);
EXPECT_EQ(output.result->at("limit").as_uint64(), AccountTxHandler::LIMIT_MAX);
EXPECT_EQ(output.result->at("transactions").as_array().size(), 2);
});
}
TEST_F(RPCAccountTxHandlerTest, LimitAllowedUpToCap)
{
backend->setRange(MINSEQ, MAXSEQ);
auto const transactions = genTransactions(MINSEQ + 1, MAXSEQ - 1);
auto const transCursor = TransactionsAndCursor{transactions, TransactionsCursor{12, 34}};
EXPECT_CALL(*backend, fetchAccountTransactions(testing::_, testing::_, false, testing::_, testing::_))
.WillOnce(Return(transCursor));
runSpawn([&, this](auto yield) {
auto const handler = AnyHandler{AccountTxHandler{backend}};
auto static const input = json::parse(fmt::format(
R"({{
"account": "{}",
"ledger_index_min": {},
"ledger_index_max": {},
"limit": {},
"forward": false
}})",
ACCOUNT,
-1,
-1,
AccountTxHandler::LIMIT_MAX - 1
));
auto const output = handler.process(input, Context{yield});
ASSERT_TRUE(output);
EXPECT_EQ(output.result->at("account").as_string(), ACCOUNT);
EXPECT_EQ(output.result->at("ledger_index_min").as_uint64(), MINSEQ);
EXPECT_EQ(output.result->at("ledger_index_max").as_uint64(), MAXSEQ);
EXPECT_EQ(output.result->at("limit").as_uint64(), AccountTxHandler::LIMIT_MAX - 1);
EXPECT_EQ(output.result->at("transactions").as_array().size(), 2);
});
}
TEST_F(RPCAccountTxHandlerTest, SpecificLedgerIndex)
{
backend->setRange(MINSEQ, MAXSEQ);

View File

@@ -22,6 +22,7 @@
#include "util/MockPrometheus.hpp"
#include "util/TestHttpSyncClient.hpp"
#include "util/config/Config.hpp"
#include "util/prometheus/Gauge.hpp"
#include "util/prometheus/Label.hpp"
#include "util/prometheus/Prometheus.hpp"
#include "web/DOSGuard.hpp"
@@ -42,6 +43,7 @@
#include <boost/json/parse.hpp>
#include <boost/system/system_error.hpp>
#include <fmt/core.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <chrono>
@@ -202,6 +204,8 @@ private:
std::optional<std::thread> runner;
};
struct WebServerTestsWithMockPrometheus : WebServerTest, prometheus::WithMockPrometheus {};
class EchoExecutor {
public:
void
@@ -263,7 +267,7 @@ makeServerSync(
} // namespace
TEST_F(WebServerTest, Http)
TEST_F(WebServerTestsWithMockPrometheus, Http)
{
auto e = std::make_shared<EchoExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuard, e);
@@ -271,8 +275,13 @@ TEST_F(WebServerTest, Http)
EXPECT_EQ(res, R"({"Hello":1})");
}
TEST_F(WebServerTest, Ws)
TEST_F(WebServerTestsWithMockPrometheus, Ws)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
auto e = std::make_shared<EchoExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuard, e);
WebSocketSyncClient wsClient;
@@ -282,7 +291,7 @@ TEST_F(WebServerTest, Ws)
wsClient.disconnect();
}
TEST_F(WebServerTest, HttpInternalError)
TEST_F(WebServerTestsWithMockPrometheus, HttpInternalError)
{
auto e = std::make_shared<ExceptionExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuard, e);
@@ -293,8 +302,13 @@ TEST_F(WebServerTest, HttpInternalError)
);
}
TEST_F(WebServerTest, WsInternalError)
TEST_F(WebServerTestsWithMockPrometheus, WsInternalError)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
auto e = std::make_shared<ExceptionExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuard, e);
WebSocketSyncClient wsClient;
@@ -307,8 +321,13 @@ TEST_F(WebServerTest, WsInternalError)
);
}
TEST_F(WebServerTest, WsInternalErrorNotJson)
TEST_F(WebServerTestsWithMockPrometheus, WsInternalErrorNotJson)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
auto e = std::make_shared<ExceptionExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuard, e);
WebSocketSyncClient wsClient;
@@ -321,7 +340,7 @@ TEST_F(WebServerTest, WsInternalErrorNotJson)
);
}
TEST_F(WebServerTest, Https)
TEST_F(WebServerTestsWithMockPrometheus, Https)
{
auto e = std::make_shared<EchoExecutor>();
auto sslCtx = parseCertsForTest();
@@ -331,8 +350,13 @@ TEST_F(WebServerTest, Https)
EXPECT_EQ(res, R"({"Hello":1})");
}
TEST_F(WebServerTest, Wss)
TEST_F(WebServerTestsWithMockPrometheus, Wss)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
auto e = std::make_shared<EchoExecutor>();
auto sslCtx = parseCertsForTest();
auto const ctxSslRef = sslCtx ? std::optional<std::reference_wrapper<ssl::context>>{sslCtx.value()} : std::nullopt;
@@ -345,7 +369,7 @@ TEST_F(WebServerTest, Wss)
wsClient.disconnect();
}
TEST_F(WebServerTest, HttpRequestOverload)
TEST_F(WebServerTestsWithMockPrometheus, HttpRequestOverload)
{
auto e = std::make_shared<EchoExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuardOverload, e);
@@ -358,8 +382,13 @@ TEST_F(WebServerTest, HttpRequestOverload)
);
}
TEST_F(WebServerTest, WsRequestOverload)
TEST_F(WebServerTestsWithMockPrometheus, WsRequestOverload)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1)).Times(2);
EXPECT_CALL(wsMessagesCounterMock, add(-1)).Times(2);
auto e = std::make_shared<EchoExecutor>();
auto const server = makeServerSync(cfg, ctx, std::nullopt, dosGuardOverload, e);
WebSocketSyncClient wsClient;
@@ -377,7 +406,7 @@ TEST_F(WebServerTest, WsRequestOverload)
);
}
TEST_F(WebServerTest, HttpPayloadOverload)
TEST_F(WebServerTestsWithMockPrometheus, HttpPayloadOverload)
{
std::string const s100(100, 'a');
auto e = std::make_shared<EchoExecutor>();
@@ -389,8 +418,13 @@ TEST_F(WebServerTest, HttpPayloadOverload)
);
}
TEST_F(WebServerTest, WsPayloadOverload)
TEST_F(WebServerTestsWithMockPrometheus, WsPayloadOverload)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
std::string const s100(100, 'a');
auto e = std::make_shared<EchoExecutor>();
auto server = makeServerSync(cfg, ctx, std::nullopt, dosGuardOverload, e);
@@ -404,7 +438,7 @@ TEST_F(WebServerTest, WsPayloadOverload)
);
}
TEST_F(WebServerTest, WsTooManyConnection)
TEST_F(WebServerTestsWithMockPrometheus, WsTooManyConnection)
{
auto e = std::make_shared<EchoExecutor>();
auto server = makeServerSync(cfg, ctx, std::nullopt, dosGuardOverload, e);
@@ -510,10 +544,17 @@ struct WebServerAdminTestParams {
std::string expectedResponse;
};
class WebServerAdminTest : public WebServerTest, public ::testing::WithParamInterface<WebServerAdminTestParams> {};
class WebServerAdminTest : public WebServerTest,
public ::testing::WithParamInterface<WebServerAdminTestParams>,
public prometheus::WithMockPrometheus {};
TEST_P(WebServerAdminTest, WsAdminCheck)
{
::testing::StrictMock<util::prometheus::MockCounterImplInt>& wsMessagesCounterMock =
makeMock<util::prometheus::GaugeInt>("ws_messages_length", "");
EXPECT_CALL(wsMessagesCounterMock, add(1));
EXPECT_CALL(wsMessagesCounterMock, add(-1));
auto e = std::make_shared<AdminCheckExecutor>();
Config const serverConfig{parse(GetParam().config)};
auto server = makeServerSync(serverConfig, ctx, std::nullopt, dosGuardOverload, e);