ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-15 09:15:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: Fix integration tests (#1788)

Browse Source 
Fixes #1784
This commit is contained in:
Sergey Kuznetsov
2024-12-17 15:35:17 +00:00
committed by GitHub
parent 8dc7f16ef1
commit fc3e60f17f
3 changed files with 116 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/util/newconfig/ConfigDefinition.hpp
View File

@@ -312,25 +312,33 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
{"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateIP)}},
{"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}},
{"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}},
{"forwarding.cache_timeout",
ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)},
{"forwarding.request_timeout",
ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)},
{"rpc.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)},
{"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateNumMarkers)},
{"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}},
{"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}.optional()}},
{"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000u).withConstraint(validateUint32)},
{"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)},
{"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)},
{"dos_guard.sweep_interval",
ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)},
{"workers",
ConfigValue{ConfigType::Integer}.defaultValue(std::thread::hardware_concurrency()).withConstraint(validateUint32)
},
{"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)},
{"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)},
{"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)},
@@ -341,11 +349,16 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
{"server.parallel_requests_limit", ConfigValue{ConfigType::Integer}.optional()},
{"server.ws_max_sending_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(1500)},
{"server.__ng_web_server", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateIOThreads)},
{"subscription_workers", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint32)},
{"graceful_period", ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)},
{"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)},
{"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)},
{"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},
@@ -353,28 +366,45 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
},
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)},
{"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}},
{"log_channels.[].log_level",
Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}},
{"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)},
{"log_format",
ConfigValue{ConfigType::String}.defaultValue(
R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)"
)},
{"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"log_directory", ConfigValue{ConfigType::String}.optional()},
{"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(validateUint32)},
{"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(validateUint32)
},
{"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)},
{"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none").withConstraint(validateLogTag)},
{"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(1u).withConstraint(validateUint32)},
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},
{"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
{"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
{"ssl_cert_file", ConfigValue{ConfigType::String}.optional()},
{"ssl_key_file", ConfigValue{ConfigType::String}.optional()},
{"api_version.default",
ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_DEFAULT).withConstraint(validateApiVersion)},
{"api_version.min",

126
tests/integration/data/BackendFactoryTests.cpp
View File

@@ -21,50 +21,71 @@
#include "data/cassandra/Handle.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/newconfig/ConfigConstraints.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include "util/newconfig/ConfigFileJson.hpp"
#include "util/newconfig/ConfigValue.hpp"
#include "util/newconfig/Types.hpp"
#include <TestGlobals.hpp>
#include <boost/json/parse.hpp>
#include <fmt/core.h>
#include <gtest/gtest.h>
#include <algorithm>
#include <cstdint>
#include <iostream>
#include <stdexcept>
#include <string>
#include <thread>
using namespace util::config;
namespace {
constexpr auto keyspace = "factory_test";
} // namespace
struct BackendCassandraFactoryTest : SyncAsioContextTest, util::prometheus::WithPrometheus {
constexpr static auto keyspace = "factory_test";
class BackendCassandraFactoryTest : public SyncAsioContextTest, public util::prometheus::WithPrometheus {
protected:
void
SetUp() override
{
SyncAsioContextTest::SetUp();
}
ClioConfigDefinition cfg_{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
{"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)},
{"database.cassandra.threads",
ConfigValue{ConfigType::Integer}.defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))},
{"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}
};
void
TearDown() override
useConfig(std::string config)
{
SyncAsioContextTest::TearDown();
auto jsonConfig = boost::json::parse(config).as_object();
auto const parseErrors = cfg_.parse(ConfigFileJson{jsonConfig});
if (parseErrors) {
std::ranges::for_each(*parseErrors, [](auto const& error) { std::cout << error.error << std::endl; });
FAIL() << "Failed to parse config";
}
}
};
class BackendCassandraFactoryTestWithDB : public BackendCassandraFactoryTest {
protected:
void
SetUp() override
{
BackendCassandraFactoryTest::SetUp();
}
void
TearDown() override
{
BackendCassandraFactoryTest::TearDown();
// drop the keyspace for next test
data::cassandra::Handle const handle{TestGlobals::instance().backendHost};
EXPECT_TRUE(handle.connect());
@@ -74,34 +95,28 @@ protected:
TEST_F(BackendCassandraFactoryTest, NoSuchBackend)
{
ClioConfigDefinition const cfg{{"database.type", ConfigValue{ConfigType::String}.defaultValue("unknown")}};
EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json( {"database": {"type": "unknown"}} )json");
EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect)
{
ClioConfigDefinition const cfg{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("127.0.0.2")},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)}
};
EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json(
{"database": {
"type": "cassandra",
"cassandra": {
"contact_points": "127.0.0.2"
}
}}
)json");
EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
{
ClioConfigDefinition const cfg{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};
{
auto backend = data::make_Backend(cfg);
auto backend = data::make_Backend(cfg_);
EXPECT_TRUE(backend);
// empty db does not have ledger range
@@ -115,7 +130,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
}
{
auto backend = data::make_Backend(cfg);
auto backend = data::make_Backend(cfg_);
EXPECT_TRUE(backend);
auto const range = backend->fetchLedgerRange();
@@ -126,38 +141,15 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithEmptyDB)
{
ClioConfigDefinition const cfg{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};
EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json( {"read_only": true} )json");
EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBReady)
{
ClioConfigDefinition const cfgReadOnly{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};
auto cfgReadOnly = cfg_;
ASSERT_FALSE(cfgReadOnly.parse(ConfigFileJson{boost::json::parse(R"json( {"read_only": true} )json").as_object()}));
ClioConfigDefinition const cfgWrite{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};
EXPECT_TRUE(data::make_Backend(cfgWrite));
EXPECT_TRUE(data::make_Backend(cfg_));
EXPECT_TRUE(data::make_Backend(cfgReadOnly));
}

33
tests/integration/data/cassandra/BackendTests.cpp
View File

@@ -58,6 +58,7 @@
#include <optional>
#include <random>
#include <string>
#include <thread>
#include <tuple>
#include <unordered_map>
#include <utility>
@@ -73,15 +74,33 @@ using namespace data::cassandra;
class BackendCassandraTest : public SyncAsioContextTest, public WithPrometheus {
protected:
ClioConfigDefinition cfg{
{{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}}
ClioConfigDefinition cfg_{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.keyspace",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
{"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)},
{"database.cassandra.threads",
ConfigValue{ConfigType::Integer}.defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))},
{"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}
};
ObjectView obj = cfg.getObject("database.cassandra");
ObjectView obj = cfg_.getObject("database.cassandra");
SettingsProvider settingsProvider{obj};
// recreated for each test