Separate unit tests and integration tests (#1393)

Fixes #1391
This commit is contained in:
Alex Kremer
2024-05-07 15:12:48 +01:00
committed by GitHub
parent 98ef83d470
commit cbc856b190
177 changed files with 168 additions and 106 deletions

View File

@@ -144,5 +144,5 @@ CheckOptions:
bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*'
HeaderFilterRegex: '^.*/(src|unittests)/.*\.(h|hpp)$'
HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
WarningsAsErrors: '*'

View File

@@ -8,7 +8,7 @@
echo "+ Checking code format..."
# paths to check and re-format
sources="src unittests"
sources="src tests"
formatter="clang-format -i"
version=$($formatter --version | grep -o '[0-9\.]*')

View File

@@ -6,12 +6,16 @@ runs:
- name: Run tests
shell: bash
run: |
build/clio_tests --backend_host=scylladb
build/clio_tests
- name: Run gcovr
shell: bash
run: |
gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
gcovr -e tests \
-e src/data/cassandra \
-e src/data/CassandraBackend.hpp \
-e 'src/data/BackendFactory.*' \
--xml build/coverage_report.xml -j8 --exclude-throw-branches
- name: Archive coverage report
uses: actions/upload-artifact@v4

View File

@@ -35,7 +35,7 @@ runs:
STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
run: |
cd build
conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:integration_tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
- name: Run cmake
shell: bash

View File

@@ -81,15 +81,6 @@ jobs:
runs-on: [self-hosted, "${{ matrix.os }}"]
container: ${{ matrix.container }}
services:
scylladb:
image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
options: >-
--health-cmd "cqlsh -e 'describe cluster'"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Clean workdir
if: ${{ runner.os == 'macOS' }}
@@ -143,7 +134,7 @@ jobs:
- name: Strip tests
if: ${{ !matrix.code_coverage }}
run: strip build/clio_tests
run: strip build/clio_tests && strip build/clio_integration_tests
- name: Upload clio_server
uses: actions/upload-artifact@v4
@@ -156,7 +147,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
path: build/clio_tests
path: build/clio_*tests
- name: Save cache
uses: ./.github/actions/save_cache
@@ -223,4 +214,4 @@ jobs:
- name: Run clio_tests
run: |
chmod +x ./clio_tests
./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
./clio_tests

View File

@@ -95,6 +95,15 @@ jobs:
runs-on: [self-hosted, "${{ matrix.os }}"]
container: ${{ matrix.container }}
services:
scylladb:
image: 'scylladb/scylla'
options: >-
--health-cmd "cqlsh -e 'describe cluster'"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Clean workdir
if: ${{ runner.os == 'macOS' }}
@@ -107,7 +116,12 @@ jobs:
- name: Run clio_tests
run: |
chmod +x ./clio_tests
./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
./clio_tests
- name: Run clio_integration_tests
run: |
chmod +x ./clio_integration_tests
./clio_integration_tests --backend_host=scylladb
nightly_release:
needs: run_tests
@@ -129,7 +143,7 @@ jobs:
run: |
cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
cd nightly_release
rm -r clio_tests*
rm -r clio_*tests*
for d in $(ls); do
archive_name=$(ls $d)
mv ${d}/${archive_name} ./

View File

@@ -8,7 +8,8 @@ project(clio VERSION ${CLIO_VERSION} HOMEPAGE_URL "https://github.com/XRPLF/clio
# =========================== Options ====================================== #
option(verbose "Verbose build" FALSE)
option(tests "Build tests" FALSE)
option(tests "Build unit tests" FALSE)
option(integration_tests "Build integration tests" FALSE)
option(benchmark "Build benchmarks" FALSE)
option(docs "Generate doxygen docs" FALSE)
option(coverage "Build test coverage report" FALSE)
@@ -32,15 +33,6 @@ add_library(clio_options INTERFACE)
target_compile_features(clio_options INTERFACE cxx_std_23) # Clio needs c++23 but deps can remain c++20 for now
target_include_directories(clio_options INTERFACE ${CMAKE_SOURCE_DIR}/src)
# Set coverage build options
if (coverage)
if (NOT tests)
message(FATAL_ERROR "Coverage requires tests to be enabled")
endif ()
include(CodeCoverage)
append_coverage_compiler_flags_to_target(clio_options INTERFACE)
endif ()
if (verbose)
set(CMAKE_VERBOSE_MAKEFILE TRUE)
endif ()
@@ -65,10 +57,7 @@ include(deps/cassandra)
include(deps/libbacktrace)
add_subdirectory(src)
if (tests)
add_subdirectory(unittests)
endif ()
add_subdirectory(tests)
if (benchmark)
add_subdirectory(benchmarks)

View File

@@ -6,7 +6,7 @@ To contribute, please:
2. Create a new branch on which to commit/push your changes.
3. Write and test your code.
4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate.
5. Where applicable, write test cases for your code and include those in `unittests`.
5. Where applicable, write test cases for your code and include those in the relevant subfolder under `tests`.
6. Ensure your code passes automated checks (e.g. clang-format)
7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change). See below for more details.
8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template.

View File

@@ -9,15 +9,16 @@ class Clio(ConanFile):
description = 'Clio RPC server'
settings = 'os', 'compiler', 'build_type', 'arch'
options = {
'static': [True, False], # static linkage
'fPIC': [True, False], # unused?
'static': [True, False], # static linkage
'fPIC': [True, False], # unused?
'verbose': [True, False],
'tests': [True, False], # build unit tests; create `clio_tests` binary
'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
'docs': [True, False], # doxygen API docs; create custom target 'docs'
'packaging': [True, False], # create distribution packages
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
'lint': [True, False], # run clang-tidy checks during compilation
'tests': [True, False], # build unit tests; create `clio_tests` binary
'integration_tests': [True, False], # build integration tests; create `clio_integration_tests` binary
'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
'docs': [True, False], # doxygen API docs; create custom target 'docs'
'packaging': [True, False], # create distribution packages
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
'lint': [True, False], # run clang-tidy checks during compilation
}
requires = [
@@ -36,6 +37,7 @@ class Clio(ConanFile):
'fPIC': True,
'verbose': False,
'tests': False,
'integration_tests': False,
'benchmark': False,
'packaging': False,
'coverage': False,
@@ -62,7 +64,7 @@ class Clio(ConanFile):
)
def requirements(self):
if self.options.tests:
if self.options.tests or self.options.integration_tests:
self.requires('gtest/1.14.0')
if self.options.benchmark:
self.requires('benchmark/1.8.3')
@@ -83,6 +85,7 @@ class Clio(ConanFile):
tc.variables['verbose'] = self.options.verbose
tc.variables['static'] = self.options.static
tc.variables['tests'] = self.options.tests
tc.variables['integration_tests'] = self.options.integration_tests
tc.variables['coverage'] = self.options.coverage
tc.variables['lint'] = self.options.lint
tc.variables['docs'] = self.options.docs

View File

@@ -27,15 +27,6 @@ In case of a spurious failure of unit tests, it is possible to re-run the `cover
The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `cmake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
If some unit tests predictably fail (e.g., due to absence of a Cassandra database), it is possible to set unit tests options in the `CODE_COVERAGE_TESTS_ARGS` cmake variable, as demonstrated below:
```sh
cd .build
conan install .. --output-folder . --build missing --settings build_type=Debug -o tests=True -o coverage=True
cmake -DCODE_COVERAGE_REPORT_FORMAT=json-details -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
cmake --build . --target coverage_report
```
After the `coverage_report` target is completed, the generated coverage report will be stored inside the build directory as either:
- A File named `coverage_report.*`, with a suitable extension for the report format.

View File

@@ -12,7 +12,7 @@ See the [common](https://github.com/XRPLF/clio/blob/develop/src/rpc/common) subf
## Implementing a handler
See [unittests/rpc](https://github.com/XRPLF/clio/tree/develop/unittests/rpc) for examples.
See [tests/unit/rpc](https://github.com/XRPLF/clio/tree/develop/tests/unit/rpc) for examples.
Handlers need to fulfil the requirements specified by the `SomeHandler` concept (see `rpc/common/Concepts.hpp`):

20
tests/CMakeLists.txt Normal file
View File

@@ -0,0 +1,20 @@
# Set coverage build options
if (coverage)
if (NOT tests)
message(FATAL_ERROR "Coverage requires tests to be enabled")
endif ()
include(CodeCoverage)
append_coverage_compiler_flags_to_target(clio_options INTERFACE)
endif ()
if (tests OR integration_tests)
add_subdirectory(common)
endif ()
if (tests)
add_subdirectory(unit)
endif ()
if (integration_tests)
add_subdirectory(integration)
endif ()

View File

@@ -0,0 +1,10 @@
add_library(clio_testing_common)
target_sources(
clio_testing_common PRIVATE util/StringUtils.cpp util/TestHttpServer.cpp util/TestWsServer.cpp util/TestObject.cpp
)
include(deps/gtest)
target_include_directories(clio_testing_common PUBLIC .)
target_link_libraries(clio_testing_common PUBLIC clio gtest::gtest)

View File

@@ -35,7 +35,7 @@
#include <optional>
#include <string>
namespace unittests::impl {
namespace tests::common {
// input data for the test handlers below
struct TestInput {
@@ -181,4 +181,4 @@ struct BasicDOSGuardMock : public web::BaseDOSGuard {
MOCK_METHOD(void, clear, (), (noexcept, override));
};
} // namespace unittests::impl
} // namespace tests::common

View File

@@ -36,7 +36,7 @@
#include <string>
#include <thread>
namespace unittests::util {
namespace tests::util {
struct MockXrpLedgerAPIService final : public org::xrpl::rpc::v1::XRPLedgerAPIService::Service {
~MockXrpLedgerAPIService() override = default;
@@ -101,4 +101,4 @@ private:
std::thread serverThread_;
};
} // namespace unittests::util
} // namespace tests::util

View File

@@ -0,0 +1,15 @@
add_executable(clio_integration_tests)
target_sources(
clio_integration_tests
PRIVATE data/BackendFactoryTests.cpp data/cassandra/BackendTests.cpp data/cassandra/BaseTests.cpp
# Test runner
TestGlobals.cpp Main.cpp
)
# Fix for dwarf5 bug on ci. IS STILL NEEDED???
target_compile_options(clio_options INTERFACE -gdwarf-4)
target_include_directories(clio_integration_tests PRIVATE .)
target_link_libraries(clio_integration_tests PUBLIC clio_testing_common)
set_target_properties(clio_integration_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
@@ -18,8 +18,8 @@
//==============================================================================
#include "util/TerminationHandler.hpp"
#include "util/TestGlobals.hpp"
#include <TestGlobals.hpp>
#include <gtest/gtest.h>
/*

View File

@@ -0,0 +1,10 @@
# Integration testing
Tests that hit the real database are separate from the unit test suite found under `tests/unit`.
## Requirements
### Cassandra/ScyllaDB cluster
If you wish to test the backend component you will need to have access to a **local (127.0.0.1)** Cassandra cluster, opened at port **9042**. Please ensure that the cluster is successfully running before running these tests.
## Running
To run the DB tests, first build Clio as normal, then execute `./clio_integration_tests` to run all database tests.

View File

@@ -17,8 +17,7 @@
*/
//==============================================================================
#include "util/TestGlobals.hpp"
#include <TestGlobals.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/value_semantic.hpp>

View File

@@ -20,9 +20,10 @@
#include "data/BackendFactory.hpp"
#include "data/cassandra/Handle.hpp"
#include "util/Fixtures.hpp"
#include "util/TestGlobals.hpp"
#include "util/MockPrometheus.hpp"
#include "util/config/Config.hpp"
#include <TestGlobals.hpp>
#include <boost/json/parse.hpp>
#include <fmt/core.h>
#include <gtest/gtest.h>
@@ -34,7 +35,7 @@ namespace {
constexpr auto keyspace = "factory_test";
} // namespace
class BackendCassandraFactoryTest : public SyncAsioContextTest {
class BackendCassandraFactoryTest : public SyncAsioContextTest, public util::prometheus::WithPrometheus {
protected:
void
SetUp() override

View File

@@ -30,9 +30,9 @@
#include "util/MockPrometheus.hpp"
#include "util/Random.hpp"
#include "util/StringUtils.hpp"
#include "util/TestGlobals.hpp"
#include "util/config/Config.hpp"
#include <TestGlobals.hpp>
#include <boost/asio/impl/spawn.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>

View File

@@ -20,8 +20,8 @@
#include "data/cassandra/Handle.hpp"
#include "data/cassandra/Types.hpp"
#include "util/Fixtures.hpp"
#include "util/TestGlobals.hpp"
#include <TestGlobals.hpp>
#include <cassandra.h>
#include <fmt/core.h>
#include <gtest/gtest.h>

View File

@@ -5,11 +5,8 @@ target_sources(
PRIVATE # Common
ConfigTests.cpp
data/BackendCountersTests.cpp
data/BackendFactoryTests.cpp
data/BackendInterfaceTests.cpp
data/cassandra/AsyncExecutorTests.cpp
data/cassandra/BackendTests.cpp
data/cassandra/BaseTests.cpp
data/cassandra/ExecutionStrategyTests.cpp
data/cassandra/RetryPolicyTests.cpp
data/cassandra/SettingsProviderTests.cpp
@@ -119,11 +116,6 @@ target_sources(
util/requests/WsConnectionTests.cpp
util/RetryTests.cpp
util/SignalsHandlerTests.cpp
util/StringUtils.cpp
util/TestGlobals.cpp
util/TestHttpServer.cpp
util/TestObject.cpp
util/TestWsServer.cpp
util/TxUtilTests.cpp
# Webserver
web/AdminVerificationTests.cpp
@@ -133,8 +125,6 @@ target_sources(
web/WhitelistHandlerTests.cpp
)
include(deps/gtest)
# See https://github.com/google/googletest/issues/3475
gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 90)
@@ -143,7 +133,7 @@ target_compile_options(clio_options INTERFACE -gdwarf-4)
target_compile_definitions(clio_tests PUBLIC UNITTEST_BUILD)
target_include_directories(clio_tests PRIVATE .)
target_link_libraries(clio_tests PUBLIC clio gtest::gtest)
target_link_libraries(clio_tests PUBLIC clio_testing_common)
set_target_properties(clio_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Generate `coverage_report` target if coverage is enabled
@@ -174,7 +164,10 @@ if (coverage)
--gtest_brief=1
${TESTS_ADDITIONAL_ARGS}
EXCLUDE
"unittests"
"tests"
"src/data/cassandra"
"src/data/CassandraBackend.hpp"
"src/data/BackendFactory.*"
DEPENDENCIES
clio_tests
)

31
tests/unit/Main.cpp Normal file
View File

@@ -0,0 +1,31 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "util/TerminationHandler.hpp"
#include <gtest/gtest.h>
int
main(int argc, char* argv[])
{
util::setTerminationHandler();
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

9
tests/unit/README.md Normal file
View File

@@ -0,0 +1,9 @@
# Unit testing
The correctness of new implementations can be verified via running unit tests. Below are the information on how to run unit tests.
## Running
To run the unit tests, first build Clio as normal, then execute `./clio_tests` to run all unit tests.
# Adding Unit Tests
To add unit tests, please create a separate file for the component you are trying to cover (unless it already exists) and use any other existing unit test file as an example.

View File

@@ -18,8 +18,8 @@
//==============================================================================
#include "data/cassandra/Error.hpp"
#include "data/cassandra/FakesAndMocks.hpp"
#include "data/cassandra/impl/AsyncExecutor.hpp"
#include "data/cassandra/impl/FakesAndMocks.hpp"
#include "util/Fixtures.hpp"
#include <boost/asio/io_context.hpp>

View File

@@ -18,10 +18,9 @@
//==============================================================================
#include "data/BackendInterface.hpp"
#include "data/cassandra/Error.hpp"
#include "data/cassandra/FakesAndMocks.hpp"
#include "data/cassandra/Types.hpp"
#include "data/cassandra/impl/ExecutionStrategy.hpp"
#include "data/cassandra/impl/FakesAndMocks.hpp"
#include "util/Fixtures.hpp"
#include <boost/asio/io_context.hpp>

View File

@@ -20,8 +20,8 @@
#include "data/Types.hpp"
#include "etl/CacheLoader.hpp"
#include "etl/CacheLoaderSettings.hpp"
#include "etl/FakeDiffProvider.hpp"
#include "etl/impl/CacheLoader.hpp"
#include "etl/impl/FakeDiffProvider.hpp"
#include "util/Fixtures.hpp"
#include "util/MockCache.hpp"
#include "util/MockPrometheus.hpp"

View File

@@ -18,8 +18,8 @@
//==============================================================================
#include "data/Types.hpp"
#include "etl/FakeDiffProvider.hpp"
#include "etl/impl/CursorFromFixDiffNumProvider.hpp"
#include "etl/impl/FakeDiffProvider.hpp"
#include "util/Fixtures.hpp"
#include "util/MockPrometheus.hpp"

View File

@@ -40,9 +40,7 @@
using namespace etl::impl;
struct GrpcSourceTests : NoLoggerFixture,
util::prometheus::WithPrometheus,
unittests::util::WithMockXrpLedgerAPIService {
struct GrpcSourceTests : NoLoggerFixture, util::prometheus::WithPrometheus, tests::util::WithMockXrpLedgerAPIService {
GrpcSourceTests()
: WithMockXrpLedgerAPIService("localhost:55051")
, mockBackend_(std::make_shared<testing::StrictMock<MockBackend>>(util::Config{}))

Some files were not shown because too many files have changed in this diff Show More