diff --git a/Builds/Test.py b/Builds/Test.py index 8ff129a281..555f0ca451 100755 --- a/Builds/Test.py +++ b/Builds/Test.py @@ -154,6 +154,13 @@ parser.add_argument( help='Add a prefix for unit tests', ) +parser.add_argument( + '--testjobs', + default='0', + type=int, + help='Run tests in parallel' +) + parser.add_argument( '--clean', '-c', action='store_true', @@ -377,11 +384,14 @@ def run_cmake_tests(directory, target, config): print('Unit tests for', executable) testflag = '--unittest' quiet = '' + testjobs = '' if ARGS.test: testflag += ('=' + ARGS.test) if ARGS.quiet: quiet = '-q' - resultcode, lines = shell(executable, (testflag, quiet,)) + if ARGS.testjobs: + testjobs = ('--unittest-jobs=' + str(ARGS.testjobs)) + resultcode, lines = shell(executable, (testflag, quiet, testjobs,)) if resultcode: if not ARGS.verbose: diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj index c642192e94..66f51f63e8 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj +++ b/Builds/VisualStudio2015/RippleD.vcxproj @@ -4999,8 +4999,6 @@ True True - - True True @@ -5233,6 +5231,12 @@ True True + + True + True + + + diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters index 8574b309dd..0c744ed4c8 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters @@ -502,6 +502,9 @@ {4FD99791-5191-0BFF-8D77-19500238E44E} + + {D4796FCA-4A81-C3A8-FC86-FEF2CEEFC056} + @@ -5721,9 +5724,6 @@ test\protocol - - test - test\resource @@ -5898,5 +5898,11 @@ test\unity + + test\unit_test + + + test\unit_test + diff --git a/Builds/build_all.sh b/Builds/build_all.sh index e1de6c192d..48a3d0ba3b 100755 --- a/Builds/build_all.sh +++ b/Builds/build_all.sh @@ -4,5 +4,5 @@ num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of ph path=$(cd $(dirname $0) && pwd) cd $(dirname $path) -${path}/Test.py -a -c --test=TxQ -- -j${num_procs} -${path}/Test.py -a -c -k --test=TxQ --cmake -- -j${num_procs} +${path}/Test.py -a -c --testjobs=${num_procs} -- -j${num_procs} +${path}/Test.py -a -c -k --cmake --testjobs=${num_procs} -- -j${num_procs} diff --git a/CMakeLists.txt b/CMakeLists.txt index 76c75d17ed..796fa433eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,7 +322,8 @@ foreach(curdir resource rpc server - shamap) + shamap + unit_test) file(GLOB_RECURSE cursrcs src/test/${curdir}/*.cpp) list(APPEND test_srcs "${cursrcs}") endforeach() diff --git a/SConstruct b/SConstruct index 619c6c8d79..ac9faed5d7 100644 --- a/SConstruct +++ b/SConstruct @@ -978,6 +978,7 @@ def get_classic_sources(toolchain): append_sources(result, *list_sources('src/test/shamap', '.cpp')) append_sources(result, *list_sources('src/test/jtx', '.cpp')) append_sources(result, *list_sources('src/test/csf', '.cpp')) + append_sources(result, *list_sources('src/test/unit_test', '.cpp')) if use_shp(toolchain): diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 78b1779917..93f053344b 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -18,6 +18,7 @@ //============================================================================== #include + #include #include #include @@ -39,21 +40,30 @@ #include #include #include + #include #include #include #include -#include +#include + #include + #include +#include + #include #include #include #include -#if defined(BEAST_LINUX) || defined(BEAST_MAC) || defined(BEAST_BSD) -#include +#if BOOST_VERSION >= 106400 +#define HAS_BOOST_PROCESS 1 +#endif + +#if HAS_BOOST_PROCESS +#include #endif namespace po = boost::program_options; @@ -172,23 +182,70 @@ static int runUnitTests( std::string const& pattern, std::string const& argument, bool quiet, - bool log) + bool log, + bool child, + std::size_t num_jobs, + int argc, + char** argv) { using namespace beast::unit_test; using namespace ripple::test; - beast::unit_test::dstream dout{std::cout}; - std::unique_ptr r; - if(quiet) - r = std::make_unique(dout, log); +#if HAS_BOOST_PROCESS + if (!child && num_jobs == 1) +#endif + { + multi_runner_parent parent_runner; + + multi_runner_child child_runner{num_jobs, quiet, log}; + auto const any_failed = child_runner.run_multi(match_auto(pattern)); + + if (any_failed) + return EXIT_FAILURE; + return EXIT_SUCCESS; + } +#if HAS_BOOST_PROCESS + if (!child) + { + multi_runner_parent parent_runner; + std::vector children; + + std::string const exe_name = argv[0]; + std::vector args; + { + args.reserve(argc); + for (int i = 1; i < argc; ++i) + args.emplace_back(argv[i]); + args.emplace_back("--unittest-child"); + } + + for (std::size_t i = 0; i < num_jobs; ++i) + children.emplace_back( + boost::process::exe = exe_name, boost::process::args = args); + + int bad_child_exits = 0; + for(auto& c : children) + { + c.wait(); + if (c.exit_code()) + ++bad_child_exits; + } + + if (parent_runner.any_failed() || bad_child_exits) + return EXIT_FAILURE; + return EXIT_SUCCESS; + } else - r = std::make_unique(dout); - r->arg(argument); - bool const anyFailed = r->run_each_if( - global_suites(), match_auto(pattern)); - if(anyFailed) - return EXIT_FAILURE; - return EXIT_SUCCESS; + { + // child + multi_runner_child runner{num_jobs, quiet, log}; + auto const anyFailed = runner.run_multi(match_auto(pattern)); + + if (anyFailed) + return EXIT_FAILURE; + return EXIT_SUCCESS; + } +#endif } //------------------------------------------------------------------------------ @@ -227,6 +284,10 @@ int run (int argc, char** argv) ("unittest,u", po::value ()->implicit_value (""), "Perform unit tests.") ("unittest-arg", po::value ()->implicit_value (""), "Supplies argument to unit tests.") ("unittest-log", po::value ()->implicit_value (""), "Force unit test log output, even in quiet mode.") +#if HAS_BOOST_PROCESS + ("unittest-jobs", po::value (), "Number of unittest jobs to run.") + ("unittest-child", "For internal use only. Run the process as a unit test child process.") +#endif ("parameters", po::value< vector > (), "Specify comma separated parameters.") ("quiet,q", "Reduce diagnotics.") ("quorum", po::value (), "Override the minimum validation quorum.") @@ -288,10 +349,35 @@ int run (int argc, char** argv) if (vm.count("unittest-arg")) argument = vm["unittest-arg"].as(); + + std::size_t numJobs = 1; + bool unittestChild = false; +#if HAS_BOOST_PROCESS + if (vm.count("unittest-jobs")) + numJobs = std::max(numJobs, vm["unittest-jobs"].as()); + unittestChild = bool (vm.count("unittest-child")); +#endif + return runUnitTests( vm["unittest"].as(), argument, bool (vm.count ("quiet")), - bool (vm.count ("unittest-log"))); + bool (vm.count ("unittest-log")), + unittestChild, + numJobs, + argc, + argv); + } + else + { +#if HAS_BOOST_PROCESS + if (vm.count("unittest-jobs")) + { + // unittest jobs only makes sense with `unittest` + std::cerr << "rippled: '--unittest-jobs' specified without '--unittest'.\n"; + std::cerr << "To run the unit tests the '--unittest' option must be present.\n"; + return 1; + } +#endif } auto config = std::make_unique(); diff --git a/src/test/jtx/impl/envconfig.cpp b/src/test/jtx/impl/envconfig.cpp index 33640aee48..3868725ef7 100644 --- a/src/test/jtx/impl/envconfig.cpp +++ b/src/test/jtx/impl/envconfig.cpp @@ -24,21 +24,18 @@ namespace ripple { namespace test { +int port_base = 8000; +void incPorts() +{ + port_base += 3; +} + void setupConfigForUnitTests (Config& cfg) { - static int port_base = 8000; - std::string port_peer; - std::string port_rpc; - std::string port_ws; - static std::mutex m; - { - std::lock_guard l(m); - port_peer = to_string(port_base); - port_rpc = to_string(port_base + 1); - port_ws = to_string(port_base + 2); - port_base += 3; - } + std::string const port_peer = to_string(port_base); + std::string port_rpc = to_string(port_base + 1); + std::string port_ws = to_string(port_base + 2); cfg.overwrite (ConfigSection::nodeDatabase (), "type", "memory"); cfg.overwrite (ConfigSection::nodeDatabase (), "path", "main"); diff --git a/src/test/ledger/View_test.cpp b/src/test/ledger/View_test.cpp index 42bc8db559..2c5d8f2fba 100644 --- a/src/test/ledger/View_test.cpp +++ b/src/test/ledger/View_test.cpp @@ -690,7 +690,7 @@ class View_test // The two Env's can't share the same ports, so modifiy the config // of the second Env to use higher port numbers - Env eB {*this, envconfig(port_increment, 5)}; + Env eB {*this, envconfig(port_increment, 3)}; // Make ledgers that are incompatible with the first ledgers. Note // that bob is funded before alice. diff --git a/src/test/quiet_reporter.h b/src/test/quiet_reporter.h deleted file mode 100644 index 7b6c804af9..0000000000 --- a/src/test/quiet_reporter.h +++ /dev/null @@ -1,222 +0,0 @@ -// -// Copyright (c) 2013-2016 Vinnie Falco (vinnie dot falco at gmail dot com) -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// - -#ifndef TEST_QUIET_REPORTER_H -#define TEST_QUIET_REPORTER_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace test { - -/** A simple test runner that only reports failures and a summary to the output - stream. To also report log events, set the runner argument to "log". -*/ -class quiet_reporter : public beast::unit_test::runner -{ -private: - - using clock_type = std::chrono::steady_clock; - - struct case_results - { - std::string name; - std::size_t total = 0; - std::size_t failed = 0; - - explicit - case_results(std::string name_ = "") - : name(std::move(name_)) - { - } - }; - - struct suite_results - { - std::string name; - std::size_t cases = 0; - std::size_t total = 0; - std::size_t failed = 0; - - typename clock_type::time_point start = clock_type::now(); - - explicit - suite_results(std::string name_ = "") - : name(std::move(name_)) - { - } - - void - add(case_results const& r) - { - cases++; - total += r.total; - failed += r.failed; - } - }; - - struct results - { - std::size_t suites = 0; - std::size_t cases = 0; - std::size_t total = 0; - std::size_t failed = 0; - - typename clock_type::time_point start = clock_type::now(); - - using run_time = std::pair; - - std::vector top_; - - void - add(suite_results const & s) - { - suites++; - cases += s.cases; - total += s.total; - failed += s.failed; - top_.emplace_back(s.name, clock_type::now() - s.start); - - } - }; - - std::ostream& os_; - suite_results suite_results_; - case_results case_results_; - results results_; - bool print_log_ = false; - - static - std::string - fmtdur(typename clock_type::duration const& d) - { - using namespace std::chrono; - auto const ms = duration_cast(d); - if(ms < seconds{1}) - return boost::lexical_cast( - ms.count()) + "ms"; - std::stringstream ss; - ss << std::fixed << std::setprecision(1) << - (ms.count()/1000.) << "s"; - return ss.str(); - } - -public: - quiet_reporter(quiet_reporter const&) = delete; - quiet_reporter& operator=(quiet_reporter const&) = delete; - explicit - quiet_reporter(std::ostream& os = std::cout, bool log = false) - : os_(os), print_log_{log} {} - - ~quiet_reporter() - { - using namespace beast::unit_test; - auto & top = results_.top_; - if(!top.empty()) - { - std::sort(top.begin(), top.end(), - [](auto const & a, auto const & b) - { - return b.second < a.second; - }); - - if(top.size() > 10) - top.resize(10); - - os_ << "Longest suite times:\n"; - for(auto const& i : top) - os_ << std::setw(8) << - fmtdur(i.second) << " " << i.first << '\n'; - } - - auto const elapsed = clock_type::now() - results_.start; - os_ << - fmtdur(elapsed) << ", " << - amount{results_.suites, "suite"} << ", " << - amount{results_.cases, "case"} << ", " << - amount{results_.total, "test"} << " total, " << - amount{results_.failed, "failure"} << - std::endl; - } - -private: - virtual - void - on_suite_begin(beast::unit_test::suite_info const& info) override - { - suite_results_ = suite_results{info.full_name()}; - } - - virtual - void - on_suite_end() override - { - results_.add(suite_results_); - } - - virtual - void - on_case_begin(std::string const& name) override - { - case_results_ = case_results(name); - } - - virtual - void - on_case_end() override - { - suite_results_.add(case_results_); - } - - virtual - void - on_pass() override - { - ++case_results_.total; - } - - virtual - void - on_fail(std::string const& reason) override - { - ++case_results_.failed; - ++case_results_.total; - os_ << suite_results_.name << - (case_results_.name.empty() ? "" : - (" " + case_results_.name)) - << " #" << case_results_.total << " failed" << - (reason.empty() ? "" : ": ") << reason << std::endl; - } - - virtual - void - on_log(std::string const& s) override - { - if (print_log_) - { - os_ << suite_results_.name << - (case_results_.name.empty() ? "" : - (" " + case_results_.name)) - << " " << s; - os_.flush(); - } - } -}; -} // ripple -} // test - -#endif diff --git a/src/test/rpc/RPCOverload_test.cpp b/src/test/rpc/RPCOverload_test.cpp index 10c287c2ac..efb3d37c3b 100644 --- a/src/test/rpc/RPCOverload_test.cpp +++ b/src/test/rpc/RPCOverload_test.cpp @@ -63,7 +63,7 @@ public: { // Don't use BEAST_EXPECT above b/c it will be called a non-deterministic number of times // and the number of tests run should be deterministic - fail(); + fail("", __FILE__, __LINE__); } if(jv.isMember(jss::warning)) diff --git a/src/test/rpc/Subscribe_test.cpp b/src/test/rpc/Subscribe_test.cpp index a8102becac..3e99db0ae4 100644 --- a/src/test/rpc/Subscribe_test.cpp +++ b/src/test/rpc/Subscribe_test.cpp @@ -454,7 +454,7 @@ public: } { - Env env_nonadmin {*this, no_admin(envconfig(port_increment, 2))}; + Env env_nonadmin {*this, no_admin(envconfig(port_increment, 3))}; Json::Value jv; jv[jss::url] = "no-url"; auto jr = env_nonadmin.rpc("json", method, to_string(jv)) [jss::result]; diff --git a/src/test/shamap/SHAMapSync_test.cpp b/src/test/shamap/SHAMapSync_test.cpp index 417b20c834..1459507f1e 100644 --- a/src/test/shamap/SHAMapSync_test.cpp +++ b/src/test/shamap/SHAMapSync_test.cpp @@ -179,14 +179,14 @@ public: gotNodes_b, rand_bool(eng_), rand_int(eng_, 2))) - fail(); + fail("", __FILE__, __LINE__); } // Don't use BEAST_EXPECT here b/c it will be called a non-deterministic number of times // and the number of tests run should be deterministic if (gotNodeIDs_b.size() != gotNodes_b.size() || gotNodeIDs_b.empty()) - fail(); + fail("", __FILE__, __LINE__); for (std::size_t i = 0; i < gotNodeIDs_b.size(); ++i) { @@ -196,7 +196,7 @@ public: .addKnownNode( gotNodeIDs_b[i], makeSlice(gotNodes_b[i]), nullptr) .isUseful()) - fail(); + fail("", __FILE__, __LINE__); } } while (true); diff --git a/src/test/unit_test/multi_runner.cpp b/src/test/unit_test/multi_runner.cpp new file mode 100644 index 0000000000..ee8df7ee32 --- /dev/null +++ b/src/test/unit_test/multi_runner.cpp @@ -0,0 +1,545 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2017 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +#include + +#include +#include +#include +#include + +namespace ripple { +namespace test { + +extern void +incPorts(); + +namespace detail { + +std::string +fmtdur(typename clock_type::duration const& d) +{ + using namespace std::chrono; + auto const ms = duration_cast(d); + if (ms < seconds{1}) + return boost::lexical_cast(ms.count()) + "ms"; + std::stringstream ss; + ss << std::fixed << std::setprecision(1) << (ms.count() / 1000.) << "s"; + return ss.str(); +} + +//------------------------------------------------------------------------------ + +void +suite_results::add(case_results const& r) +{ + ++cases; + total += r.total; + failed += r.failed; +} + +//------------------------------------------------------------------------------ + +void +results::add(suite_results const& r) +{ + ++suites; + total += r.total; + cases += r.cases; + failed += r.failed; + auto const elapsed = clock_type::now() - r.start; + if (elapsed >= std::chrono::seconds{1}) + { + auto const iter = std::lower_bound( + top.begin(), + top.end(), + elapsed, + [](run_time const& t1, typename clock_type::duration const& t2) { + return t1.second > t2; + }); + + if (iter != top.end()) + { + if (top.size() == max_top && iter == top.end() - 1) + { + // avoid invalidating the iterator + *iter = run_time{ + static_string{static_string::string_view_type{r.name}}, + elapsed}; + } + else + { + if (top.size() == max_top) + top.resize(top.size() - 1); + top.emplace( + iter, + static_string{static_string::string_view_type{r.name}}, + elapsed); + } + } + else if (top.size() < max_top) + { + top.emplace_back( + static_string{static_string::string_view_type{r.name}}, + elapsed); + } + } +} + +void +results::merge(results const& r) +{ + suites += r.suites; + total += r.total; + cases += r.cases; + failed += r.failed; + + // combine the two top collections + boost::container::static_vector top_result; + top_result.resize(top.size() + r.top.size()); + std::merge( + top.begin(), + top.end(), + r.top.begin(), + r.top.end(), + top_result.begin(), + [](run_time const& t1, run_time const& t2) { + return t1.second > t2.second; + }); + + if (top_result.size() > max_top) + top_result.resize(max_top); + + top = top_result; +} + +template +void +results::print(S& s) +{ + using namespace beast::unit_test; + + if (top.size() > 0) + { + s << "Longest suite times:\n"; + for (auto const& i : top) + s << std::setw(8) << fmtdur(i.second) << " " << i.first << '\n'; + } + + auto const elapsed = clock_type::now() - start; + s << fmtdur(elapsed) << ", " << amount{suites, "suite"} << ", " + << amount{cases, "case"} << ", " << amount{total, "test"} << " total, " + << amount{failed, "failure"} << std::endl; +} + +//------------------------------------------------------------------------------ + +template +std::size_t +multi_runner_base::inner::checkout_job_index() +{ + return job_index_++; +} + +template +std::size_t +multi_runner_base::inner::checkout_test_index() +{ + return test_index_++; +} + +template +bool +multi_runner_base::inner::any_failed() const +{ + return any_failed_; +} + +template +void +multi_runner_base::inner::any_failed(bool v) +{ + any_failed_ = any_failed_ || v; +} + +template +void +multi_runner_base::inner::inc_keep_alive_count() +{ + ++keep_alive_; +} + +template +std::size_t +multi_runner_base::inner::get_keep_alive_count() +{ + return keep_alive_; +} + +template +void +multi_runner_base::inner::add(results const& r) +{ + std::lock_guard l{m_}; + results_.merge(r); +} + +template +template +void +multi_runner_base::inner::print_results(S& s) +{ + std::lock_guard l{m_}; + results_.print(s); +} + +template +multi_runner_base::multi_runner_base() +{ + try + { + if (IsParent) + { + // cleanup any leftover state for any previous failed runs + boost::interprocess::shared_memory_object::remove(shared_mem_name_); + boost::interprocess::message_queue::remove(message_queue_name_); + } + + shared_mem_ = boost::interprocess::shared_memory_object{ + std::conditional_t< + IsParent, + boost::interprocess::create_only_t, + boost::interprocess::open_only_t>{}, + shared_mem_name_, + boost::interprocess::read_write}; + + if (IsParent) + { + shared_mem_.truncate(sizeof(inner)); + message_queue_ = + std::make_unique( + boost::interprocess::create_only, + message_queue_name_, + /*max messages*/ 16, + /*max message size*/ 1 << 20); + } + else + { + message_queue_ = + std::make_unique( + boost::interprocess::open_only, message_queue_name_); + } + + region_ = boost::interprocess::mapped_region{ + shared_mem_, boost::interprocess::read_write}; + if (IsParent) + inner_ = new (region_.get_address()) inner{}; + else + inner_ = reinterpret_cast(region_.get_address()); + } + catch (...) + { + if (IsParent) + { + boost::interprocess::shared_memory_object::remove(shared_mem_name_); + boost::interprocess::message_queue::remove(message_queue_name_); + } + throw; + } +} + +template +multi_runner_base::~multi_runner_base() +{ + if (IsParent) + { + inner_->~inner(); + boost::interprocess::shared_memory_object::remove(shared_mem_name_); + boost::interprocess::message_queue::remove(message_queue_name_); + } +} + +template +std::size_t +multi_runner_base::checkout_test_index() +{ + return inner_->checkout_test_index(); +} + +template +std::size_t +multi_runner_base::checkout_job_index() +{ + return inner_->checkout_job_index(); +} + +template +bool +multi_runner_base::any_failed() const +{ + return inner_->any_failed(); +} + +template +void +multi_runner_base::any_failed(bool v) +{ + return inner_->any_failed(v); +} + +template +void +multi_runner_base::add(results const& r) +{ + inner_->add(r); +} + +template +void +multi_runner_base::inc_keep_alive_count() +{ + inner_->inc_keep_alive_count(); +} + +template +std::size_t +multi_runner_base::get_keep_alive_count() +{ + return inner_->get_keep_alive_count(); +} + +template +template +void +multi_runner_base::print_results(S& s) +{ + inner_->print_results(s); +} + +template +void +multi_runner_base::message_queue_send(std::string const& s) +{ + // Even though the message queue does _not_ live in shared memory, child + // processes (the only ones using "send" need to protect access with a mutex + // on the OSX platform (access does not appear to need to be protection on + // linux or windows). This is likely due to the different back end implementation + // of message queue in boost, though that has not been confirmed. + std::lock_guard l{inner_->m_}; + message_queue_->send(s.c_str(), s.size(), /*priority*/ 0); +} + +template +constexpr const char* multi_runner_base::shared_mem_name_; +template +constexpr const char* multi_runner_base::message_queue_name_; + +} // detail + +//------------------------------------------------------------------------------ + +multi_runner_parent::multi_runner_parent() + : os_(std::cout) +{ + message_queue_thread_ = std::thread([this] { + std::vector buf(1 << 20); + while (this->continue_message_queue_) + { + // let children know the parent is still alive + this->inc_keep_alive_count(); + if (!this->message_queue_->get_num_msg()) + { + // If a child does not see the keep alive count incremented, + // it will assume the parent has died. This sleep time needs + // to be small enough so the child will see increments from + // a live parent. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + continue; + } + try + { + std::size_t recvd_size = 0; + unsigned int priority = 0; + this->message_queue_->receive( + buf.data(), buf.size(), recvd_size, priority); + if (recvd_size) + { + std::string s{buf.data(), recvd_size}; + this->os_ << s; + this->os_.flush(); + } + } + catch (...) + { + std::cerr << "Error reading unit test message queue.\n"; + return; + } + } + }); +} + +multi_runner_parent::~multi_runner_parent() +{ + using namespace beast::unit_test; + + continue_message_queue_ = false; + message_queue_thread_.join(); + + print_results(os_); +} + +bool +multi_runner_parent::any_failed() const +{ + return multi_runner_base::any_failed(); +} + +//------------------------------------------------------------------------------ + +multi_runner_child::multi_runner_child( + std::size_t num_jobs, + bool quiet, + bool print_log) + : job_index_{checkout_job_index()} + , num_jobs_{num_jobs} + , quiet_{quiet} + , print_log_{print_log} +{ + // incPort twice (2*jobIndex_) because some tests need two envs + for (std::size_t i = 0; i < 2 * job_index_; ++i) + test::incPorts(); + + if (num_jobs_ > 1) + { + keep_alive_thread_ = std::thread([this] { + std::size_t last_count = get_keep_alive_count(); + while (this->continue_keep_alive_) + { + // Use a small sleep time so in the normal case the child + // process may shutdown quickly. However, to protect against + // false alarms, use a longer sleep time later on. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + auto cur_count = this->get_keep_alive_count(); + if (cur_count == last_count) + { + // longer sleep time to protect against false alarms + std::this_thread::sleep_for(std::chrono::seconds(2)); + cur_count = this->get_keep_alive_count(); + if (cur_count == last_count) + { + // assume parent process is no longer alive + std::cerr << "multi_runner_child " << job_index_ + << ": Assuming parent died, exiting.\n"; + std::exit(EXIT_FAILURE); + } + } + last_count = cur_count; + } + }); + } +} + +multi_runner_child::~multi_runner_child() +{ + if (num_jobs_ > 1) + { + continue_keep_alive_ = false; + keep_alive_thread_.join(); + } + + add(results_); +} + +void +multi_runner_child::on_suite_begin(beast::unit_test::suite_info const& info) +{ + suite_results_ = detail::suite_results{info.full_name()}; +} + +void +multi_runner_child::on_suite_end() +{ + results_.add(suite_results_); +} + +void +multi_runner_child::on_case_begin(std::string const& name) +{ + case_results_ = detail::case_results(name); + + if (quiet_) + return; + + std::stringstream s; + if (num_jobs_ > 1) + s << job_index_ << "> "; + s << suite_results_.name + << (case_results_.name.empty() ? "" : (" " + case_results_.name)) << '\n'; + message_queue_send(s.str()); +} + +void +multi_runner_child::on_case_end() +{ + suite_results_.add(case_results_); +} + +void +multi_runner_child::on_pass() +{ + ++case_results_.total; +} + +void +multi_runner_child::on_fail(std::string const& reason) +{ + ++case_results_.failed; + ++case_results_.total; + std::stringstream s; + if (num_jobs_ > 1) + s << job_index_ << "> "; + s << "#" << case_results_.total << " failed" << (reason.empty() ? "" : ": ") + << reason << '\n'; + message_queue_send(s.str()); +} + +void +multi_runner_child::on_log(std::string const& msg) +{ + if (!print_log_) + return; + + std::stringstream s; + if (num_jobs_ > 1) + s << job_index_ << "> "; + s << msg; + message_queue_send(s.str()); +} + +namespace detail { +template class multi_runner_base; +template class multi_runner_base; +} + +} // unit_test +} // beast diff --git a/src/test/unit_test/multi_runner.h b/src/test/unit_test/multi_runner.h new file mode 100644 index 0000000000..dc21e0e1ba --- /dev/null +++ b/src/test/unit_test/multi_runner.h @@ -0,0 +1,350 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2017 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef TEST_UNIT_TEST_MULTI_RUNNER_H +#define TEST_UNIT_TEST_MULTI_RUNNER_H + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +namespace detail { + +using clock_type = std::chrono::steady_clock; + +struct case_results +{ + std::string name; + std::size_t total = 0; + std::size_t failed = 0; + + explicit + case_results(std::string name_ = "") + : name(std::move(name_)) + { + } +}; + +struct suite_results +{ + std::string name; + std::size_t cases = 0; + std::size_t total = 0; + std::size_t failed = 0; + typename clock_type::time_point start = clock_type::now(); + + explicit + suite_results(std::string name_ = "") + : name(std::move(name_)) + { + } + + void + add(case_results const& r); +}; + +struct results +{ + using static_string = beast::static_string<256>; + // results may be stored in shared memory. Use `static_string` to ensure + // pointers from different memory spaces do not co-mingle + using run_time = std::pair; + + enum { max_top = 10 }; + + std::size_t suites = 0; + std::size_t cases = 0; + std::size_t total = 0; + std::size_t failed = 0; + boost::container::static_vector top; + typename clock_type::time_point start = clock_type::now(); + + void + add(suite_results const& r); + + void + merge(results const& r); + + template + void + print(S& s); +}; + +template +class multi_runner_base +{ + // `inner` will be created in shared memory. This is one way + // multi_runner_parent and multi_runner_child object communicate. The other + // way they communicate is through message queues. + struct inner + { + std::atomic job_index_{0}; + std::atomic test_index_{0}; + std::atomic any_failed_{false}; + // A parent process will periodically increment `keep_alive_`. The child + // processes will check if `keep_alive_` is being incremented. If it is + // not incremented for a sufficiently long time, the child will assume the + // parent process has died. + std::atomic keep_alive_{0}; + + mutable boost::interprocess::interprocess_mutex m_; + detail::results results_; + + std::size_t + checkout_job_index(); + + std::size_t + checkout_test_index(); + + bool + any_failed() const; + + void + any_failed(bool v); + + void + inc_keep_alive_count(); + + std::size_t + get_keep_alive_count(); + + void + add(results const& r); + + template + void + print_results(S& s); + }; + + static constexpr const char* shared_mem_name_ = "RippledUnitTestSharedMem"; + // name of the message queue a multi_runner_child will use to communicate with + // multi_runner_parent + static constexpr const char* message_queue_name_ = "RippledUnitTestMessageQueue"; + + // `inner_` will be created in shared memory + inner* inner_; + // shared memory to use for the `inner` member + boost::interprocess::shared_memory_object shared_mem_; + boost::interprocess::mapped_region region_; + +protected: + std::unique_ptr message_queue_; + + void message_queue_send(std::string const& s); + +public: + multi_runner_base(); + ~multi_runner_base(); + + std::size_t + checkout_test_index(); + + std::size_t + checkout_job_index(); + + void + any_failed(bool v); + + void + add(results const& r); + + void + inc_keep_alive_count(); + + std::size_t + get_keep_alive_count(); + + template + void + print_results(S& s); + + bool + any_failed() const; +}; + +} // detail + +//------------------------------------------------------------------------------ + +/** Manager for children running unit tests + */ +class multi_runner_parent : private detail::multi_runner_base +{ +private: + // message_queue_ is used to collect log messages from the children + std::ostream& os_; + std::atomic continue_message_queue_{true}; + std::thread message_queue_thread_; + +public: + multi_runner_parent(multi_runner_parent const&) = delete; + multi_runner_parent& + operator=(multi_runner_parent const&) = delete; + + multi_runner_parent(); + ~multi_runner_parent(); + + bool + any_failed() const; +}; + +//------------------------------------------------------------------------------ + +/** A class to run a subset of unit tests + */ +class multi_runner_child : public beast::unit_test::runner, + private detail::multi_runner_base +{ +private: + std::size_t job_index_; + detail::results results_; + detail::suite_results suite_results_; + detail::case_results case_results_; + std::size_t num_jobs_{0}; + bool quiet_{false}; + bool print_log_{true}; + + std::atomic continue_keep_alive_{true}; + std::thread keep_alive_thread_; + +public: + multi_runner_child(multi_runner_child const&) = delete; + multi_runner_child& + operator=(multi_runner_child const&) = delete; + + multi_runner_child(std::size_t num_jobs, bool quiet, bool print_log); + ~multi_runner_child(); + + template + bool + run_multi(Pred pred); + +private: + virtual void + on_suite_begin(beast::unit_test::suite_info const& info) override; + + virtual void + on_suite_end() override; + + virtual void + on_case_begin(std::string const& name) override; + + virtual void + on_case_end() override; + + virtual void + on_pass() override; + + virtual void + on_fail(std::string const& reason) override; + + virtual void + on_log(std::string const& s) override; +}; + +//------------------------------------------------------------------------------ + +template +bool +multi_runner_child::run_multi(Pred pred) +{ + auto const& suite = beast::unit_test::global_suites(); + auto const num_tests = suite.size(); + // actual order to run the tests. Use this to move longer running tests to + // the beginning to better take advantage of a multi process run. + std::vector order(num_tests); + std::iota(order.begin(), order.end(), 0); + { + std::unordered_set prioritize{ + "ripple.app.Flow", "ripple.tx.Offer"}; + std::vector to_swap; + to_swap.reserve(prioritize.size()); + + size_t i = 0; + for (auto const& t : suite) + { + auto const full_name = t.full_name(); + if (prioritize.count(full_name)) + { + to_swap.push_back(i); + if (to_swap.size() == prioritize.size()) + break; + } + ++i; + } + + for (std::size_t i = 0; i < to_swap.size(); ++i) + std::swap(order[to_swap[i]], order[i]); + } + bool failed = false; + + auto get_test = [&]() -> beast::unit_test::suite_info const* { + auto const cur_test_index = checkout_test_index(); + if (cur_test_index >= num_tests) + return nullptr; + auto iter = suite.begin(); + std::advance(iter, order[cur_test_index]); + return &*iter; + }; + while (auto t = get_test()) + { + if (!pred(*t)) + continue; + try + { + failed = run(*t) || failed; + } + catch (...) + { + if (num_jobs_ <= 1) + throw; // a single process can die + + // inform the parent + std::stringstream s; + s << job_index_ << "> failed Unhandled exception in test.\n"; + message_queue_send(s.str()); + failed = true; + } + } + any_failed(failed); + return failed; +} + + +} // unit_test +} // beast + +#endif diff --git a/src/test/unity/app_test_unity1.cpp b/src/test/unity/app_test_unity1.cpp index 9c7e11e0f6..f7bfb84d54 100644 --- a/src/test/unity/app_test_unity1.cpp +++ b/src/test/unity/app_test_unity1.cpp @@ -34,3 +34,5 @@ #include #include #include + +#include