From 60416b18a51306f95019f039ab39866d2563c94f Mon Sep 17 00:00:00 2001 From: Brad Chase Date: Tue, 14 Feb 2017 15:57:16 -0500 Subject: [PATCH] Add quiet unit test reporter --- Builds/Test.py | 11 +- Builds/VisualStudio2015/RippleD.vcxproj | 2 + .../VisualStudio2015/RippleD.vcxproj.filters | 3 + appveyor.yml | 2 +- bin/ci/ubuntu/build-and-test.sh | 4 +- circle.yml | 2 +- src/ripple/app/main/Main.cpp | 23 +- src/test/quiet_reporter.h | 221 ++++++++++++++++++ 8 files changed, 257 insertions(+), 11 deletions(-) create mode 100644 src/test/quiet_reporter.h diff --git a/Builds/Test.py b/Builds/Test.py index ccf2c1622..e17dba568 100755 --- a/Builds/Test.py +++ b/Builds/Test.py @@ -123,6 +123,12 @@ parser.add_argument( help='delete all build artifacts after testing', ) +parser.add_argument( + '--quiet', '-q', + action='store_true', + help='Reduce output where possible (unit tests)', +) + parser.add_argument( 'scons_args', default=(), @@ -186,9 +192,12 @@ def run_tests(args): print('Unit tests for', target) testflag = '--unittest' + quiet = '' if ARGS.test: testflag += ('=' + ARGS.test) - resultcode, lines = shell(executable, (testflag,)) + if ARGS.quiet: + quiet = '-q' + resultcode, lines = shell(executable, (testflag, quiet,)) if resultcode: if not ARGS.verbose: diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj index b6dc1bc0c..d0157e405 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj +++ b/Builds/VisualStudio2015/RippleD.vcxproj @@ -4815,6 +4815,8 @@ True True + + True True diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters index 3113a973c..9cd64989a 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters @@ -5511,6 +5511,9 @@ test\protocol + + test + test\resource diff --git a/appveyor.yml b/appveyor.yml index b02dd9587..63b822ff4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -127,7 +127,7 @@ test_script: - ps: | & { # Run the rippled unit tests - & $exe --unittest + & $exe --unittest --quiet --unittest-log # https://connect.microsoft.com/PowerShell/feedback/details/751703/option-to-stop-script-if-command-line-exe-fails if ($LastExitCode -ne 0) { throw "Unit tests failed" } } diff --git a/bin/ci/ubuntu/build-and-test.sh b/bin/ci/ubuntu/build-and-test.sh index 3f0a49425..1e8be3fbf 100755 --- a/bin/ci/ubuntu/build-and-test.sh +++ b/bin/ci/ubuntu/build-and-test.sh @@ -43,7 +43,7 @@ rm -f build/${APP} ldd $APP_PATH if [[ ${APP} == "rippled" ]]; then - export APP_ARGS="--unittest" + export APP_ARGS="--unittest --quiet --unittest-log" # Only report on src/ripple files export LCOV_FILES="*/src/ripple/*" # Nothing to explicitly exclude @@ -70,7 +70,7 @@ gdb -return-child-result -quiet -batch \ -ex run \ -ex "thread apply all backtrace full" \ -ex "quit" \ - --args $APP_PATH --unittest + --args $APP_PATH --unittest --quiet --unittest-log if [[ $TARGET == "coverage" ]]; then # Create test coverage data file diff --git a/circle.yml b/circle.yml index a457f2f3f..2080004d0 100644 --- a/circle.yml +++ b/circle.yml @@ -21,4 +21,4 @@ test: - scons clang.debug override: # Execute unit tests under gdb - - gdb -return-child-result -quiet -batch -ex "set env MALLOC_CHECK_=3" -ex "set print thread-events off" -ex run -ex "thread apply all backtrace full" -ex "quit" --args build/clang.debug/rippled --unittest + - gdb -return-child-result -quiet -batch -ex "set env MALLOC_CHECK_=3" -ex "set print thread-events off" -ex run -ex "thread apply all backtrace full" -ex "quit" --args build/clang.debug/rippled --unittest --quiet --unittest-log diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 441b9e5fd..8312bbc60 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -168,13 +169,21 @@ void printHelp (const po::options_description& desc) static int runUnitTests( std::string const& pattern, - std::string const& argument) + std::string const& argument, + bool quiet, + bool log) { using namespace beast::unit_test; + using namespace ripple::test; beast::unit_test::dstream dout{std::cout}; - reporter r{dout}; - r.arg(argument); - bool const anyFailed = r.run_each_if( + + std::unique_ptr r; + if(quiet) + r = std::make_unique(dout, log); + else + r = std::make_unique(dout); + r->arg(argument); + bool const anyFailed = r->run_each_if( global_suites(), match_auto(pattern)); if(anyFailed) return EXIT_FAILURE; @@ -216,6 +225,7 @@ int run (int argc, char** argv) ("standalone,a", "Run with no peers.") ("unittest,u", po::value ()->implicit_value (""), "Perform unit tests.") ("unittest-arg", po::value ()->implicit_value (""), "Supplies argument to unit tests.") + ("unittest-log", po::value ()->implicit_value (""), "Force unit test log output, even in quiet mode.") ("parameters", po::value< vector > (), "Specify comma separated parameters.") ("quiet,q", "Reduce diagnotics.") ("quorum", po::value (), "Override the minimum validation quorum.") @@ -277,9 +287,10 @@ int run (int argc, char** argv) if (vm.count("unittest-arg")) argument = vm["unittest-arg"].as(); - return runUnitTests( - vm["unittest"].as(), argument); + vm["unittest"].as(), argument, + bool (vm.count ("quiet")), + bool (vm.count ("unittest-log"))); } auto config = std::make_unique(); diff --git a/src/test/quiet_reporter.h b/src/test/quiet_reporter.h new file mode 100644 index 000000000..da5a3bbd9 --- /dev/null +++ b/src/test/quiet_reporter.h @@ -0,0 +1,221 @@ +// +// Copyright (c) 2013-2016 Vinnie Falco (vinnie dot falco at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef TEST_QUIET_REPORTER_H +#define TEST_QUIET_REPORTER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +/** A simple test runner that only reports failures and a summary to the output + stream. To also report log events, set the runner argument to "log". +*/ +class quiet_reporter : public beast::unit_test::runner +{ +private: + + using clock_type = std::chrono::steady_clock; + + struct case_results + { + std::string name; + std::size_t total = 0; + std::size_t failed = 0; + + explicit + case_results(std::string name_ = "") + : name(std::move(name_)) + { + } + }; + + struct suite_results + { + std::string name; + std::size_t cases = 0; + std::size_t total = 0; + std::size_t failed = 0; + + typename clock_type::time_point start = clock_type::now(); + + explicit + suite_results(std::string name_ = "") + : name(std::move(name_)) + { + } + + void + add(case_results const& r) + { + cases++; + total += r.total; + failed += r.failed; + } + }; + + struct results + { + std::size_t suites = 0; + std::size_t cases = 0; + std::size_t total = 0; + std::size_t failed = 0; + + typename clock_type::time_point start = clock_type::now(); + + using run_time = std::pair; + + std::vector top_; + + void + add(suite_results const & s) + { + suites++; + cases += s.cases; + total += s.total; + failed += s.failed; + top_.emplace_back(s.name, clock_type::now() - s.start); + + } + }; + + std::ostream& os_; + suite_results suite_results_; + case_results case_results_; + results results_; + bool print_log_ = false; + + static + std::string + fmtdur(typename clock_type::duration const& d) + { + using namespace std::chrono; + auto const ms = duration_cast(d); + if(ms < seconds{1}) + return boost::lexical_cast( + ms.count()) + "ms"; + std::stringstream ss; + ss << std::fixed << std::setprecision(1) << + (ms.count()/1000.) << "s"; + return ss.str(); + } + +public: + quiet_reporter(quiet_reporter const&) = delete; + quiet_reporter& operator=(quiet_reporter const&) = delete; + explicit + quiet_reporter(std::ostream& os = std::cout, bool log = false) + : os_(os), print_log_{log} {} + + ~quiet_reporter() + { + using namespace beast::unit_test; + auto & top = results_.top_; + if(!top.empty()) + { + std::sort(top.begin(), top.end(), + [](auto const & a, auto const & b) + { + return b.second < a.second; + }); + + top.resize(10); + + os_ << "Longest suite times:\n"; + for(auto const& i : top) + os_ << std::setw(8) << + fmtdur(i.second) << " " << i.first << '\n'; + } + + auto const elapsed = clock_type::now() - results_.start; + os_ << + fmtdur(elapsed) << ", " << + amount{results_.suites, "suite"} << ", " << + amount{results_.cases, "case"} << ", " << + amount{results_.total, "test"} << " total, " << + amount{results_.failed, "failure"} << + std::endl; + } + +private: + virtual + void + on_suite_begin(beast::unit_test::suite_info const& info) override + { + suite_results_ = suite_results{info.full_name()}; + } + + virtual + void + on_suite_end() override + { + results_.add(suite_results_); + } + + virtual + void + on_case_begin(std::string const& name) override + { + case_results_ = case_results(name); + } + + virtual + void + on_case_end() override + { + suite_results_.add(case_results_); + } + + virtual + void + on_pass() override + { + ++case_results_.total; + } + + virtual + void + on_fail(std::string const& reason) override + { + ++case_results_.failed; + ++case_results_.total; + os_ << suite_results_.name << + (case_results_.name.empty() ? "" : + (" " + case_results_.name)) + << " #" << case_results_.total << " failed" << + (reason.empty() ? "" : ": ") << reason << std::endl; + } + + virtual + void + on_log(std::string const& s) override + { + if (print_log_) + { + os_ << suite_results_.name << + (case_results_.name.empty() ? "" : + (" " + case_results_.name)) + << " " << s; + os_.flush(); + } + } +}; +} // ripple +} // test + +#endif