Add quiet unit test reporter

Builds/Test.py

@@ -123,6 +123,12 @@ parser.add_argument(
     help='delete all build artifacts after testing',
 )
 
+parser.add_argument(
+    '--quiet', '-q',
+    action='store_true',
+    help='Reduce output where possible (unit tests)',
+)
+
 parser.add_argument(
     'scons_args',
     default=(),
@@ -186,9 +192,12 @@ def run_tests(args):
 
             print('Unit tests for', target)
             testflag = '--unittest'
+            quiet = ''
             if ARGS.test:
                 testflag += ('=' + ARGS.test)
-            resultcode, lines = shell(executable, (testflag,))
+            if ARGS.quiet:
+                quiet = '-q'
+            resultcode, lines = shell(executable, (testflag, quiet,))
 
             if resultcode:
                 if not ARGS.verbose:

Builds/VisualStudio2015/RippleD.vcxproj

@@ -4815,6 +4815,8 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClInclude Include="..\..\src\test\quiet_reporter.h">
+    </ClInclude>
     <ClCompile Include="..\..\src\test\resource\Logic_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>

Builds/VisualStudio2015/RippleD.vcxproj.filters

@@ -5511,6 +5511,9 @@
     <ClCompile Include="..\..\src\test\protocol\XRPAmount_test.cpp">
       <Filter>test\protocol</Filter>
     </ClCompile>
+    <ClInclude Include="..\..\src\test\quiet_reporter.h">
+      <Filter>test</Filter>
+    </ClInclude>
     <ClCompile Include="..\..\src\test\resource\Logic_test.cpp">
       <Filter>test\resource</Filter>
     </ClCompile>

appveyor.yml

@@ -127,7 +127,7 @@ test_script:
   - ps: |
         & {
           # Run the rippled unit tests
-          & $exe --unittest
+          & $exe --unittest --quiet --unittest-log
           # https://connect.microsoft.com/PowerShell/feedback/details/751703/option-to-stop-script-if-command-line-exe-fails
           if ($LastExitCode -ne 0) { throw "Unit tests failed" }
         }

bin/ci/ubuntu/build-and-test.sh

@@ -43,7 +43,7 @@ rm -f build/${APP}
 ldd $APP_PATH
 
 if [[ ${APP} == "rippled" ]]; then
-  export APP_ARGS="--unittest"
+  export APP_ARGS="--unittest --quiet --unittest-log"
   # Only report on src/ripple files
   export LCOV_FILES="*/src/ripple/*"
   # Nothing to explicitly exclude
@@ -70,7 +70,7 @@ gdb -return-child-result -quiet -batch \
     -ex run \
     -ex "thread apply all backtrace full" \
     -ex "quit" \
-    --args $APP_PATH --unittest
+    --args $APP_PATH --unittest --quiet --unittest-log
 
 if [[ $TARGET == "coverage" ]]; then
   # Create test coverage data file

circle.yml

@@ -21,4 +21,4 @@ test:
     - scons clang.debug
   override:
     # Execute unit tests under gdb
-    - gdb -return-child-result -quiet -batch -ex "set env MALLOC_CHECK_=3" -ex "set print thread-events off" -ex run -ex "thread apply all backtrace full" -ex "quit" --args build/clang.debug/rippled --unittest
+    - gdb -return-child-result -quiet -batch -ex "set env MALLOC_CHECK_=3" -ex "set print thread-events off" -ex run -ex "thread apply all backtrace full" -ex "quit" --args build/clang.debug/rippled --unittest --quiet --unittest-log

src/ripple/app/main/Main.cpp

@@ -43,6 +43,7 @@
 #include <beast/unit_test/global_suites.hpp>
 #include <beast/unit_test/match.hpp>
 #include <beast/unit_test/reporter.hpp>
+#include <test/quiet_reporter.h>
 #include <google/protobuf/stubs/common.h>
 #include <boost/program_options.hpp>
 #include <cstdlib>
@@ -168,13 +169,21 @@ void printHelp (const po::options_description& desc)
 
 static int runUnitTests(
     std::string const& pattern,
-    std::string const& argument)
+    std::string const& argument,
+    bool quiet,
+    bool log)
 {
     using namespace beast::unit_test;
+    using namespace ripple::test;
     beast::unit_test::dstream dout{std::cout};
-    reporter r{dout};
-    r.arg(argument);
-    bool const anyFailed = r.run_each_if(
+
+    std::unique_ptr<runner> r;
+    if(quiet)
+        r = std::make_unique<quiet_reporter>(dout, log);
+    else
+        r = std::make_unique<reporter>(dout);
+    r->arg(argument);
+    bool const anyFailed = r->run_each_if(
         global_suites(), match_auto(pattern));
     if(anyFailed)
         return EXIT_FAILURE;
@@ -216,6 +225,7 @@ int run (int argc, char** argv)
     ("standalone,a", "Run with no peers.")
     ("unittest,u", po::value <std::string> ()->implicit_value (""), "Perform unit tests.")
     ("unittest-arg", po::value <std::string> ()->implicit_value (""), "Supplies argument to unit tests.")
+    ("unittest-log", po::value <std::string> ()->implicit_value (""), "Force unit test log output, even in quiet mode.")
     ("parameters", po::value< vector<string> > (), "Specify comma separated parameters.")
     ("quiet,q", "Reduce diagnotics.")
     ("quorum", po::value <std::size_t> (), "Override the minimum validation quorum.")
@@ -277,9 +287,10 @@ int run (int argc, char** argv)
 
         if (vm.count("unittest-arg"))
             argument = vm["unittest-arg"].as<std::string>();
-
         return runUnitTests(
-            vm["unittest"].as<std::string>(), argument);
+            vm["unittest"].as<std::string>(), argument,
+            bool (vm.count ("quiet")),
+            bool (vm.count ("unittest-log")));
     }
 
     auto config = std::make_unique<Config>();

src/test/quiet_reporter.h

@@ -0,0 +1,221 @@
+//
+// Copyright (c) 2013-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef TEST_QUIET_REPORTER_H
+#define TEST_QUIET_REPORTER_H
+
+#include <beast/unit_test/amount.hpp>
+#include <beast/unit_test/recorder.hpp>
+#include <boost/lexical_cast.hpp>
+#include <algorithm>
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <utility>
+
+namespace ripple {
+namespace test {
+
+/** A simple test runner that only reports failures and a summary to the output
+    stream.  To also report log events, set the runner argument to "log".
+*/
+class quiet_reporter : public beast::unit_test::runner
+{
+private:
+
+    using clock_type = std::chrono::steady_clock;
+
+    struct case_results
+    {
+        std::string name;
+        std::size_t total = 0;
+        std::size_t failed = 0;
+
+        explicit
+        case_results(std::string name_ = "")
+            : name(std::move(name_))
+        {
+        }
+    };
+
+    struct suite_results
+    {
+        std::string name;
+        std::size_t cases = 0;
+        std::size_t total = 0;
+        std::size_t failed = 0;
+
+        typename clock_type::time_point start = clock_type::now();
+
+        explicit
+        suite_results(std::string name_ = "")
+            : name(std::move(name_))
+        {
+        }
+
+        void
+        add(case_results const& r)
+        {
+            cases++;
+            total += r.total;
+            failed += r.failed;
+        }
+    };
+
+    struct results
+    {
+        std::size_t suites = 0;
+        std::size_t cases = 0;
+        std::size_t total = 0;
+        std::size_t failed = 0;
+
+        typename clock_type::time_point start = clock_type::now();
+
+        using run_time = std::pair<std::string,
+            typename clock_type::duration>;
+
+        std::vector<run_time> top_;
+
+        void
+        add(suite_results const & s)
+        {
+            suites++;
+            cases += s.cases;
+            total += s.total;
+            failed += s.failed;
+            top_.emplace_back(s.name, clock_type::now() - s.start);
+
+        }
+    };
+
+    std::ostream& os_;
+    suite_results suite_results_;
+    case_results case_results_;
+    results results_;
+    bool print_log_ = false;
+
+    static
+    std::string
+    fmtdur(typename clock_type::duration const& d)
+    {
+        using namespace std::chrono;
+        auto const ms = duration_cast<milliseconds>(d);
+        if(ms < seconds{1})
+            return boost::lexical_cast<std::string>(
+                ms.count()) + "ms";
+        std::stringstream ss;
+        ss << std::fixed << std::setprecision(1) <<
+           (ms.count()/1000.) << "s";
+        return ss.str();
+    }
+
+public:
+    quiet_reporter(quiet_reporter const&) = delete;
+    quiet_reporter& operator=(quiet_reporter const&) = delete;
+    explicit
+    quiet_reporter(std::ostream& os = std::cout, bool log = false)
+        : os_(os), print_log_{log} {}
+
+    ~quiet_reporter()
+    {
+        using namespace beast::unit_test;
+        auto & top = results_.top_;
+        if(!top.empty())
+        {
+            std::sort(top.begin(), top.end(),
+                [](auto const & a, auto const & b)
+                {
+                    return b.second < a.second;
+                });
+
+            top.resize(10);
+
+            os_ << "Longest suite times:\n";
+            for(auto const& i : top)
+                os_ << std::setw(8) <<
+                    fmtdur(i.second) << " " << i.first << '\n';
+        }
+
+        auto const elapsed = clock_type::now() - results_.start;
+        os_ <<
+            fmtdur(elapsed) << ", " <<
+            amount{results_.suites, "suite"} << ", " <<
+            amount{results_.cases, "case"} << ", " <<
+            amount{results_.total, "test"} << " total, " <<
+            amount{results_.failed, "failure"} <<
+            std::endl;
+    }
+
+private:
+    virtual
+    void
+    on_suite_begin(beast::unit_test::suite_info const& info) override
+    {
+        suite_results_ = suite_results{info.full_name()};
+    }
+
+    virtual
+    void
+    on_suite_end() override
+    {
+        results_.add(suite_results_);
+    }
+
+    virtual
+    void
+    on_case_begin(std::string const& name) override
+    {
+        case_results_ = case_results(name);
+    }
+
+    virtual
+    void
+    on_case_end() override
+    {
+        suite_results_.add(case_results_);
+    }
+
+    virtual
+    void
+    on_pass() override
+    {
+        ++case_results_.total;
+    }
+
+    virtual
+    void
+    on_fail(std::string const& reason) override
+    {
+        ++case_results_.failed;
+        ++case_results_.total;
+        os_ << suite_results_.name <<
+            (case_results_.name.empty() ? "" :
+                (" " + case_results_.name))
+            << " #" << case_results_.total << " failed" <<
+            (reason.empty() ? "" : ": ") << reason << std::endl;
+    }
+
+    virtual
+    void
+    on_log(std::string const& s) override
+    {
+        if (print_log_)
+        {
+            os_ << suite_results_.name <<
+                (case_results_.name.empty() ? "" :
+                (" " + case_results_.name))
+                << " " << s;
+            os_.flush();
+        }
+    }
+};
+} // ripple
+} // test
+
+#endif