Run unit tests in parallel

Builds/Test.py

@@ -154,6 +154,13 @@ parser.add_argument(
     help='Add a prefix for unit tests',
 )
 
+parser.add_argument(
+    '--testjobs',
+    default='0',
+    type=int,
+    help='Run tests in parallel'
+)
+
 parser.add_argument(
     '--clean', '-c',
     action='store_true',
@@ -377,11 +384,14 @@ def run_cmake_tests(directory, target, config):
     print('Unit tests for', executable)
     testflag = '--unittest'
     quiet = ''
+    testjobs = ''
     if ARGS.test:
         testflag += ('=' + ARGS.test)
     if ARGS.quiet:
         quiet = '-q'
-    resultcode, lines = shell(executable, (testflag, quiet,))
+    if ARGS.testjobs:
+        testjobs = ('--unittest-jobs=' + str(ARGS.testjobs))
+    resultcode, lines = shell(executable, (testflag, quiet, testjobs,))
 
     if resultcode:
         if not ARGS.verbose:

Builds/VisualStudio2015/RippleD.vcxproj

@@ -4999,8 +4999,6 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
-    <ClInclude Include="..\..\src\test\quiet_reporter.h">
-    </ClInclude>
     <ClCompile Include="..\..\src\test\resource\Logic_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -5233,6 +5231,12 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\unit_test\multi_runner.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\unit_test\multi_runner.h">
+    </ClInclude>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

Builds/VisualStudio2015/RippleD.vcxproj.filters

@@ -502,6 +502,9 @@
     <Filter Include="test\unity">
       <UniqueIdentifier>{4FD99791-5191-0BFF-8D77-19500238E44E}</UniqueIdentifier>
     </Filter>
+    <Filter Include="test\unit_test">
+      <UniqueIdentifier>{D4796FCA-4A81-C3A8-FC86-FEF2CEEFC056}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\build\proto\ripple.pb.cc">
@@ -5721,9 +5724,6 @@
     <ClCompile Include="..\..\src\test\protocol\XRPAmount_test.cpp">
       <Filter>test\protocol</Filter>
     </ClCompile>
-    <ClInclude Include="..\..\src\test\quiet_reporter.h">
-      <Filter>test</Filter>
-    </ClInclude>
     <ClCompile Include="..\..\src\test\resource\Logic_test.cpp">
       <Filter>test\resource</Filter>
     </ClCompile>
@@ -5898,5 +5898,11 @@
     <ClCompile Include="..\..\src\test\unity\shamap_test_unity.cpp">
       <Filter>test\unity</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\unit_test\multi_runner.cpp">
+      <Filter>test\unit_test</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\unit_test\multi_runner.h">
+      <Filter>test\unit_test</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

Builds/build_all.sh

@@ -4,5 +4,5 @@ num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of ph
 
 path=$(cd $(dirname $0) && pwd)
 cd $(dirname $path)
-${path}/Test.py -a -c --test=TxQ -- -j${num_procs}
+${path}/Test.py -a -c --testjobs=${num_procs} -- -j${num_procs}
-${path}/Test.py -a -c -k --test=TxQ --cmake -- -j${num_procs}
+${path}/Test.py -a -c -k --cmake --testjobs=${num_procs} -- -j${num_procs}

CMakeLists.txt

@@ -322,7 +322,8 @@ foreach(curdir
         resource
         rpc
         server
-        shamap)
+        shamap
+        unit_test)
     file(GLOB_RECURSE cursrcs src/test/${curdir}/*.cpp)
     list(APPEND test_srcs "${cursrcs}")
 endforeach()

SConstruct

@@ -978,6 +978,7 @@ def get_classic_sources(toolchain):
     append_sources(result, *list_sources('src/test/shamap', '.cpp'))
     append_sources(result, *list_sources('src/test/jtx', '.cpp'))
     append_sources(result, *list_sources('src/test/csf', '.cpp'))
+    append_sources(result, *list_sources('src/test/unit_test', '.cpp'))
 
 
     if use_shp(toolchain):

src/ripple/app/main/Main.cpp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <BeastConfig.h>
+
 #include <ripple/basics/Log.h>
 #include <ripple/protocol/digest.h>
 #include <ripple/app/main/Application.h>
@@ -39,21 +40,30 @@
 #include <ripple/beast/core/CurrentThreadName.h>
 #include <ripple/beast/core/Time.h>
 #include <ripple/beast/utility/Debug.h>
+
 #include <beast/unit_test/dstream.hpp>
 #include <beast/unit_test/global_suites.hpp>
 #include <beast/unit_test/match.hpp>
 #include <beast/unit_test/reporter.hpp>
-#include <test/quiet_reporter.h>
+#include <test/unit_test/multi_runner.h>
+
 #include <google/protobuf/stubs/common.h>
+
 #include <boost/program_options.hpp>
+#include <boost/version.hpp>
+
 #include <cstdlib>
 #include <iostream>
 #include <utility>
 #include <stdexcept>
 
 
-#if defined(BEAST_LINUX) || defined(BEAST_MAC) || defined(BEAST_BSD)
+#if BOOST_VERSION >= 106400
-#include <sys/resource.h>
+#define HAS_BOOST_PROCESS 1
+#endif
+
+#if HAS_BOOST_PROCESS
+#include <boost/process.hpp>
 #endif
 
 namespace po = boost::program_options;
@@ -172,23 +182,70 @@ static int runUnitTests(
     std::string const& pattern,
     std::string const& argument,
     bool quiet,
-    bool log)
+    bool log,
+    bool child,
+    std::size_t num_jobs,
+    int argc,
+    char** argv)
 {
     using namespace beast::unit_test;
     using namespace ripple::test;
-    beast::unit_test::dstream dout{std::cout};
 
-    std::unique_ptr<runner> r;
+#if HAS_BOOST_PROCESS
-    if(quiet)
+    if (!child && num_jobs == 1)
-        r = std::make_unique<quiet_reporter>(dout, log);
+#endif
+    {
+        multi_runner_parent parent_runner;
+
+        multi_runner_child child_runner{num_jobs, quiet, log};
+        auto const any_failed = child_runner.run_multi(match_auto(pattern));
+
+        if (any_failed)
+            return EXIT_FAILURE;
+        return EXIT_SUCCESS;
+    }
+#if HAS_BOOST_PROCESS
+    if (!child)
+    {
+        multi_runner_parent parent_runner;
+        std::vector<boost::process::child> children;
+
+        std::string const exe_name = argv[0];
+        std::vector<std::string> args;
+        {
+            args.reserve(argc);
+            for (int i = 1; i < argc; ++i)
+                args.emplace_back(argv[i]);
+            args.emplace_back("--unittest-child");
+        }
+
+        for (std::size_t i = 0; i < num_jobs; ++i)
+            children.emplace_back(
+                boost::process::exe = exe_name, boost::process::args = args);
+
+        int bad_child_exits = 0;
+        for(auto& c : children)
+        {
+            c.wait();
+            if (c.exit_code())
+                ++bad_child_exits;
+        }
+
+        if (parent_runner.any_failed() || bad_child_exits)
+            return EXIT_FAILURE;
+        return EXIT_SUCCESS;
+    }
     else
-        r = std::make_unique<reporter>(dout);
+    {
-    r->arg(argument);
+        // child
-    bool const anyFailed = r->run_each_if(
+        multi_runner_child runner{num_jobs, quiet, log};
-        global_suites(), match_auto(pattern));
+        auto const anyFailed = runner.run_multi(match_auto(pattern));
-    if(anyFailed)
+
-        return EXIT_FAILURE;
+        if (anyFailed)
-    return EXIT_SUCCESS;
+            return EXIT_FAILURE;
+        return EXIT_SUCCESS;
+    }
+#endif
 }
 
 //------------------------------------------------------------------------------
@@ -227,6 +284,10 @@ int run (int argc, char** argv)
     ("unittest,u", po::value <std::string> ()->implicit_value (""), "Perform unit tests.")
     ("unittest-arg", po::value <std::string> ()->implicit_value (""), "Supplies argument to unit tests.")
     ("unittest-log", po::value <std::string> ()->implicit_value (""), "Force unit test log output, even in quiet mode.")
+#if HAS_BOOST_PROCESS
+    ("unittest-jobs", po::value <std::size_t> (), "Number of unittest jobs to run.")
+    ("unittest-child", "For internal use only. Run the process as a unit test child process.")
+#endif
     ("parameters", po::value< vector<string> > (), "Specify comma separated parameters.")
     ("quiet,q", "Reduce diagnotics.")
     ("quorum", po::value <std::size_t> (), "Override the minimum validation quorum.")
@@ -288,10 +349,35 @@ int run (int argc, char** argv)
 
         if (vm.count("unittest-arg"))
             argument = vm["unittest-arg"].as<std::string>();
+
+        std::size_t numJobs = 1;
+        bool unittestChild = false;
+#if HAS_BOOST_PROCESS
+        if (vm.count("unittest-jobs"))
+            numJobs = std::max(numJobs, vm["unittest-jobs"].as<std::size_t>());
+        unittestChild = bool (vm.count("unittest-child"));
+#endif
+
         return runUnitTests(
             vm["unittest"].as<std::string>(), argument,
             bool (vm.count ("quiet")),
-            bool (vm.count ("unittest-log")));
+            bool (vm.count ("unittest-log")),
+            unittestChild,
+            numJobs,
+            argc,
+            argv);
+    }
+    else
+    {
+#if HAS_BOOST_PROCESS
+        if (vm.count("unittest-jobs"))
+        {
+            // unittest jobs only makes sense with `unittest`
+            std::cerr << "rippled: '--unittest-jobs' specified without '--unittest'.\n";
+            std::cerr << "To run the unit tests the '--unittest' option must be present.\n";
+            return 1;
+        }
+#endif
     }
 
     auto config = std::make_unique<Config>();

src/test/jtx/impl/envconfig.cpp

@@ -24,21 +24,18 @@
 namespace ripple {
 namespace test {
 
+int port_base = 8000;
+void incPorts()
+{
+    port_base += 3;
+}
+
 void
 setupConfigForUnitTests (Config& cfg)
 {
-    static int port_base = 8000;
+    std::string const port_peer = to_string(port_base);
-    std::string port_peer;
+    std::string port_rpc = to_string(port_base + 1);
-    std::string port_rpc;
+    std::string port_ws = to_string(port_base + 2);
-    std::string port_ws;
-    static std::mutex m;
-    {
-        std::lock_guard<std::mutex> l(m);
-        port_peer = to_string(port_base);
-        port_rpc = to_string(port_base + 1);
-        port_ws = to_string(port_base + 2);
-        port_base += 3;
-    }
 
     cfg.overwrite (ConfigSection::nodeDatabase (), "type", "memory");
     cfg.overwrite (ConfigSection::nodeDatabase (), "path", "main");

src/test/ledger/View_test.cpp

@@ -690,7 +690,7 @@ class View_test
 
         // The two Env's can't share the same ports, so modifiy the config
         // of the second Env to use higher port numbers
-        Env eB {*this, envconfig(port_increment, 5)};
+        Env eB {*this, envconfig(port_increment, 3)};
 
         // Make ledgers that are incompatible with the first ledgers.  Note
         // that bob is funded before alice.

src/test/quiet_reporter.h

@@ -1,222 +0,0 @@
-//
-// Copyright (c) 2013-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
-// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-//
-
-#ifndef TEST_QUIET_REPORTER_H
-#define TEST_QUIET_REPORTER_H
-
-#include <beast/unit_test/amount.hpp>
-#include <beast/unit_test/recorder.hpp>
-#include <boost/lexical_cast.hpp>
-#include <algorithm>
-#include <chrono>
-#include <iomanip>
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <utility>
-
-namespace ripple {
-namespace test {
-
-/** A simple test runner that only reports failures and a summary to the output
-    stream.  To also report log events, set the runner argument to "log".
-*/
-class quiet_reporter : public beast::unit_test::runner
-{
-private:
-
-    using clock_type = std::chrono::steady_clock;
-
-    struct case_results
-    {
-        std::string name;
-        std::size_t total = 0;
-        std::size_t failed = 0;
-
-        explicit
-        case_results(std::string name_ = "")
-            : name(std::move(name_))
-        {
-        }
-    };
-
-    struct suite_results
-    {
-        std::string name;
-        std::size_t cases = 0;
-        std::size_t total = 0;
-        std::size_t failed = 0;
-
-        typename clock_type::time_point start = clock_type::now();
-
-        explicit
-        suite_results(std::string name_ = "")
-            : name(std::move(name_))
-        {
-        }
-
-        void
-        add(case_results const& r)
-        {
-            cases++;
-            total += r.total;
-            failed += r.failed;
-        }
-    };
-
-    struct results
-    {
-        std::size_t suites = 0;
-        std::size_t cases = 0;
-        std::size_t total = 0;
-        std::size_t failed = 0;
-
-        typename clock_type::time_point start = clock_type::now();
-
-        using run_time = std::pair<std::string,
-            typename clock_type::duration>;
-
-        std::vector<run_time> top_;
-
-        void
-        add(suite_results const & s)
-        {
-            suites++;
-            cases += s.cases;
-            total += s.total;
-            failed += s.failed;
-            top_.emplace_back(s.name, clock_type::now() - s.start);
-
-        }
-    };
-
-    std::ostream& os_;
-    suite_results suite_results_;
-    case_results case_results_;
-    results results_;
-    bool print_log_ = false;
-
-    static
-    std::string
-    fmtdur(typename clock_type::duration const& d)
-    {
-        using namespace std::chrono;
-        auto const ms = duration_cast<milliseconds>(d);
-        if(ms < seconds{1})
-            return boost::lexical_cast<std::string>(
-                ms.count()) + "ms";
-        std::stringstream ss;
-        ss << std::fixed << std::setprecision(1) <<
-           (ms.count()/1000.) << "s";
-        return ss.str();
-    }
-
-public:
-    quiet_reporter(quiet_reporter const&) = delete;
-    quiet_reporter& operator=(quiet_reporter const&) = delete;
-    explicit
-    quiet_reporter(std::ostream& os = std::cout, bool log = false)
-        : os_(os), print_log_{log} {}
-
-    ~quiet_reporter()
-    {
-        using namespace beast::unit_test;
-        auto & top = results_.top_;
-        if(!top.empty())
-        {
-            std::sort(top.begin(), top.end(),
-                [](auto const & a, auto const & b)
-                {
-                    return b.second < a.second;
-                });
-
-            if(top.size() > 10)
-                top.resize(10);
-
-            os_ << "Longest suite times:\n";
-            for(auto const& i : top)
-                os_ << std::setw(8) <<
-                    fmtdur(i.second) << " " << i.first << '\n';
-        }
-
-        auto const elapsed = clock_type::now() - results_.start;
-        os_ <<
-            fmtdur(elapsed) << ", " <<
-            amount{results_.suites, "suite"} << ", " <<
-            amount{results_.cases, "case"} << ", " <<
-            amount{results_.total, "test"} << " total, " <<
-            amount{results_.failed, "failure"} <<
-            std::endl;
-    }
-
-private:
-    virtual
-    void
-    on_suite_begin(beast::unit_test::suite_info const& info) override
-    {
-        suite_results_ = suite_results{info.full_name()};
-    }
-
-    virtual
-    void
-    on_suite_end() override
-    {
-        results_.add(suite_results_);
-    }
-
-    virtual
-    void
-    on_case_begin(std::string const& name) override
-    {
-        case_results_ = case_results(name);
-    }
-
-    virtual
-    void
-    on_case_end() override
-    {
-        suite_results_.add(case_results_);
-    }
-
-    virtual
-    void
-    on_pass() override
-    {
-        ++case_results_.total;
-    }
-
-    virtual
-    void
-    on_fail(std::string const& reason) override
-    {
-        ++case_results_.failed;
-        ++case_results_.total;
-        os_ << suite_results_.name <<
-            (case_results_.name.empty() ? "" :
-                (" " + case_results_.name))
-            << " #" << case_results_.total << " failed" <<
-            (reason.empty() ? "" : ": ") << reason << std::endl;
-    }
-
-    virtual
-    void
-    on_log(std::string const& s) override
-    {
-        if (print_log_)
-        {
-            os_ << suite_results_.name <<
-                (case_results_.name.empty() ? "" :
-                (" " + case_results_.name))
-                << " " << s;
-            os_.flush();
-        }
-    }
-};
-} // ripple
-} // test
-
-#endif

src/test/rpc/RPCOverload_test.cpp

@@ -63,7 +63,7 @@ public:
             {
                 // Don't use BEAST_EXPECT above b/c it will be called a non-deterministic number of times
                 // and the number of tests run should be deterministic
-                fail();
+                fail("", __FILE__, __LINE__);
             }
 
             if(jv.isMember(jss::warning))

src/test/rpc/Subscribe_test.cpp

@@ -454,7 +454,7 @@ public:
         }
 
         {
-            Env env_nonadmin {*this, no_admin(envconfig(port_increment, 2))};
+            Env env_nonadmin {*this, no_admin(envconfig(port_increment, 3))};
             Json::Value jv;
             jv[jss::url] = "no-url";
             auto jr = env_nonadmin.rpc("json", method, to_string(jv)) [jss::result];

src/test/shamap/SHAMapSync_test.cpp

@@ -179,14 +179,14 @@ public:
                         gotNodes_b,
                         rand_bool(eng_),
                         rand_int(eng_, 2)))
-                    fail();
+                    fail("", __FILE__, __LINE__);
             }
 
             // Don't use BEAST_EXPECT here b/c it will be called a non-deterministic number of times
             // and the number of tests run should be deterministic
             if (gotNodeIDs_b.size() != gotNodes_b.size() ||
                 gotNodeIDs_b.empty())
-                fail();
+                fail("", __FILE__, __LINE__);
 
             for (std::size_t i = 0; i < gotNodeIDs_b.size(); ++i)
             {
@@ -196,7 +196,7 @@ public:
                          .addKnownNode(
                              gotNodeIDs_b[i], makeSlice(gotNodes_b[i]), nullptr)
                          .isUseful())
-                    fail();
+                    fail("", __FILE__, __LINE__);
             }
         }
         while (true);

src/test/unit_test/multi_runner.cpp

@@ -0,0 +1,545 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2017 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <test/unit_test/multi_runner.h>
+
+#include <beast/unit_test/amount.hpp>
+
+#include <boost/lexical_cast.hpp>
+
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+
+namespace ripple {
+namespace test {
+
+extern void
+incPorts();
+
+namespace detail {
+
+std::string
+fmtdur(typename clock_type::duration const& d)
+{
+    using namespace std::chrono;
+    auto const ms = duration_cast<milliseconds>(d);
+    if (ms < seconds{1})
+        return boost::lexical_cast<std::string>(ms.count()) + "ms";
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(1) << (ms.count() / 1000.) << "s";
+    return ss.str();
+}
+
+//------------------------------------------------------------------------------
+
+void
+suite_results::add(case_results const& r)
+{
+    ++cases;
+    total += r.total;
+    failed += r.failed;
+}
+
+//------------------------------------------------------------------------------
+
+void
+results::add(suite_results const& r)
+{
+    ++suites;
+    total += r.total;
+    cases += r.cases;
+    failed += r.failed;
+    auto const elapsed = clock_type::now() - r.start;
+    if (elapsed >= std::chrono::seconds{1})
+    {
+        auto const iter = std::lower_bound(
+            top.begin(),
+            top.end(),
+            elapsed,
+            [](run_time const& t1, typename clock_type::duration const& t2) {
+                return t1.second > t2;
+            });
+
+        if (iter != top.end())
+        {
+            if (top.size() == max_top && iter == top.end() - 1)
+            {
+                // avoid invalidating the iterator
+                *iter = run_time{
+                    static_string{static_string::string_view_type{r.name}},
+                    elapsed};
+            }
+            else
+            {
+                if (top.size() == max_top)
+                    top.resize(top.size() - 1);
+                top.emplace(
+                    iter,
+                    static_string{static_string::string_view_type{r.name}},
+                    elapsed);
+            }
+        }
+        else if (top.size() < max_top)
+        {
+            top.emplace_back(
+                static_string{static_string::string_view_type{r.name}},
+                elapsed);
+        }
+    }
+}
+
+void
+results::merge(results const& r)
+{
+    suites += r.suites;
+    total += r.total;
+    cases += r.cases;
+    failed += r.failed;
+
+    // combine the two top collections
+    boost::container::static_vector<run_time, 2 * max_top> top_result;
+    top_result.resize(top.size() + r.top.size());
+    std::merge(
+        top.begin(),
+        top.end(),
+        r.top.begin(),
+        r.top.end(),
+        top_result.begin(),
+        [](run_time const& t1, run_time const& t2) {
+            return t1.second > t2.second;
+        });
+
+    if (top_result.size() > max_top)
+        top_result.resize(max_top);
+
+    top = top_result;
+}
+
+template <class S>
+void
+results::print(S& s)
+{
+    using namespace beast::unit_test;
+
+    if (top.size() > 0)
+    {
+        s << "Longest suite times:\n";
+        for (auto const& i : top)
+            s << std::setw(8) << fmtdur(i.second) << " " << i.first << '\n';
+    }
+
+    auto const elapsed = clock_type::now() - start;
+    s << fmtdur(elapsed) << ", " << amount{suites, "suite"} << ", "
+      << amount{cases, "case"} << ", " << amount{total, "test"} << " total, "
+      << amount{failed, "failure"} << std::endl;
+}
+
+//------------------------------------------------------------------------------
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::inner::checkout_job_index()
+{
+    return job_index_++;
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::inner::checkout_test_index()
+{
+    return test_index_++;
+}
+
+template <bool IsParent>
+bool
+multi_runner_base<IsParent>::inner::any_failed() const
+{
+    return any_failed_;
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::inner::any_failed(bool v)
+{
+    any_failed_ = any_failed_ || v;
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::inner::inc_keep_alive_count()
+{
+    ++keep_alive_;
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::inner::get_keep_alive_count()
+{
+    return keep_alive_;
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::inner::add(results const& r)
+{
+    std::lock_guard<boost::interprocess::interprocess_mutex> l{m_};
+    results_.merge(r);
+}
+
+template <bool IsParent>
+template <class S>
+void
+multi_runner_base<IsParent>::inner::print_results(S& s)
+{
+    std::lock_guard<boost::interprocess::interprocess_mutex> l{m_};
+    results_.print(s);
+}
+
+template <bool IsParent>
+multi_runner_base<IsParent>::multi_runner_base()
+{
+    try
+    {
+        if (IsParent)
+        {
+            // cleanup any leftover state for any previous failed runs
+            boost::interprocess::shared_memory_object::remove(shared_mem_name_);
+            boost::interprocess::message_queue::remove(message_queue_name_);
+        }
+
+        shared_mem_ = boost::interprocess::shared_memory_object{
+            std::conditional_t<
+                IsParent,
+                boost::interprocess::create_only_t,
+                boost::interprocess::open_only_t>{},
+            shared_mem_name_,
+            boost::interprocess::read_write};
+
+        if (IsParent)
+        {
+            shared_mem_.truncate(sizeof(inner));
+            message_queue_ =
+                std::make_unique<boost::interprocess::message_queue>(
+                    boost::interprocess::create_only,
+                    message_queue_name_,
+                    /*max messages*/ 16,
+                    /*max message size*/ 1 << 20);
+        }
+        else
+        {
+            message_queue_ =
+                std::make_unique<boost::interprocess::message_queue>(
+                    boost::interprocess::open_only, message_queue_name_);
+        }
+
+        region_ = boost::interprocess::mapped_region{
+            shared_mem_, boost::interprocess::read_write};
+        if (IsParent)
+            inner_ = new (region_.get_address()) inner{};
+        else
+            inner_ = reinterpret_cast<inner*>(region_.get_address());
+    }
+    catch (...)
+    {
+        if (IsParent)
+        {
+            boost::interprocess::shared_memory_object::remove(shared_mem_name_);
+            boost::interprocess::message_queue::remove(message_queue_name_);
+        }
+        throw;
+    }
+}
+
+template <bool IsParent>
+multi_runner_base<IsParent>::~multi_runner_base()
+{
+    if (IsParent)
+    {
+        inner_->~inner();
+        boost::interprocess::shared_memory_object::remove(shared_mem_name_);
+        boost::interprocess::message_queue::remove(message_queue_name_);
+    }
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::checkout_test_index()
+{
+    return inner_->checkout_test_index();
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::checkout_job_index()
+{
+    return inner_->checkout_job_index();
+}
+
+template <bool IsParent>
+bool
+multi_runner_base<IsParent>::any_failed() const
+{
+    return inner_->any_failed();
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::any_failed(bool v)
+{
+    return inner_->any_failed(v);
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::add(results const& r)
+{
+    inner_->add(r);
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::inc_keep_alive_count()
+{
+    inner_->inc_keep_alive_count();
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::get_keep_alive_count()
+{
+    return inner_->get_keep_alive_count();
+}
+
+template <bool IsParent>
+template <class S>
+void
+multi_runner_base<IsParent>::print_results(S& s)
+{
+    inner_->print_results(s);
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::message_queue_send(std::string const& s)
+{
+    // Even though the message queue does _not_ live in shared memory, child
+    // processes (the only ones using "send" need to protect access with a mutex
+    // on the OSX platform (access does not appear to need to be protection on
+    // linux or windows). This is likely due to the different back end implementation
+    // of message queue in boost, though that has not been confirmed.
+    std::lock_guard<boost::interprocess::interprocess_mutex> l{inner_->m_};
+    message_queue_->send(s.c_str(), s.size(), /*priority*/ 0);
+}
+
+template <bool IsParent>
+constexpr const char* multi_runner_base<IsParent>::shared_mem_name_;
+template <bool IsParent>
+constexpr const char* multi_runner_base<IsParent>::message_queue_name_;
+
+}  // detail
+
+//------------------------------------------------------------------------------
+
+multi_runner_parent::multi_runner_parent()
+    : os_(std::cout)
+{
+    message_queue_thread_ = std::thread([this] {
+        std::vector<char> buf(1 << 20);
+        while (this->continue_message_queue_)
+        {
+            // let children know the parent is still alive
+            this->inc_keep_alive_count();
+            if (!this->message_queue_->get_num_msg())
+            {
+                // If a child does not see the keep alive count incremented,
+                // it will assume the parent has died. This sleep time needs
+                // to be small enough so the child will see increments from
+                // a live parent.
+                std::this_thread::sleep_for(std::chrono::milliseconds(100));
+                continue;
+            }
+            try
+            {
+                std::size_t recvd_size = 0;
+                unsigned int priority = 0;
+                this->message_queue_->receive(
+                    buf.data(), buf.size(), recvd_size, priority);
+                if (recvd_size)
+                {
+                    std::string s{buf.data(), recvd_size};
+                    this->os_ << s;
+                    this->os_.flush();
+                }
+            }
+            catch (...)
+            {
+                std::cerr << "Error reading unit test message queue.\n";
+                return;
+            }
+        }
+    });
+}
+
+multi_runner_parent::~multi_runner_parent()
+{
+    using namespace beast::unit_test;
+
+    continue_message_queue_ = false;
+    message_queue_thread_.join();
+
+    print_results(os_);
+}
+
+bool
+multi_runner_parent::any_failed() const
+{
+    return multi_runner_base<true>::any_failed();
+}
+
+//------------------------------------------------------------------------------
+
+multi_runner_child::multi_runner_child(
+    std::size_t num_jobs,
+    bool quiet,
+    bool print_log)
+    : job_index_{checkout_job_index()}
+    , num_jobs_{num_jobs}
+    , quiet_{quiet}
+    , print_log_{print_log}
+{
+    // incPort twice (2*jobIndex_) because some tests need two envs
+    for (std::size_t i = 0; i < 2 * job_index_; ++i)
+        test::incPorts();
+
+    if (num_jobs_ > 1)
+    {
+        keep_alive_thread_ = std::thread([this] {
+            std::size_t last_count = get_keep_alive_count();
+            while (this->continue_keep_alive_)
+            {
+                // Use a small sleep time so in the normal case the child
+                // process may shutdown quickly. However, to protect against
+                // false alarms, use a longer sleep time later on.
+                std::this_thread::sleep_for(std::chrono::milliseconds(500));
+                auto cur_count = this->get_keep_alive_count();
+                if (cur_count == last_count)
+                {
+                    // longer sleep time to protect against false alarms
+                    std::this_thread::sleep_for(std::chrono::seconds(2));
+                    cur_count = this->get_keep_alive_count();
+                    if (cur_count == last_count)
+                    {
+                        // assume parent process is no longer alive
+                        std::cerr << "multi_runner_child " << job_index_
+                                  << ": Assuming parent died, exiting.\n";
+                        std::exit(EXIT_FAILURE);
+                    }
+                }
+                last_count = cur_count;
+            }
+        });
+    }
+}
+
+multi_runner_child::~multi_runner_child()
+{
+    if (num_jobs_ > 1)
+    {
+        continue_keep_alive_ = false;
+        keep_alive_thread_.join();
+    }
+
+    add(results_);
+}
+
+void
+multi_runner_child::on_suite_begin(beast::unit_test::suite_info const& info)
+{
+    suite_results_ = detail::suite_results{info.full_name()};
+}
+
+void
+multi_runner_child::on_suite_end()
+{
+    results_.add(suite_results_);
+}
+
+void
+multi_runner_child::on_case_begin(std::string const& name)
+{
+    case_results_ = detail::case_results(name);
+
+    if (quiet_)
+        return;
+
+    std::stringstream s;
+    if (num_jobs_ > 1)
+        s << job_index_ << "> ";
+    s << suite_results_.name
+      << (case_results_.name.empty() ? "" : (" " + case_results_.name)) << '\n';
+    message_queue_send(s.str());
+}
+
+void
+multi_runner_child::on_case_end()
+{
+    suite_results_.add(case_results_);
+}
+
+void
+multi_runner_child::on_pass()
+{
+    ++case_results_.total;
+}
+
+void
+multi_runner_child::on_fail(std::string const& reason)
+{
+    ++case_results_.failed;
+    ++case_results_.total;
+    std::stringstream s;
+    if (num_jobs_ > 1)
+        s << job_index_ << "> ";
+    s << "#" << case_results_.total << " failed" << (reason.empty() ? "" : ": ")
+      << reason << '\n';
+    message_queue_send(s.str());
+}
+
+void
+multi_runner_child::on_log(std::string const& msg)
+{
+    if (!print_log_)
+        return;
+
+    std::stringstream s;
+    if (num_jobs_ > 1)
+        s << job_index_ << "> ";
+    s << msg;
+    message_queue_send(s.str());
+}
+
+namespace detail {
+template class multi_runner_base<true>;
+template class multi_runner_base<false>;
+}
+
+}  // unit_test
+}  // beast

src/test/unit_test/multi_runner.h

@@ -0,0 +1,350 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2017 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef TEST_UNIT_TEST_MULTI_RUNNER_H
+#define TEST_UNIT_TEST_MULTI_RUNNER_H
+
+#include <beast/include/beast/core/static_string.hpp>
+#include <beast/unit_test/global_suites.hpp>
+#include <beast/unit_test/runner.hpp>
+
+#include <boost/container/static_vector.hpp>
+#include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/interprocess/mapped_region.hpp>
+#include <boost/interprocess/shared_memory_object.hpp>
+#include <boost/interprocess/sync/interprocess_mutex.hpp>
+
+#include <atomic>
+#include <chrono>
+#include <numeric>
+#include <string>
+#include <thread>
+#include <unordered_set>
+#include <utility>
+
+namespace ripple {
+namespace test {
+
+namespace detail {
+
+using clock_type = std::chrono::steady_clock;
+
+struct case_results
+{
+    std::string name;
+    std::size_t total = 0;
+    std::size_t failed = 0;
+
+    explicit
+    case_results(std::string name_ = "")
+        : name(std::move(name_))
+    {
+    }
+};
+
+struct suite_results
+{
+    std::string name;
+    std::size_t cases = 0;
+    std::size_t total = 0;
+    std::size_t failed = 0;
+    typename clock_type::time_point start = clock_type::now();
+
+    explicit
+    suite_results(std::string name_ = "")
+        : name(std::move(name_))
+    {
+    }
+
+    void
+    add(case_results const& r);
+};
+
+struct results
+{
+    using static_string = beast::static_string<256>;
+    // results may be stored in shared memory. Use `static_string` to ensure
+    // pointers from different memory spaces do not co-mingle
+    using run_time = std::pair<static_string, typename clock_type::duration>;
+
+    enum { max_top = 10 };
+
+    std::size_t suites = 0;
+    std::size_t cases = 0;
+    std::size_t total = 0;
+    std::size_t failed = 0;
+    boost::container::static_vector<run_time, max_top> top;
+    typename clock_type::time_point start = clock_type::now();
+
+    void
+    add(suite_results const& r);
+
+    void
+    merge(results const& r);
+
+    template <class S>
+    void
+    print(S& s);
+};
+
+template <bool IsParent>
+class multi_runner_base
+{
+    // `inner` will be created in shared memory. This is one way
+    // multi_runner_parent and multi_runner_child object communicate. The other
+    // way they communicate is through message queues.
+    struct inner
+    {
+        std::atomic<std::size_t> job_index_{0};
+        std::atomic<std::size_t> test_index_{0};
+        std::atomic<bool> any_failed_{false};
+        // A parent process will periodically increment `keep_alive_`. The child
+        // processes will check if `keep_alive_` is being incremented. If it is
+        // not incremented for a sufficiently long time, the child will assume the
+        // parent process has died.
+        std::atomic<std::size_t> keep_alive_{0};
+
+        mutable boost::interprocess::interprocess_mutex m_;
+        detail::results results_;
+
+        std::size_t
+        checkout_job_index();
+
+        std::size_t
+        checkout_test_index();
+
+        bool
+        any_failed() const;
+
+        void
+        any_failed(bool v);
+
+        void
+        inc_keep_alive_count();
+
+        std::size_t
+        get_keep_alive_count();
+
+        void
+        add(results const& r);
+
+        template <class S>
+        void
+        print_results(S& s);
+    };
+
+    static constexpr const char* shared_mem_name_ = "RippledUnitTestSharedMem";
+    // name of the message queue a multi_runner_child will use to communicate with
+    // multi_runner_parent
+    static constexpr const char* message_queue_name_ = "RippledUnitTestMessageQueue";
+
+    // `inner_` will be created in shared memory
+    inner* inner_;
+    // shared memory to use for the `inner` member
+    boost::interprocess::shared_memory_object shared_mem_;
+    boost::interprocess::mapped_region region_;
+
+protected:
+    std::unique_ptr<boost::interprocess::message_queue> message_queue_;
+
+    void message_queue_send(std::string const& s);
+
+public:
+    multi_runner_base();
+    ~multi_runner_base();
+
+    std::size_t
+    checkout_test_index();
+
+    std::size_t
+    checkout_job_index();
+
+    void
+    any_failed(bool v);
+
+    void
+    add(results const& r);
+
+    void
+    inc_keep_alive_count();
+
+    std::size_t
+    get_keep_alive_count();
+
+    template <class S>
+    void
+    print_results(S& s);
+
+    bool
+    any_failed() const;
+};
+
+}  // detail
+
+//------------------------------------------------------------------------------
+
+/** Manager for children running unit tests
+ */
+class multi_runner_parent : private detail::multi_runner_base</*IsParent*/true>
+{
+private:
+    // message_queue_ is used to collect log messages from the children
+    std::ostream& os_;
+    std::atomic<bool> continue_message_queue_{true};
+    std::thread message_queue_thread_;
+
+public:
+    multi_runner_parent(multi_runner_parent const&) = delete;
+    multi_runner_parent&
+    operator=(multi_runner_parent const&) = delete;
+
+    multi_runner_parent();
+    ~multi_runner_parent();
+
+    bool
+    any_failed() const;
+};
+
+//------------------------------------------------------------------------------
+
+/** A class to run a subset of unit tests
+ */
+class multi_runner_child : public beast::unit_test::runner,
+                           private detail::multi_runner_base</*IsParent*/ false>
+{
+private:
+    std::size_t job_index_;
+    detail::results results_;
+    detail::suite_results suite_results_;
+    detail::case_results case_results_;
+    std::size_t num_jobs_{0};
+    bool quiet_{false};
+    bool print_log_{true};
+
+    std::atomic<bool> continue_keep_alive_{true};
+    std::thread keep_alive_thread_;
+
+public:
+    multi_runner_child(multi_runner_child const&) = delete;
+    multi_runner_child&
+    operator=(multi_runner_child const&) = delete;
+
+    multi_runner_child(std::size_t num_jobs, bool quiet, bool print_log);
+    ~multi_runner_child();
+
+    template <class Pred>
+    bool
+    run_multi(Pred pred);
+
+private:
+    virtual void
+    on_suite_begin(beast::unit_test::suite_info const& info) override;
+
+    virtual void
+    on_suite_end() override;
+
+    virtual void
+    on_case_begin(std::string const& name) override;
+
+    virtual void
+    on_case_end() override;
+
+    virtual void
+    on_pass() override;
+
+    virtual void
+    on_fail(std::string const& reason) override;
+
+    virtual void
+    on_log(std::string const& s) override;
+};
+
+//------------------------------------------------------------------------------
+
+template <class Pred>
+bool
+multi_runner_child::run_multi(Pred pred)
+{
+    auto const& suite = beast::unit_test::global_suites();
+    auto const num_tests = suite.size();
+    // actual order to run the tests. Use this to move longer running tests to
+    // the beginning to better take advantage of a multi process run.
+    std::vector<std::size_t> order(num_tests);
+    std::iota(order.begin(), order.end(), 0);
+    {
+        std::unordered_set<std::string> prioritize{
+            "ripple.app.Flow", "ripple.tx.Offer"};
+        std::vector<std::size_t> to_swap;
+        to_swap.reserve(prioritize.size());
+
+        size_t i = 0;
+        for (auto const& t : suite)
+        {
+            auto const full_name = t.full_name();
+            if (prioritize.count(full_name))
+            {
+                to_swap.push_back(i);
+                if (to_swap.size() == prioritize.size())
+                    break;
+            }
+            ++i;
+        }
+
+        for (std::size_t i = 0; i < to_swap.size(); ++i)
+            std::swap(order[to_swap[i]], order[i]);
+    }
+    bool failed = false;
+
+    auto get_test = [&]() -> beast::unit_test::suite_info const* {
+        auto const cur_test_index = checkout_test_index();
+        if (cur_test_index >= num_tests)
+            return nullptr;
+        auto iter = suite.begin();
+        std::advance(iter, order[cur_test_index]);
+        return &*iter;
+    };
+    while (auto t = get_test())
+    {
+        if (!pred(*t))
+            continue;
+        try
+        {
+            failed = run(*t) || failed;
+        }
+        catch (...)
+        {
+            if (num_jobs_ <= 1)
+                throw;  // a single process can die
+
+            // inform the parent
+            std::stringstream s;
+            s << job_index_ << ">  failed Unhandled exception in test.\n";
+            message_queue_send(s.str());
+            failed = true;
+        }
+    }
+    any_failed(failed);
+    return failed;
+}
+
+
+}  // unit_test
+}  // beast
+
+#endif

src/test/unity/app_test_unity1.cpp

@@ -34,3 +34,5 @@
 #include <test/app/OfferStream_test.cpp>
 #include <test/app/Offer_test.cpp>
 #include <test/app/OversizeMeta_test.cpp>
+
+#include <test/unit_test/multi_runner.cpp>