Improve reporting of unusual unit test failures:

* Jobs with no unit tests are counted as failures. Resolves #3474
* Crashed processes are counted as failures. Resolves #3600
* Any tests specified on the command line test do not have matching
  suites are counted as failures.
* Remove unused CI manual test.

bin/ci/ubuntu/build-and-test.sh

@@ -139,7 +139,6 @@ else
         'ripple.ripple_data.reduce_relay_simulate'
         'ripple.ripple_data.digest'
         'ripple.tx.Offer_manual'
-        'ripple.app.PayStrandAllPairs'
         'ripple.tx.CrossingLimits'
         'ripple.tx.PlumpBook'
         'ripple.app.Flow_manual'

src/ripple/app/main/Main.cpp

@@ -213,12 +213,40 @@ public:
                 return true;
         return false;
     }
+
+    std::size_t
+    size() const
+    {
+        return selectors_.size();
+    }
 };
 
 namespace test {
 extern std::atomic<bool> envUseIPv4;
 }
 
+template <class Runner>
+static bool
+anyMissing(Runner& runner, multi_selector const& pred)
+{
+    if (runner.tests() == 0)
+    {
+        runner.add_failures(1);
+        std::cout << "Failed: No tests run" << std::endl;
+        return true;
+    }
+    if (runner.suites() < pred.size())
+    {
+        auto const missing = pred.size() - runner.suites();
+        runner.add_failures(missing);
+        std::cout << "Failed: " << missing
+                  << " filters did not match any existing test suites"
+                  << std::endl;
+        return true;
+    }
+    return false;
+}
+
 static int
 runUnitTests(
     std::string const& pattern,
@@ -242,7 +270,9 @@ runUnitTests(
 
         multi_runner_child child_runner{num_jobs, quiet, log};
         child_runner.arg(argument);
-        auto const any_failed = child_runner.run_multi(multi_selector(pattern));
+        multi_selector pred(pattern);
+        auto const any_failed =
+            child_runner.run_multi(pred) || anyMissing(child_runner, pred);
 
         if (any_failed)
             return EXIT_FAILURE;
@@ -282,6 +312,9 @@ runUnitTests(
             }
         }
 
+        parent_runner.add_failures(bad_child_exits);
+        anyMissing(parent_runner, multi_selector(pattern));
+
         if (parent_runner.any_failed() || bad_child_exits)
             return EXIT_FAILURE;
         return EXIT_SUCCESS;

src/test/unit_test/multi_runner.cpp

@@ -182,6 +182,22 @@ multi_runner_base<IsParent>::inner::any_failed(bool v)
     any_failed_ = any_failed_ || v;
 }
 
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::inner::tests() const
+{
+    std::lock_guard l{m_};
+    return results_.total;
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::inner::suites() const
+{
+    std::lock_guard l{m_};
+    return results_.suites;
+}
+
 template <bool IsParent>
 void
 multi_runner_base<IsParent>::inner::inc_keep_alive_count()
@@ -348,6 +364,30 @@ multi_runner_base<IsParent>::message_queue_send(
     message_queue_->send(s.c_str(), s.size(), /*priority*/ 0);
 }
 
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::tests() const
+{
+    return inner_->tests();
+}
+
+template <bool IsParent>
+std::size_t
+multi_runner_base<IsParent>::suites() const
+{
+    return inner_->suites();
+}
+
+template <bool IsParent>
+void
+multi_runner_base<IsParent>::add_failures(std::size_t failures)
+{
+    results results;
+    results.failed += failures;
+    add(results);
+    any_failed(failures != 0);
+}
+
 template <bool IsParent>
 constexpr const char* multi_runner_base<IsParent>::shared_mem_name_;
 template <bool IsParent>
@@ -445,6 +485,24 @@ multi_runner_parent::any_failed() const
     return multi_runner_base<true>::any_failed();
 }
 
+std::size_t
+multi_runner_parent::tests() const
+{
+    return multi_runner_base<true>::tests();
+}
+
+std::size_t
+multi_runner_parent::suites() const
+{
+    return multi_runner_base<true>::suites();
+}
+
+void
+multi_runner_parent::add_failures(std::size_t failures)
+{
+    multi_runner_base<true>::add_failures(failures);
+}
+
 //------------------------------------------------------------------------------
 
 multi_runner_child::multi_runner_child(
@@ -501,6 +559,25 @@ multi_runner_child::~multi_runner_child()
     add(results_);
 }
 
+std::size_t
+multi_runner_child::tests() const
+{
+    return results_.total;
+}
+
+std::size_t
+multi_runner_child::suites() const
+{
+    return results_.suites;
+}
+
+void
+multi_runner_child::add_failures(std::size_t failures)
+{
+    results_.failed += failures;
+    any_failed(failures != 0);
+}
+
 void
 multi_runner_child::on_suite_begin(beast::unit_test::suite_info const& info)
 {

src/test/unit_test/multi_runner.h

@@ -131,6 +131,12 @@ class multi_runner_base
         void
         any_failed(bool v);
 
+        std::size_t
+        tests() const;
+
+        std::size_t
+        suites() const;
+
         void
         inc_keep_alive_count();
 
@@ -192,6 +198,15 @@ public:
 
     bool
     any_failed() const;
+
+    std::size_t
+    tests() const;
+
+    std::size_t
+    suites() const;
+
+    void
+    add_failures(std::size_t failures);
 };
 
 }  // namespace detail
@@ -220,6 +235,15 @@ public:
 
     bool
     any_failed() const;
+
+    std::size_t
+    tests() const;
+
+    std::size_t
+    suites() const;
+
+    void
+    add_failures(std::size_t failures);
 };
 
 //------------------------------------------------------------------------------
@@ -249,6 +273,15 @@ public:
     multi_runner_child(std::size_t num_jobs, bool quiet, bool print_log);
     ~multi_runner_child();
 
+    std::size_t
+    tests() const;
+
+    std::size_t
+    suites() const;
+
+    void
+    add_failures(std::size_t failures);
+
     template <class Pred>
     bool
     run_multi(Pred pred);