Improve reporting of unusual unit test failures:

* Jobs with no unit tests are counted as failures. Resolves #3474
* Crashed processes are counted as failures. Resolves #3600
* Any tests specified on the command line test do not have matching
  suites are counted as failures.
* Remove unused CI manual test.
This commit is contained in:
Edward Hennis
2020-09-02 18:27:34 -04:00
committed by Nik Bougalis
parent 1129110be3
commit dbde686a97
4 changed files with 144 additions and 2 deletions

View File

@@ -213,12 +213,40 @@ public:
return true;
return false;
}
std::size_t
size() const
{
return selectors_.size();
}
};
namespace test {
extern std::atomic<bool> envUseIPv4;
}
template <class Runner>
static bool
anyMissing(Runner& runner, multi_selector const& pred)
{
if (runner.tests() == 0)
{
runner.add_failures(1);
std::cout << "Failed: No tests run" << std::endl;
return true;
}
if (runner.suites() < pred.size())
{
auto const missing = pred.size() - runner.suites();
runner.add_failures(missing);
std::cout << "Failed: " << missing
<< " filters did not match any existing test suites"
<< std::endl;
return true;
}
return false;
}
static int
runUnitTests(
std::string const& pattern,
@@ -242,7 +270,9 @@ runUnitTests(
multi_runner_child child_runner{num_jobs, quiet, log};
child_runner.arg(argument);
auto const any_failed = child_runner.run_multi(multi_selector(pattern));
multi_selector pred(pattern);
auto const any_failed =
child_runner.run_multi(pred) || anyMissing(child_runner, pred);
if (any_failed)
return EXIT_FAILURE;
@@ -282,6 +312,9 @@ runUnitTests(
}
}
parent_runner.add_failures(bad_child_exits);
anyMissing(parent_runner, multi_selector(pattern));
if (parent_runner.any_failed() || bad_child_exits)
return EXIT_FAILURE;
return EXIT_SUCCESS;