From abd3668b65c72ffcd65f38601ba71f6e9861dc94 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 28 Jul 2013 02:38:19 -0700 Subject: [PATCH] Upgrade UnitTest and provide JUnit XML output formatting --- modules/beast_core/beast_core.h | 6 +- .../beast_core/diagnostic/beast_UnitTest.cpp | 386 ++++++++++-------- .../beast_core/diagnostic/beast_UnitTest.h | 289 ++++++++----- .../diagnostic/beast_UnitTestUtilities.cpp | 145 ++++++- .../diagnostic/beast_UnitTestUtilities.h | 35 +- modules/beast_core/files/beast_File.cpp | 8 +- .../files/beast_RandomAccessFile.cpp | 2 +- modules/beast_core/json/beast_JSON.cpp | 2 +- modules/beast_core/maths/beast_Random.cpp | 2 +- .../streams/beast_MemoryInputStream.cpp | 2 +- modules/beast_core/text/beast_String.cpp | 12 +- modules/beast_core/text/beast_TextDiff.cpp | 2 +- .../beast_core/threads/beast_ChildProcess.cpp | 2 +- modules/beast_core/threads/beast_Thread.cpp | 24 +- .../zip/beast_GZIPCompressorOutputStream.cpp | 2 +- .../math/beast_UnsignedInteger.cpp | 2 +- modules/beast_db/keyvalue/beast_KeyvaDB.cpp | 2 +- 17 files changed, 603 insertions(+), 320 deletions(-) diff --git a/modules/beast_core/beast_core.h b/modules/beast_core/beast_core.h index 62be31d08b..dc89b66f5a 100644 --- a/modules/beast_core/beast_core.h +++ b/modules/beast_core/beast_core.h @@ -225,14 +225,13 @@ namespace beast #include "diagnostic/beast_SafeBool.h" #include "diagnostic/beast_Error.h" #include "diagnostic/beast_FPUFlags.h" -#include "diagnostic/beast_UnitTest.h" -#include "diagnostic/beast_UnitTestUtilities.h" #include "diagnostic/beast_Throw.h" #include "containers/beast_AbstractFifo.h" #include "containers/beast_Array.h" #include "containers/beast_ArrayAllocationBase.h" #include "containers/beast_DynamicObject.h" #include "containers/beast_ElementComparator.h" +#include "maths/beast_Random.h" #include "containers/beast_HashMap.h" #include "containers/beast_List.h" #include "containers/beast_LinkedListPointer.h" @@ -263,7 +262,6 @@ namespace beast #include "maths/beast_Interval.h" #include "maths/beast_MathsFunctions.h" #include "maths/beast_MurmurHash.h" -#include "maths/beast_Random.h" #include "maths/beast_Range.h" #include "memory/beast_ByteOrder.h" #include "memory/beast_HeapBlock.h" @@ -320,8 +318,10 @@ namespace beast #include "time/beast_PerformanceCounter.h" #include "time/beast_RelativeTime.h" #include "time/beast_Time.h" +#include "diagnostic/beast_UnitTest.h" #include "xml/beast_XmlDocument.h" #include "xml/beast_XmlElement.h" +#include "diagnostic/beast_UnitTestUtilities.h" #include "zip/beast_GZIPCompressorOutputStream.h" #include "zip/beast_GZIPDecompressorInputStream.h" #include "zip/beast_ZipFile.h" diff --git a/modules/beast_core/diagnostic/beast_UnitTest.cpp b/modules/beast_core/diagnostic/beast_UnitTest.cpp index 42ed690d34..0c541023ba 100644 --- a/modules/beast_core/diagnostic/beast_UnitTest.cpp +++ b/modules/beast_core/diagnostic/beast_UnitTest.cpp @@ -21,11 +21,11 @@ */ //============================================================================== -UnitTest::UnitTest (String const& name, - String const& group, +UnitTest::UnitTest (String const& className, + String const& packageName, When when) - : m_name (name) - , m_group (group) + : m_className (className) + , m_packageName (packageName) , m_when (when) , m_runner (nullptr) { @@ -37,6 +37,16 @@ UnitTest::~UnitTest() getAllTests().removeFirstMatchingValue (this); } +String const& UnitTest::getClassName() const noexcept +{ + return m_className; +} + +String const& UnitTest::getPackageName() const noexcept +{ + return m_packageName; +} + UnitTest::TestList& UnitTest::getAllTests() { static TestList s_tests; @@ -52,39 +62,53 @@ void UnitTest::shutdown() { } -void UnitTest::performTest (UnitTests* const runner) +ScopedPointer & UnitTest::run (UnitTests* const runner) { bassert (runner != nullptr); m_runner = runner; + m_suite = new Suite (m_className, m_packageName); + initialise(); - runTest(); + + try + { + runTest(); + } + catch (...) + { + failException (); + } + shutdown(); + + finishCase (); + + m_suite->secondsElapsed = RelativeTime ( + Time::getCurrentTime () - m_suite->whenStarted).inSeconds (); + + return m_suite; } -void UnitTest::logMessage (const String& message) +void UnitTest::logMessage (String const& message) { m_runner->logMessage (message); } -void UnitTest::beginTest (const String& testName) +void UnitTest::beginTestCase (String const& name) { - m_runner->beginNewTest (this, testName); + finishCase (); + + String s; + s << m_packageName << "/" << m_className << ": " << name; + logMessage (s); + + m_case = new Case (name, m_className); } -void UnitTest::pass () +void UnitTest::expect (bool trueCondition, String const& failureMessage) { - m_runner->addPass(); -} - -void UnitTest::fail (String const& failureMessage) -{ - m_runner->addFail (failureMessage); -} - -void UnitTest::expect (const bool result, const String& failureMessage) -{ - if (result) + if (trueCondition) { pass (); } @@ -94,12 +118,96 @@ void UnitTest::expect (const bool result, const String& failureMessage) } } +void UnitTest::unexpected (bool falseCondition, String const& failureMessage) +{ + if (! falseCondition) + { + pass (); + } + else + { + fail (failureMessage); + } +} + +void UnitTest::pass () +{ + // If this goes off it means you forgot to call beginTestCase()! + bassert (m_case != nullptr); + + Item item (true); + + m_case->items.add (item); +} + +void UnitTest::fail (String const& failureMessage) +{ + // If this goes off it means you forgot to call beginTestCase()! + bassert (m_case != nullptr); + + Item item (false, failureMessage); + + m_case->failures++; + int const caseNumber = m_case->items.add (item); + + String s; + s << "#" << String (caseNumber) << " failed: " << failureMessage; + logMessage (s); + + m_runner->onFailure (); +} + +void UnitTest::failException () +{ + Item item (false, "An exception was thrown"); + + if (m_case != nullptr) + { + m_case->failures++; + } + else + { + // This hack gives us a test case, to handle the condition where an + // exception was thrown before beginTestCase() was called. + // + beginTestCase ("Exception outside test case"); + } + + int const caseNumber = m_case->items.add (item); + + String s; + s << "#" << String (caseNumber) << " threw an exception "; + logMessage (s); + + m_runner->onFailure (); +} + +//------------------------------------------------------------------------------ + +void UnitTest::finishCase () +{ + if (m_case != nullptr) + { + // If this goes off it means you forgot to + // report any passing test case items! + // + bassert (m_case->items.size () > 0); + + m_case->secondsElapsed = RelativeTime ( + Time::getCurrentTime () - m_case->whenStarted).inSeconds (); + + m_suite->tests += m_case->items.size (); + m_suite->failures += m_case->failures; + + m_suite->cases.add (m_case.release ()); + } +} + //============================================================================== UnitTests::UnitTests() - : currentTest (nullptr), - assertOnFailure (true), - logPasses (false) + : m_assertOnFailure (false) + , m_currentTest (nullptr) { } @@ -109,106 +217,86 @@ UnitTests::~UnitTests() void UnitTests::setAssertOnFailure (bool shouldAssert) noexcept { - assertOnFailure = shouldAssert; + m_assertOnFailure = shouldAssert; } -void UnitTests::setPassesAreLogged (bool shouldDisplayPasses) noexcept +UnitTests::Results const& UnitTests::getResults () const noexcept { - logPasses = shouldDisplayPasses; -} - -int UnitTests::getNumResults() const noexcept -{ - return results.size(); -} - -const UnitTests::TestResult* UnitTests::getResult (int index) const noexcept -{ - return results [index]; + return *m_results; } bool UnitTests::anyTestsFailed () const noexcept { - for (int i = 0; i < results.size (); ++i) - { - if (results [i]->failures > 0) - return true; - } - - return false; + return m_results->failures > 0; } -void UnitTests::resultsUpdated() +void UnitTests::runTests (Array const& tests) { -} + m_results = new Results; -void UnitTests::runTest (UnitTest& test) -{ - try - { - test.performTest (this); - } - catch (std::exception& e) - { - String s; - s << "Got an exception: " << e.what (); - addFail (s); - } - catch (...) - { - addFail ("Got an unhandled exception"); - } -} - -void UnitTests::runTest (String const& name) -{ - results.clear(); - resultsUpdated(); - - UnitTest::TestList& tests (UnitTest::getAllTests ()); - - for (int i = 0; i < tests.size(); ++i) - { - UnitTest& test = *tests [i]; - - if (test.getGroup () == name && test.getWhen () == UnitTest::runAlways) - { - runTest (test); - } - else if (test.getName () == name) - { - runTest (test); - break; - } - - } -} - -void UnitTests::runAllTests () -{ - UnitTest::TestList& tests (UnitTest::getAllTests ()); - - results.clear(); - resultsUpdated(); - - for (int i = 0; i < tests.size(); ++i) + for (int i = 0; i < tests.size (); ++i) { if (shouldAbortTests()) break; - UnitTest& test = *tests [i]; - - if (test.getWhen () == UnitTest::runAlways) - runTest (test); + runTest (*tests [i]); } - endTest(); - + m_results->secondsElapsed = RelativeTime ( + Time::getCurrentTime () - m_results->whenStarted).inSeconds (); } -void UnitTests::logMessage (const String& message) +void UnitTests::runAllTests () { - Logger::writeToLog (message); + UnitTest::TestList const& allTests (UnitTest::getAllTests ()); + + Array tests; + + tests.ensureStorageAllocated (allTests.size ()); + + for (int i = 0; i < allTests.size(); ++i) + { + UnitTest* const test = allTests [i]; + + if (test->getWhen () == UnitTest::runAlways) + { + tests.add (test); + } + } + + runTests (tests); +} + +void UnitTests::runTestsByName (String const& name) +{ + UnitTest::TestList const& allTests (UnitTest::getAllTests ()); + + Array tests; + + tests.ensureStorageAllocated (allTests.size ()); + + for (int i = 0; i < allTests.size(); ++i) + { + UnitTest* const test = allTests [i]; + + if (test->getPackageName () == name && test->getWhen () == UnitTest::runAlways) + { + tests.add (test); + } + else if (test->getClassName () == name) + { + tests.add (test); + break; + } + } + + runTests (tests); +} + +void UnitTests::onFailure () +{ + // A failure occurred and the setting to assert on failures is turned on. + bassert (! m_assertOnFailure) } bool UnitTests::shouldAbortTests() @@ -216,89 +304,25 @@ bool UnitTests::shouldAbortTests() return false; } -void UnitTests::beginNewTest (UnitTest* const test, const String& subCategory) +void UnitTests::logMessage (const String& message) { - endTest(); - currentTest = test; - - TestResult* const r = new TestResult(); - results.add (r); - r->unitTestName = test->getGroup() + "::" + test->getName(); - r->subcategoryName = subCategory; - r->passes = 0; - r->failures = 0; - - logMessage ("Test '" + r->unitTestName + "': " + subCategory); - - resultsUpdated (); + Logger::writeToLog (message); } -void UnitTests::endTest() +void UnitTests::runTest (UnitTest& test) { - if (results.size() > 0) + try { - TestResult* const r = results.getLast(); + ScopedPointer suite (test.run (this).release ()); - if (r->failures > 0) - { - String m ("FAILED!! "); - m << r->failures << (r->failures == 1 ? " test" : " tests") - << " failed, out of a total of " << (r->passes + r->failures); + m_results->tests += suite->cases.size (); + m_results->failures += suite->failures; - logMessage (String::empty); - logMessage (m); - logMessage (String::empty); - } - else - { - //logMessage ("All tests completed successfully"); - } + m_results->suites.add (suite.release ()); + } + catch (...) + { + // Should never get here. + Throw (std::runtime_error ("unhandled exception during unit tests")); } } - -void UnitTests::addPass() -{ - { - const ScopedLock sl (results.getLock()); - - TestResult* const r = results.getLast(); - bassert (r != nullptr); // You need to call UnitTest::beginTest() before performing any tests! - - r->passes++; - - if (logPasses) - { - String message ("Test "); - message << (r->failures + r->passes) << " passed"; - logMessage (message); - } - } - - resultsUpdated(); -} - -void UnitTests::addFail (const String& failureMessage) -{ - { - const ScopedLock sl (results.getLock()); - - TestResult* const r = results.getLast(); - bassert (r != nullptr); // You need to call UnitTest::beginTest() before performing any tests! - - r->failures++; - - String message ("Failure, #"); - message << (r->failures + r->passes); - - if (failureMessage.isNotEmpty()) - message << ": " << failureMessage; - - r->messages.add (message); - - logMessage (message); - } - - resultsUpdated(); - - if (assertOnFailure) { bassertfalse; } -} diff --git a/modules/beast_core/diagnostic/beast_UnitTest.h b/modules/beast_core/diagnostic/beast_UnitTest.h index 43498f9c34..25c72c905f 100644 --- a/modules/beast_core/diagnostic/beast_UnitTest.h +++ b/modules/beast_core/diagnostic/beast_UnitTest.h @@ -24,8 +24,6 @@ #ifndef BEAST_UNITTEST_BEASTHEADER #define BEAST_UNITTEST_BEASTHEADER -#include "../text/beast_StringArray.h" -#include "../containers/beast_OwnedArray.h" class UnitTests; /** This is a base class for classes that perform a unit test. @@ -34,6 +32,17 @@ class UnitTests; @code + + + + + FIX THE EXAMPLE FOR THE NEW API! + + + + + + class MyTest : public UnitTest { public: @@ -41,12 +50,12 @@ class UnitTests; void runTest() { - beginTest ("Part 1"); + beginTestCase ("Part 1"); expect (myFoobar.doesSomething()); expect (myFoobar.doesSomethingElse()); - beginTest ("Part 2"); + beginTestCase ("Part 2"); expect (myOtherFoobar.doesSomething()); expect (myOtherFoobar.doesSomethingElse()); @@ -69,17 +78,93 @@ class UnitTests; class BEAST_API UnitTest : Uncopyable { public: + /** When the test should be run. + + Tests that run always will be incuded in all tests or in a group test. + Manual tests will only run when they are individually targeted. This + lets you leave out slow tests or peformance tests from the main test set. + */ enum When { runAlways, runManual }; + /** Describes a single test item. + + An item created for each call to the test functions, such as @ref expect + or @expectEquals. + */ + struct Item + { + explicit Item (bool passed_, String failureMessage_ = "") + : passed (passed_) + , failureMessage (failureMessage_) + { + } + + bool passed; + String failureMessage; + }; + + /** Describes a test case. + A test case represents a group of Item objects. + */ + struct Case + { + explicit Case (String const& name_, String const& className_) + : name (name_) + , className (className_) + , whenStarted (Time::getCurrentTime ()) + , secondsElapsed (0) + , failures (0) + { + } + + String name; + String className; + + Time whenStarted; + double secondsElapsed; + + int failures; + + Array items; + }; + + /** Contains the results of a test. + + One of these objects is instantiated each time UnitTest::beginTestCase() is called, and + it contains details of the number of subsequent UnitTest::expect() calls that are + made. + */ + struct Suite + { + Suite (String const& className_, String const& packageName_) + : className (className_) + , packageName (packageName_) + , whenStarted (Time::getCurrentTime ()) // hack for now + , secondsElapsed (0) + , tests (0) + , failures (0) + { + } + + String className; + String packageName; + Time whenStarted; + double secondsElapsed; + int tests; + int failures; + OwnedArray cases; + }; + /** The type of a list of tests. */ typedef Array TestList; - //============================================================================== + //-------------------------------------------------------------------------- + /** Creates a test with the given name, group, and run option. The group is used when you want to run all tests in a particular group @@ -88,16 +173,26 @@ public: test that takes a long time which you might not want to run every time you run all tests. */ - explicit UnitTest (String const& name, String const& group = "", When when = runAlways); + /* + suiteName: A name + className: The name of the class that the unit test exercises + packageName: A real or pseudo "namespace" describing the general area of + functionality to which the specified class belongs. + Examples: "network", "core", "ui" + A package name can appear in multiple testsuite instances. + */ + explicit UnitTest (String const& name, + String const& group = "", + When when = runAlways); /** Destructor. */ virtual ~UnitTest(); - /** Returns the name of the test. */ - const String& getName() const noexcept { return m_name; } + /** Returns the class name of the test. */ + const String& getClassName() const noexcept; - /** Returns the group of the test. */ - String const& getGroup () const noexcept { return m_group; } + /** Returns the package name of the test. */ + String const& getPackageName () const noexcept; /** Returns the run option of the test. */ When getWhen () const noexcept { return m_when; } @@ -106,12 +201,13 @@ public: You shouldn't need to call this method directly - use UnitTests::runTests() instead. */ - void performTest (UnitTests* runner); + ScopedPointer & run (UnitTests* runner); /** Returns the set of all UnitTest objects that currently exist. */ static TestList& getAllTests(); - //============================================================================== + //-------------------------------------------------------------------------- + /** You can optionally implement this method to set up your test. This method will be called before runTest(). */ @@ -124,27 +220,19 @@ public: /** Implement this method in your subclass to actually run your tests. - The content of your implementation should call beginTest() and expect() + The content of your implementation should call beginTestCase() and expect() to perform the tests. */ virtual void runTest() = 0; - //============================================================================== /** Tells the system that a new subsection of tests is beginning. This should be called from your runTest() method, and may be called as many times as you like, to demarcate different sets of tests. */ - void beginTest (const String& testName); + void beginTestCase (String const& name); - /** Passes a test. - */ - void pass (); + // beginTestCase () - /** Fails a test with the specified message. - */ - void fail (String const& failureMessage); - - //============================================================================== /** Checks that the result of a test is true, and logs this result. In your runTest() method, you should call this method for each condition that @@ -153,17 +241,25 @@ public: @code void runTest() { - beginTest ("basic tests"); + beginTestCase ("basic tests"); expect (x + y == 2); expect (getThing() == someThing); ...etc... } @endcode - If testResult is true, a pass is logged; if it's false, a failure is logged. + If Suite is true, a pass is logged; if it's false, a failure is logged. If the failure message is specified, it will be written to the log if the test fails. */ - void expect (bool testResult, const String& failureMessage = String::empty); + void expect (bool trueCondition, String const& failureMessage = String::empty); + + /** Checks that the result of a test is false, and logs this result. + + This is basically the opposite of expect(). + + @see expect + */ + void unexpected (bool falseCondition, String const& failureMessage = String::empty); /** Compares two values, and if they don't match, prints out a message containing the expected and actual result values. @@ -184,18 +280,32 @@ public: expect (result, failureMessage); } + /** Causes the test item to pass. */ + void pass (); + + /** Causes the test item to fail. */ + void fail (String const& failureMessage); + + /** Records an exception in the test item. */ + void failException (); + //============================================================================== /** Writes a message to the test log. This can only be called during your runTest() method. */ void logMessage (const String& message); +private: + void finishCase (); + private: //============================================================================== - String const m_name; - String const m_group; + String const m_className; + String const m_packageName; When const m_when; UnitTests* m_runner; + ScopedPointer m_suite; + ScopedPointer m_case; }; //============================================================================== @@ -213,80 +323,67 @@ private: class BEAST_API UnitTests : Uncopyable { public: - //============================================================================== + struct Results + { + Results () + : whenStarted (Time::getCurrentTime ()) + , tests (0) + , failures (0) + { + } + + Time whenStarted; + double secondsElapsed; + int tests; + int failures; + + OwnedArray suites; + }; + /** */ UnitTests(); /** Destructor. */ virtual ~UnitTests(); - /** Run the specified unit test. - - Subclasses can override this to do extra stuff. + /** Sets a flag to indicate whether an assertion should be triggered if a test fails. + This is true by default. */ - virtual void runTest (UnitTest& test); + void setAssertOnFailure (bool shouldAssert) noexcept; - /** Run a particular test or group. */ - void runTest (String const& name); + /** Retrieve the information on all the suites that were run. + This is overwritten every time new tests are run. + */ + Results const& getResults () const noexcept; + + /** Returns `true` if any test failed. */ + bool anyTestsFailed () const noexcept; + + //-------------------------------------------------------------------------- + + /** Runs the specified list of tests. + This is used internally and won't normally need to be called. + */ + void runTests (Array const& tests); /** Runs all the UnitTest objects that currently exist. This calls runTests() for all the objects listed in UnitTest::getAllTests(). */ void runAllTests (); - /** Sets a flag to indicate whether an assertion should be triggered if a test fails. - This is true by default. - */ - void setAssertOnFailure (bool shouldAssert) noexcept; - - /** Sets a flag to indicate whether successful tests should be logged. - By default, this is set to false, so that only failures will be displayed in the log. - */ - void setPassesAreLogged (bool shouldDisplayPasses) noexcept; - - //============================================================================== - /** Contains the results of a test. - - One of these objects is instantiated each time UnitTest::beginTest() is called, and - it contains details of the number of subsequent UnitTest::expect() calls that are - made. - */ - struct TestResult - { - /** The main name of this test (i.e. the name of the UnitTest object being run). */ - String unitTestName; - - /** The name of the current subcategory (i.e. the name that was set when UnitTest::beginTest() was called). */ - String subcategoryName; - - /** The number of UnitTest::expect() calls that succeeded. */ - int passes; - - /** The number of UnitTest::expect() calls that failed. */ - int failures; - - /** A list of messages describing the failed tests. */ - StringArray messages; - }; - - /** Returns the number of TestResult objects that have been performed. - @see getResult - */ - int getNumResults() const noexcept; - - /** Returns one of the TestResult objects that describes a test that has been run. - @see getNumResults - */ - const TestResult* getResult (int index) const noexcept; - - /** Returns `true` if any test failed. */ - bool anyTestsFailed () const noexcept; + /** Run a particular test or group. */ + void runTestsByName (String const& name); protected: - /** Called when the list of results changes. - You can override this to perform some sort of behaviour when results are added. + friend class UnitTest; + + /** Called on a failure. */ + void onFailure (); + + /** This can be overridden to let the runner know that it should abort the tests + as soon as possible, e.g. because the thread needs to stop. */ - virtual void resultsUpdated (); + virtual bool shouldAbortTests (); /** Logs a message about the current test progress. By default this just writes the message to the Logger class, but you could override @@ -294,25 +391,13 @@ protected: */ virtual void logMessage (String const& message); - /** This can be overridden to let the runner know that it should abort the tests - as soon as possible, e.g. because the thread needs to stop. - */ - virtual bool shouldAbortTests (); +private: + void runTest (UnitTest& test); private: - friend class UnitTest; - - void beginNewTest (UnitTest* test, const String& subCategory); - void endTest(); - - void addPass(); - void addFail (const String& failureMessage); - - UnitTest* currentTest; - String currentSubCategory; - OwnedArray results; - bool assertOnFailure; - bool logPasses; + bool m_assertOnFailure; + ScopedPointer m_results; + UnitTest* m_currentTest; }; #endif diff --git a/modules/beast_core/diagnostic/beast_UnitTestUtilities.cpp b/modules/beast_core/diagnostic/beast_UnitTestUtilities.cpp index 0a553cffa4..18d5825601 100644 --- a/modules/beast_core/diagnostic/beast_UnitTestUtilities.cpp +++ b/modules/beast_core/diagnostic/beast_UnitTestUtilities.cpp @@ -17,6 +17,149 @@ */ //============================================================================== +namespace UnitTestUtilities +{ + +JUnitXMLFormatter::JUnitXMLFormatter (UnitTests const& tests) + : m_tests (tests) + , m_currentTime (timeToString (Time::getCurrentTime ())) + , m_hostName (SystemStats::getComputerName ()) +{ +} + +// This is the closest thing to documentation on JUnit XML I could find: +// +// http://junitpdfreport.sourceforge.net/managedcontent/PdfTranslation +// +String JUnitXMLFormatter::createDocumentString () +{ + UnitTests::Results const& results (m_tests.getResults ()); + + ScopedPointer testsuites (new XmlElement ("testsuites")); + testsuites->setAttribute ("tests", String (results.tests)); + if (results.failures != 0) + testsuites->setAttribute ("failures", String (results.failures)); + testsuites->setAttribute ("time", secondsToString (results.secondsElapsed)); + + for (int i = 0; i < results.suites.size (); ++i) + { + UnitTest::Suite const& suite (*results.suites [i]); + + XmlElement* testsuite (new XmlElement ("testsuite"));; + testsuite->setAttribute ("name", suite.className); + testsuite->setAttribute ("tests", String (suite.tests)); + if (suite.failures != 0) + testsuite->setAttribute ("failures", String (suite.failures)); + testsuite->setAttribute ("time", secondsToString (suite.secondsElapsed)); + testsuite->setAttribute ("timestamp", timeToString (suite.whenStarted)); + testsuite->setAttribute ("hostname", m_hostName); + testsuite->setAttribute ("package", suite.packageName); + + testsuites->addChildElement (testsuite); + + for (int i = 0; i < suite.cases.size (); ++i) + { + UnitTest::Case const& testCase (*suite.cases [i]); + + XmlElement* testcase (new XmlElement ("testcase")); + testcase->setAttribute ("name", testCase.name); + testcase->setAttribute ("time", secondsToString (testCase.secondsElapsed)); + testcase->setAttribute ("classname", suite.className); + + testsuite->addChildElement (testcase); + + for (int i = 0; i < testCase.items.size (); ++i) + { + UnitTest::Item const& item (testCase.items.getUnchecked (i)); + + if (!item.passed) + { + XmlElement* failure (new XmlElement ("failure")); + + String s; + s << "#" << String (i+1) << " " << item.failureMessage; + failure->setAttribute ("message", s); + + testcase->addChildElement (failure); + } + } + } + } + + return testsuites->createDocument ( + //"https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-4.xsd", + "", + false, + true, + "UTF-8", + 999); +}; + +String JUnitXMLFormatter::timeToString (Time const& time) +{ + return time.toString (true, true, false, true); +} + +String JUnitXMLFormatter::secondsToString (double seconds) +{ + if (seconds < .01) + return String (seconds, 4); + else if (seconds < 1) + return String (seconds, 2); + else if (seconds < 10) + return String (seconds, 1); + else + return String (int (seconds)); +} + +//------------------------------------------------------------------------------ + +/** A unit test that always passes. + This can be useful to diagnose continuous integration systems. +*/ +class PassUnitTest : public UnitTest +{ +public: + PassUnitTest () : UnitTest ("Pass", "beast", runManual) + { + } + + void runTest () + { + beginTestCase ("pass"); + + pass (); + } +}; + +static PassUnitTest passUnitTest; + +//------------------------------------------------------------------------------ + +/** A unit test that always fails. + This can be useful to diagnose continuous integration systems. +*/ +class FailUnitTest : public UnitTest +{ +public: + FailUnitTest () : UnitTest ("Fail", "beast", runManual) + { + } + + void runTest () + { + beginTestCase ("pass"); + + fail ("Intentional failure"); + } +}; + +static FailUnitTest failUnitTest; + +} + +//------------------------------------------------------------------------------ + class UnitTestUtilitiesTests : public UnitTest { public: @@ -33,7 +176,7 @@ public: int const numberOfItems = 100; int64 const seedValue = 50; - beginTest ("Payload"); + beginTestCase ("Payload"); Payload p1 (maxBufferSize); Payload p2 (maxBufferSize); diff --git a/modules/beast_core/diagnostic/beast_UnitTestUtilities.h b/modules/beast_core/diagnostic/beast_UnitTestUtilities.h index b2fa7792c0..234e7ac097 100644 --- a/modules/beast_core/diagnostic/beast_UnitTestUtilities.h +++ b/modules/beast_core/diagnostic/beast_UnitTestUtilities.h @@ -20,8 +20,6 @@ #ifndef BEAST_UNITTESTUTILITIES_H_INCLUDED #define BEAST_UNITTESTUTILITIES_H_INCLUDED -#include "../maths/beast_Random.h" - namespace UnitTestUtilities { @@ -40,6 +38,8 @@ void repeatableShuffle (int const numberOfItems, T& arrayOfItems, int64 seedValu } } +//------------------------------------------------------------------------------ + /** A block of memory used for test data. */ struct Payload @@ -95,6 +95,37 @@ public: HeapBlock data; }; +//------------------------------------------------------------------------------ + +/** Format unit test results in JUnit XML format. + + The output can be used directly with the Jenkins CI server with + the appropriate JUnit plugin. + + JUnit FAQ: http://junit.sourceforge.net/doc/faq/faq.htm + + Jenkins Home: http://jenkins-ci.org/ + + @see UnitTest, UnitTests +*/ + +class JUnitXMLFormatter : Uncopyable +{ +public: + JUnitXMLFormatter (UnitTests const& tests); + + String createDocumentString (); + +private: + static String timeToString (Time const& time); + static String secondsToString (double seconds); + +private: + UnitTests const& m_tests; + String const m_currentTime; + String const m_hostName; +}; + } #endif diff --git a/modules/beast_core/files/beast_File.cpp b/modules/beast_core/files/beast_File.cpp index 55bc1cafbd..74c59eecd8 100644 --- a/modules/beast_core/files/beast_File.cpp +++ b/modules/beast_core/files/beast_File.cpp @@ -930,7 +930,7 @@ public: void runTest() { - beginTest ("Reading"); + beginTestCase ("Reading"); const File home (File::getSpecialLocation (File::userHomeDirectory)); const File temp (File::getSpecialLocation (File::tempDirectory)); @@ -967,7 +967,7 @@ public: expect (numRootsExisting > 0); } - beginTest ("Writing"); + beginTestCase ("Writing"); File demoFolder (temp.getChildFile ("Beast UnitTests Temp Folder.folder")); expect (demoFolder.deleteRecursively()); @@ -1053,7 +1053,7 @@ public: expect (tempFile.getSize() == 10); } - beginTest ("Memory-mapped files"); + beginTestCase ("Memory-mapped files"); { MemoryMappedFile mmf (tempFile, MemoryMappedFile::readOnly); @@ -1084,7 +1084,7 @@ public: expect (tempFile2.deleteFile()); } - beginTest ("More writing"); + beginTestCase ("More writing"); expect (tempFile.appendData ("abcdefghij", 10)); expect (tempFile.getSize() == 20); diff --git a/modules/beast_core/files/beast_RandomAccessFile.cpp b/modules/beast_core/files/beast_RandomAccessFile.cpp index 28c029cd8b..84814b56c1 100644 --- a/modules/beast_core/files/beast_RandomAccessFile.cpp +++ b/modules/beast_core/files/beast_RandomAccessFile.cpp @@ -218,7 +218,7 @@ public: int const seedValue = 50; - beginTest (String ("numRecords=") + String (numRecords)); + beginTestCase (String ("numRecords=") + String (numRecords)); // Calculate the path File const path (File::createTempFile ("RandomAccessFile")); diff --git a/modules/beast_core/json/beast_JSON.cpp b/modules/beast_core/json/beast_JSON.cpp index 216bdf7741..d80df86fcf 100644 --- a/modules/beast_core/json/beast_JSON.cpp +++ b/modules/beast_core/json/beast_JSON.cpp @@ -609,7 +609,7 @@ public: void runTest() { - beginTest ("JSON"); + beginTestCase ("JSON"); Random r; r.setSeedRandomly(); diff --git a/modules/beast_core/maths/beast_Random.cpp b/modules/beast_core/maths/beast_Random.cpp index 9f381e8479..97fdc0a42b 100644 --- a/modules/beast_core/maths/beast_Random.cpp +++ b/modules/beast_core/maths/beast_Random.cpp @@ -163,7 +163,7 @@ public: void runTest() { - beginTest ("Random"); + beginTestCase ("Random"); for (int j = 10; --j >= 0;) { diff --git a/modules/beast_core/streams/beast_MemoryInputStream.cpp b/modules/beast_core/streams/beast_MemoryInputStream.cpp index 2cdd4198a2..e0679c60f8 100644 --- a/modules/beast_core/streams/beast_MemoryInputStream.cpp +++ b/modules/beast_core/streams/beast_MemoryInputStream.cpp @@ -96,7 +96,7 @@ public: void runTest() { - beginTest ("Basics"); + beginTestCase ("Basics"); Random r; int randomInt = r.nextInt(); diff --git a/modules/beast_core/text/beast_String.cpp b/modules/beast_core/text/beast_String.cpp index 55ad6d5929..f151a24673 100644 --- a/modules/beast_core/text/beast_String.cpp +++ b/modules/beast_core/text/beast_String.cpp @@ -2130,7 +2130,7 @@ public: void runTest() { { - beginTest ("Basics"); + beginTestCase ("Basics"); expect (String().length() == 0); expect (String() == String::empty); @@ -2163,7 +2163,7 @@ public: } { - beginTest ("Operations"); + beginTestCase ("Operations"); String s ("012345678"); expect (s.hashCode() != 0); @@ -2208,7 +2208,7 @@ public: s2 += "xyz"; expect (s2 == "1234567890xyz"); - beginTest ("Numeric conversions"); + beginTestCase ("Numeric conversions"); expect (String::empty.getIntValue() == 0); expect (String::empty.getDoubleValue() == 0.0); expect (String::empty.getFloatValue() == 0.0f); @@ -2232,7 +2232,7 @@ public: expect (String::toHexString (data, 8, 1).equalsIgnoreCase ("01 02 03 04 0a 0b 0c 0d")); expect (String::toHexString (data, 8, 2).equalsIgnoreCase ("0102 0304 0a0b 0c0d")); - beginTest ("Subsections"); + beginTestCase ("Subsections"); String s3; s3 = "abcdeFGHIJ"; expect (s3.equalsIgnoreCase ("ABCdeFGhiJ")); @@ -2363,7 +2363,7 @@ public: } { - beginTest ("UTF conversions"); + beginTestCase ("UTF conversions"); TestUTFConversion ::test (*this); TestUTFConversion ::test (*this); @@ -2371,7 +2371,7 @@ public: } { - beginTest ("StringArray"); + beginTestCase ("StringArray"); StringArray s; s.addTokens ("4,3,2,1,0", ";,", "x"); diff --git a/modules/beast_core/text/beast_TextDiff.cpp b/modules/beast_core/text/beast_TextDiff.cpp index 6da5b587bd..8dfbce64bb 100644 --- a/modules/beast_core/text/beast_TextDiff.cpp +++ b/modules/beast_core/text/beast_TextDiff.cpp @@ -210,7 +210,7 @@ public: void runTest() { - beginTest ("TextDiff"); + beginTestCase ("TextDiff"); testDiff (String::empty, String::empty); testDiff ("x", String::empty); diff --git a/modules/beast_core/threads/beast_ChildProcess.cpp b/modules/beast_core/threads/beast_ChildProcess.cpp index 0c08aa82d4..24618f1e05 100644 --- a/modules/beast_core/threads/beast_ChildProcess.cpp +++ b/modules/beast_core/threads/beast_ChildProcess.cpp @@ -65,7 +65,7 @@ public: void runTest() { - beginTest ("Child Processes"); + beginTestCase ("Child Processes"); #if BEAST_WINDOWS || BEAST_MAC || BEAST_LINUX ChildProcess p; diff --git a/modules/beast_core/threads/beast_Thread.cpp b/modules/beast_core/threads/beast_Thread.cpp index 7685cfa00a..b413818d83 100644 --- a/modules/beast_core/threads/beast_Thread.cpp +++ b/modules/beast_core/threads/beast_Thread.cpp @@ -259,7 +259,7 @@ public: void runTest() { - beginTest ("Misc"); + beginTestCase ("Misc"); char a1[7]; expect (numElementsInArray(a1) == 7); @@ -270,28 +270,28 @@ public: expect (ByteOrder::swap ((uint32) 0x11223344) == 0x44332211); expect (ByteOrder::swap ((uint64) literal64bit (0x1122334455667788)) == literal64bit (0x8877665544332211)); - beginTest ("Atomic int"); + beginTestCase ("int"); AtomicTester ::testInteger (*this); - beginTest ("Atomic unsigned int"); + beginTestCase ("unsigned int"); AtomicTester ::testInteger (*this); - beginTest ("Atomic int32"); + beginTestCase ("int32"); AtomicTester ::testInteger (*this); - beginTest ("Atomic uint32"); + beginTestCase ("uint32"); AtomicTester ::testInteger (*this); - beginTest ("Atomic long"); + beginTestCase ("long"); AtomicTester ::testInteger (*this); - beginTest ("Atomic void*"); + beginTestCase ("void*"); AtomicTester ::testInteger (*this); - beginTest ("Atomic int*"); + beginTestCase ("int*"); AtomicTester ::testInteger (*this); - beginTest ("Atomic float"); + beginTestCase ("float"); AtomicTester ::testFloat (*this); #if ! BEAST_64BIT_ATOMICS_UNAVAILABLE // 64-bit intrinsics aren't available on some old platforms - beginTest ("Atomic int64"); + beginTestCase ("int64"); AtomicTester ::testInteger (*this); - beginTest ("Atomic uint64"); + beginTestCase ("uint64"); AtomicTester ::testInteger (*this); - beginTest ("Atomic double"); + beginTestCase ("double"); AtomicTester ::testFloat (*this); #endif } diff --git a/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp b/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp index 7d66da47a0..6cb71be235 100644 --- a/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp +++ b/modules/beast_core/zip/beast_GZIPCompressorOutputStream.cpp @@ -165,7 +165,7 @@ public: void runTest() { - beginTest ("GZIP"); + beginTestCase ("GZIP"); Random rng; for (int i = 100; --i >= 0;) diff --git a/modules/beast_crypto/math/beast_UnsignedInteger.cpp b/modules/beast_crypto/math/beast_UnsignedInteger.cpp index b9a2b5d18f..2315d43449 100644 --- a/modules/beast_crypto/math/beast_UnsignedInteger.cpp +++ b/modules/beast_crypto/math/beast_UnsignedInteger.cpp @@ -31,7 +31,7 @@ public: s << "Bytes=" << String(Bytes); - beginTest (s); + beginTestCase (s); UnsignedInteger zero; zero.fill (0); diff --git a/modules/beast_db/keyvalue/beast_KeyvaDB.cpp b/modules/beast_db/keyvalue/beast_KeyvaDB.cpp index 7867292d74..2fff06655d 100644 --- a/modules/beast_db/keyvalue/beast_KeyvaDB.cpp +++ b/modules/beast_db/keyvalue/beast_KeyvaDB.cpp @@ -760,7 +760,7 @@ public: String s; s << "keyBytes=" << String (uint64(KeyBytes)) << ", maxItems=" << String (maxItems); - beginTest (s); + beginTestCase (s); // Set up the key and value files File const path (File::createTempFile (""));