diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 8f355644bc2..cf555a8ecc6 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -679,6 +679,13 @@ jobs: test -z "$(nm processexecutor.o)" # TODO: test NO_* defines + - name: Test testrunner inclusion/exclusion + run: | + ! ./testrunner -d TestUtils | grep -v TestUtils > /dev/null + ! ./testrunner -d TestUtils::trim | grep -v TestUtils::trim > /dev/null + ! ./testrunner -d -x TestUtils | grep TestUtils > /dev/null + ! ./testrunner -d -x TestUtils:trim | grep TestUtils:trim > /dev/null + - name: Show all ignored files if: false # TODO: currently lists all the contents of ignored folders - we only need what actually matched run: | diff --git a/test/fixture.cpp b/test/fixture.cpp index 00c844815fa..952cddd0330 100644 --- a/test/fixture.cpp +++ b/test/fixture.cpp @@ -97,19 +97,22 @@ bool TestFixture::prepareTest(const char testname[]) prepareTestInternal(); // Check if tests should be executed - if (testToRun.empty() || testToRun == testname) { - // Tests will be executed - prepare them - mTestname = testname; - ++countTests; - if (quiet_tests) { - std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr - std::fflush(stdout); - } else { - std::cout << classname << "::" << mTestname << std::endl; - } - return !dry_run; + if (!testsToRun.empty()) { + const bool match = testsToRun.count(testname); + if ((match && exclude_tests) || (!match && !exclude_tests)) + return false; + } + + // Tests will be executed - prepare them + mTestname = testname; + ++countTests; + if (quiet_tests) { + std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr + std::fflush(stdout); + } else { + std::cout << classname << "::" << mTestname << std::endl; } - return false; + return !dry_run; } void TestFixture::teardownTest() @@ -350,9 +353,9 @@ void TestFixture::printHelp() " -x Exclude the specified tests.\n"; } -void TestFixture::run(const std::string &str) +void TestFixture::run(const std::set &tests) { - testToRun = str; + testsToRun = tests; try { if (quiet_tests) { std::cout << '\n' << classname << ':'; @@ -380,6 +383,7 @@ void TestFixture::processOptions(const options& args) { quiet_tests = args.quiet(); dry_run = args.dry_run(); + exclude_tests = args.exclude_tests(); exename = args.exe(); } @@ -388,27 +392,27 @@ std::size_t TestFixture::runTests(const options& args) countTests = 0; errmsg.str(""); + const auto& which_tests = args.which_tests(); + const auto exclude_tests = args.exclude_tests(); + // TODO: bail out when given class/test is not found? - for (std::string classname : args.which_test()) { - std::string testname; - const std::string::size_type pos = classname.find("::"); - if (pos != std::string::npos) { - // TODO: excluding indiviual tests is not supported yet - testname = classname.substr(pos + 2); - classname.erase(pos); + for (TestInstance * test : TestRegistry::theInstance().tests()) + { + std::set tests; + if (!which_tests.empty()) { + const auto it = which_tests.find(test->classname); + const bool match = it != which_tests.cend(); + if (match && exclude_tests && it->second.empty()) // only bailout when the whole fixture is excluded + continue; + if (!match && !exclude_tests) + continue; + if (match) + tests = it->second; } - for (TestInstance * test : TestRegistry::theInstance().tests()) { - if (!classname.empty()) { - const bool match = test->classname == classname; - if ((match && args.exclude_tests()) || (!match && !args.exclude_tests())) - continue; - } - - TestFixture* fixture = test->create(); - fixture->processOptions(args); - fixture->run(testname); - } + TestFixture* fixture = test->create(); + fixture->processOptions(args); + fixture->run(tests); } if (args.summary() && !args.dry_run()) { diff --git a/test/fixture.h b/test/fixture.h index 35fb9ed6277..35239f0797c 100644 --- a/test/fixture.h +++ b/test/fixture.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -56,9 +57,10 @@ class TestFixture : public ErrorLogger { protected: std::string exename; - std::string testToRun; + std::set testsToRun; bool quiet_tests{}; bool dry_run{}; + bool exclude_tests{}; bool mNewTemplate{}; virtual void run() = 0; @@ -290,7 +292,7 @@ class TestFixture : public ErrorLogger { { (void) metric; } - void run(const std::string &str); + void run(const std::set &tests); public: static void printHelp(); diff --git a/test/options.cpp b/test/options.cpp index 17e44eeb6bf..658c4c76d38 100644 --- a/test/options.cpp +++ b/test/options.cpp @@ -17,23 +17,30 @@ #include "options.h" options::options(int argc, const char* const argv[]) - : mWhichTests(argv + 1, argv + argc) - ,mQuiet(mWhichTests.count("-q") != 0) - ,mHelp(mWhichTests.count("-h") != 0 || mWhichTests.count("--help")) - ,mSummary(mWhichTests.count("-n") == 0) - ,mDryRun(mWhichTests.count("-d") != 0) - ,mExcludeTests(mWhichTests.count("-x") != 0) + : mArgs(argv + 1, argv + argc) + ,mQuiet(mArgs.count("-q") != 0) + ,mHelp(mArgs.count("-h") != 0 || mArgs.count("--help")) + ,mSummary(mArgs.count("-n") == 0) + ,mDryRun(mArgs.count("-d") != 0) + ,mExcludeTests(mArgs.count("-x") != 0) ,mExe(argv[0]) { - for (auto it = mWhichTests.cbegin(); it != mWhichTests.cend();) { - if (!it->empty() && (((*it)[0] == '-') || (it->find("::") != std::string::npos && mWhichTests.count(it->substr(0, it->find("::")))))) - it = mWhichTests.erase(it); - else - ++it; - } - - if (mWhichTests.empty()) { - mWhichTests.insert(""); + for (const auto& arg : mArgs) { + if (arg.empty()) + continue; // empty argument + if (arg[0] == '-') + continue; // command-line switch + const auto pos = arg.find("::"); + if (pos == std::string::npos) { + mWhichTests[arg] = {}; // run whole fixture + continue; + } + const std::string fixture = arg.substr(0, pos); + const auto it = mWhichTests.find(fixture); + if (it != mWhichTests.cend() && it->second.empty()) + continue; // whole fixture is already included + const std::string test = arg.substr(pos+2); + mWhichTests[fixture].emplace(test); // run individual test } } @@ -57,7 +64,7 @@ bool options::dry_run() const return mDryRun; } -const std::set& options::which_test() const +const std::map>& options::which_tests() const { return mWhichTests; } diff --git a/test/options.h b/test/options.h index 5be6ca34e61..5bba8e70115 100644 --- a/test/options.h +++ b/test/options.h @@ -17,6 +17,7 @@ #ifndef OPTIONS_H #define OPTIONS_H +#include #include #include @@ -39,8 +40,8 @@ class options { bool dry_run() const; /** Exclude provided lists of tests. */ bool exclude_tests() const; - /** Which test should be run. Empty string means 'all tests' */ - const std::set& which_test() const; + /** Which tests should be run. */ + const std::map>& which_tests() const; const std::string& exe() const; @@ -49,7 +50,8 @@ class options { options& operator =(const options&) = delete; private: - std::set mWhichTests; + std::set mArgs; + std::map> mWhichTests; const bool mQuiet; const bool mHelp; const bool mSummary; diff --git a/test/testoptions.cpp b/test/testoptions.cpp index 7c72b2de336..6c290823a17 100644 --- a/test/testoptions.cpp +++ b/test/testoptions.cpp @@ -51,21 +51,28 @@ class TestOptions : public TestFixture { void which_test() const { const char* argv[] = {"./test_runner", "TestClass"}; options args(getArrayLength(argv), argv); - ASSERT(std::set {"TestClass"} == args.which_test()); + const std::map> expected{ + { "TestClass", {} } + }; + ASSERT(expected == args.which_tests()); } void which_test_method() const { const char* argv[] = {"./test_runner", "TestClass::TestMethod"}; options args(getArrayLength(argv), argv); - ASSERT(std::set {"TestClass::TestMethod"} == args.which_test()); + const std::map> expected{ + { "TestClass", {"TestMethod"} } + }; + ASSERT(expected == args.which_tests()); } void no_test_method() const { const char* argv[] = {"./test_runner"}; options args(getArrayLength(argv), argv); - ASSERT(std::set {""} == args.which_test()); + const std::map> expected{}; + ASSERT(expected == args.which_tests()); } @@ -105,22 +112,28 @@ class TestOptions : public TestFixture { void multiple_testcases() const { const char* argv[] = {"./test_runner", "TestClass::TestMethod", "TestClass::AnotherTestMethod"}; options args(getArrayLength(argv), argv); - std::set expected {"TestClass::TestMethod", "TestClass::AnotherTestMethod"}; - ASSERT(expected == args.which_test()); + const std::map> expected{ + { "TestClass", { "TestMethod", "AnotherTestMethod" } } + }; + ASSERT(expected == args.which_tests()); } void multiple_testcases_ignore_duplicates() const { const char* argv[] = {"./test_runner", "TestClass::TestMethod", "TestClass"}; options args(getArrayLength(argv), argv); - std::set expected {"TestClass"}; - ASSERT(expected == args.which_test()); + const std::map> expected{ + { "TestClass", {} } + }; + ASSERT(expected == args.which_tests()); } void invalid_switches() const { const char* argv[] = {"./test_runner", "TestClass::TestMethod", "-a", "-v", "-q"}; options args(getArrayLength(argv), argv); - std::set expected {"TestClass::TestMethod"}; - ASSERT(expected == args.which_test()); + const std::map> expected { + { "TestClass", { "TestMethod" } } + }; + ASSERT(expected == args.which_tests()); ASSERT_EQUALS(true, args.quiet()); }