feat(test): add batch test selection to the test runner

lib/src/drogon_test.cc

@@ -1,6 +1,8 @@
 #include <drogon/drogon_test.h>
 
 #include <set>
+#include <unordered_map>
+#include <unordered_set>
 #include <future>
 #include <condition_variable>
 
@@ -61,10 +63,12 @@ static void printHelp(std::string_view argv0)
     print() << "A Drogon Test application:\n\n"
             << "Usage: " << argv0 << " [options]\n"
             << "options:\n"
-            << "    -r            Run a specific test\n"
+            << "    -r <tests...> Run one or more specific tests\n"
             << "    -s            Print successful tests\n"
             << "    -l            List available tests\n"
-            << "    -h | --help   Print this help message\n";
+            << "    -h | --help   Print this help message\n"
+            << "\n"
+            << "Example: " << argv0 << " -r $(cat selected-tests.txt)\n";
 }
 
 void printTestStats()
@@ -144,31 +148,38 @@ int run(int argc, char **argv)
     internal::numTestCases = 0;
     internal::printSuccessfulTests = false;
 
-    std::string targetTest;
+    std::vector<std::string> orderedTestNames;
+    std::unordered_set<std::string> uniqueTestNames;
     bool listTests = false;
     for (int i = 1; i < argc; i++)
     {
         const std::string param = argv[i];
         if (param == "-r")
         {
-            if (!targetTest.empty())
-            {
-                printErr() << "Only one test can be specified to run\n";
-                exit(1);
-            }
-            else if (i + 1 >= argc)
+            if (i + 1 >= argc || argv[i + 1][0] == '-')
             {
                 printErr() << "Missing test name after -r.\n";
-                exit(1);
+                return 1;
             }
 
-            targetTest = argv[i + 1];
-            i++;
+            while (i + 1 < argc && argv[i + 1][0] != '-')
+            {
+                const std::string testName = argv[++i];
+                if (uniqueTestNames.emplace(testName).second)
+                {
+                    orderedTestNames.emplace_back(std::move(testName));
+                }
+                else
+                {
+                    printErr() << "Duplicate test name: " << testName << "\n";
+                    return 1;
+                }
+            }
         }
         else if (param == "-h" || param == "--help")
         {
             printHelp(argv[0]);
-            exit(0);
+            return 0;
         }
         else if (param == "-s")
         {
@@ -182,7 +193,7 @@ int run(int argc, char **argv)
         {
             printErr() << "Unknown parameter: " << param << "\n";
             printHelp(argv[0]);
-            exit(1);
+            return 1;
         }
     }
     auto classNames = DrClassMap::getAllClassName();
@@ -202,42 +213,59 @@ int run(int argc, char **argv)
                 print() << "  " << ptr->name() << "\n";
             }
         }
-        exit(0);
+        return 0;
     }
 
-    std::vector<std::shared_ptr<TestCase>> testCases;
-    // NOTE: Registering a dummy case prevents the test-end signal to be
-    // emitted too early as there's always an case that hasn't finish
-    std::shared_ptr<Case> dummyCase = std::make_shared<Case>("__dummy_dummy_");
+    std::unordered_map<std::string, std::shared_ptr<TestCase>>
+        availableTestCases;
+    std::vector<std::string> availableTestNames;
     for (const auto &name : classNames)
     {
-        if (name.find(DROGON_TESTCASE_PREIX_STR_) == 0)
+        if (name.find(DROGON_TESTCASE_PREIX_STR_) != 0)
+            continue;
+
+        auto obj = std::shared_ptr<DrObjectBase>(DrClassMap::newObject(name));
+        auto test = std::dynamic_pointer_cast<TestCase>(obj);
+        if (test == nullptr)
         {
-            auto obj =
-                std::shared_ptr<DrObjectBase>(DrClassMap::newObject(name));
-            auto test = std::dynamic_pointer_cast<TestCase>(obj);
-            if (test == nullptr)
-            {
-                LOG_WARN << "Class " << name
-                         << " seems to be a test case. But type information "
-                            "disagrees.";
-                continue;
-            }
-            if (targetTest.empty() || test->name() == targetTest)
-            {
-                internal::numTestCases++;
-                test->doTest_(std::make_shared<Case>(test->name()));
-                testCases.emplace_back(std::move(test));
-            }
+            LOG_WARN << "Class " << name
+                     << " seems to be a test case. But type information "
+                        "disagrees.";
+            continue;
         }
+        const auto testName = test->name();
+        if (availableTestCases.emplace(testName, std::move(test)).second)
+            availableTestNames.emplace_back(testName);
+    }
+
+    std::vector<std::string> missingTestNames;
+    for (const auto &name : orderedTestNames)
+    {
+        if (availableTestCases.find(name) == availableTestCases.end())
+            missingTestNames.emplace_back(name);
+    }
+    if (!missingTestNames.empty())
+    {
+        printErr() << "Cannot find test(s) named:\n";
+        for (const auto &name : missingTestNames)
+            printErr() << "  " << name << "\n";
+        return 1;
     }
-    dummyCase = {};
 
-    if (targetTest != "" && internal::numTestCases == 0)
+    std::vector<std::shared_ptr<TestCase>> testCases;
+    // NOTE: Registering a dummy case prevents the test-end signal to be
+    // emitted too early as there's always an case that hasn't finish
+    std::shared_ptr<Case> dummyCase = std::make_shared<Case>("__dummy_dummy_");
+    const auto &testNames =
+        orderedTestNames.empty() ? availableTestNames : orderedTestNames;
+    for (const auto &name : testNames)
     {
-        printErr() << "Cannot find test named " << targetTest << "\n";
-        exit(1);
+        auto &test = availableTestCases.at(name);
+        internal::numTestCases++;
+        test->doTest_(std::make_shared<Case>(test->name()));
+        testCases.emplace_back(std::move(test));
     }
+    dummyCase = {};
 
     std::unique_lock<std::mutex> l(internal::mtxRegister);
     if (internal::registeredTests.empty() == false)

lib/tests/CMakeLists.txt

@@ -149,3 +149,9 @@ set_property(TARGET ${tests} PROPERTY CXX_EXTENSIONS OFF)
 ParseAndAddDrogonTests(unittest)
 ParseAndAddDrogonTests(cookie_same_site)
 ParseAndAddDrogonTests(real_ip_resolver)
+
+add_test(
+  NAME drogon_test_runner_cli
+  COMMAND ${CMAKE_COMMAND}
+          -DTEST_RUNNER=$<TARGET_FILE:unittest>
+          -P ${CMAKE_CURRENT_SOURCE_DIR}/DrogonTestRunnerCliTest.cmake)

lib/tests/DrogonTestRunnerCliTest.cmake

@@ -0,0 +1,92 @@
+function(run_test_runner result output)
+  execute_process(
+    COMMAND "${TEST_RUNNER}" ${ARGN}
+    RESULT_VARIABLE command_result
+    OUTPUT_VARIABLE command_output
+    ERROR_VARIABLE command_error)
+  set(${result} "${command_result}" PARENT_SCOPE)
+  set(${output} "${command_output}${command_error}" PARENT_SCOPE)
+endfunction()
+
+function(require_success description result output)
+  if(NOT result EQUAL 0)
+    message(FATAL_ERROR "${description} failed with exit code ${result}:\n${output}")
+  endif()
+endfunction()
+
+function(require_failure description result output)
+  if(result EQUAL 0)
+    message(FATAL_ERROR "${description} unexpectedly succeeded:\n${output}")
+  endif()
+endfunction()
+
+run_test_runner(result output -r TestFrameworkSelfTest)
+require_success("single test selection" "${result}" "${output}")
+if(NOT output MATCHES "1 tests cases")
+  message(FATAL_ERROR "single test selection changed unexpectedly:\n${output}")
+endif()
+
+run_test_runner(result output -s -r TestFrameworkSelfTest URLCodec)
+require_success("batch test selection" "${result}" "${output}")
+
+string(FIND "${output}" "In test case TestFrameworkSelfTest" first_test_position)
+string(FIND "${output}" "In test case URLCodec" second_test_position)
+if(first_test_position EQUAL -1 OR second_test_position EQUAL -1 OR
+   first_test_position GREATER second_test_position)
+  message(FATAL_ERROR "selected tests did not start in the requested order:\n${output}")
+endif()
+if(NOT output MATCHES "2 tests cases")
+  message(FATAL_ERROR "batch test selection changed unexpectedly:\n${output}")
+endif()
+
+run_test_runner(result output -r TestFrameworkSelfTest URLCodec TestFrameworkSelfTest)
+require_failure("duplicate test selection" "${result}" "${output}")
+if(NOT output MATCHES "Duplicate test name: TestFrameworkSelfTest")
+  message(FATAL_ERROR "duplicate test name was not reported:\n${output}")
+endif()
+if(output MATCHES "In test case")
+  message(FATAL_ERROR "a test started despite a duplicate requested name:\n${output}")
+endif()
+
+run_test_runner(result output -r missing-one TestFrameworkSelfTest missing-two)
+require_failure("unknown test selection" "${result}" "${output}")
+foreach(test_name missing-one missing-two)
+  if(NOT output MATCHES "${test_name}")
+    message(FATAL_ERROR "missing test name ${test_name} was not reported:\n${output}")
+  endif()
+endforeach()
+if(output MATCHES "In test case")
+  message(FATAL_ERROR "a test started despite an unknown requested name:\n${output}")
+endif()
+
+run_test_runner(result output -r)
+require_failure("missing name after -r" "${result}" "${output}")
+if(NOT output MATCHES "Missing test name after -r")
+  message(FATAL_ERROR "missing-name error was not reported:\n${output}")
+endif()
+
+run_test_runner(result output --unknown)
+require_failure("unknown option" "${result}" "${output}")
+if(NOT output MATCHES "Unknown parameter: --unknown")
+  message(FATAL_ERROR "unknown-option error was not reported:\n${output}")
+endif()
+
+run_test_runner(result output -l)
+require_success("test listing" "${result}" "${output}")
+if(NOT output MATCHES "Available Tests:" OR NOT output MATCHES "URLCodec")
+  message(FATAL_ERROR "test listing changed unexpectedly:\n${output}")
+endif()
+
+run_test_runner(result output --help)
+require_success("help output" "${result}" "${output}")
+string(FIND "${output}" "-r <tests...>" batch_syntax_position)
+string(FIND "${output}" "-r $(cat selected-tests.txt)" batch_example_position)
+if(batch_syntax_position EQUAL -1 OR batch_example_position EQUAL -1)
+  message(FATAL_ERROR "batch selection help was not reported:\n${output}")
+endif()
+
+run_test_runner(result output -h)
+require_success("short help output" "${result}" "${output}")
+if(NOT output MATCHES "A Drogon Test application")
+  message(FATAL_ERROR "short help output changed unexpectedly:\n${output}")
+endif()