ctest: Add a CoverageTool setting to enable integration with llvm-cov

Add a `CTEST_TEST_COVERAGE_TOOL` variable to tell CTest to integrate
with `llvm-cov` source-based code coverage by setting the
`LLVM_PROFILE_FILE` environment variable for each test.

Issue: #26932

Help/command/ctest_test.rst

@@ -183,6 +183,13 @@ The options are:
   file is **not** uploaded to CDash because it would be redundant with
   CTest's ``Test.xml`` file.
 
+``COVERAGE_TOOL <tool>``
+  .. versionadded:: 4.4
+
+  Specify the ``<tool>`` used for collecting coverage during the running
+  of the tests.  See the CTest :ref:`CoverageTool <ctest-CoverageTool>`
+  setting for details.
+
 ``QUIET``
   .. versionadded:: 3.3
 

Help/manual/cmake-variables.7.rst

@@ -806,6 +806,7 @@ Variables for CTest
    /variable/CTEST_SVN_UPDATE_OPTIONS
    /variable/CTEST_TEST_LOAD
    /variable/CTEST_TEST_TIMEOUT
+   /variable/CTEST_TEST_COVERAGE_TOOL
    /variable/CTEST_TLS_VERIFY
    /variable/CTEST_TLS_VERSION
    /variable/CTEST_UPDATE_COMMAND

Help/manual/ctest.1.rst

@@ -1362,8 +1362,41 @@ Configuration settings include:
   * `CTest Script`_ variable: :variable:`CTEST_TEST_TIMEOUT`
   * :module:`CTest` module variable: ``DART_TESTING_TIMEOUT``
 
+.. _`ctest-CoverageTool`:
+
+``CoverageTool``
+  .. versionadded:: 4.4
+
+  Specify the tool used for collecting coverage during the running
+  of the tests.  The tool may be one of:
+
+  ``LLVM-COV``
+    This value indicates the usage of Clang's source-based code coverage
+    which "operates on AST and preprocessor information directly".
+    For more information, see the `Clang Source-based Code Coverage`_
+    documentation.
+
+    CTest will run each test with the ``LLVM_PROFILE_FILE`` environment
+    variable set to ``<directory>/<test-name>_<process-id>.profraw``,
+    where ``<directory>`` is the absolute path to the directory where
+    the ``CTestTestfile.cmake`` that describes the test is located.
+    This configures Clang Source-based Code Coverage to uniquely identify
+    the coverage information gathered by each test without any collisions.
+
+    Any existing ``<directory>/<test-name>_*.profraw`` files for a test
+    are removed before running the test.  Any existing ``LLVM_PROFILE_FILE``
+    environment variable is ignored.
+
+  If no value is specified, no work is done and nothing is added to the
+  environment.
+
+  * `CTest Script`_ variable: :variable:`CTEST_TEST_COVERAGE_TOOL`
+  * :module:`CTest` module variable: :variable:`CTEST_TEST_COVERAGE_TOOL`
+
 To report extra test values to CDash, see :ref:`Additional Test Measurements`.
 
+.. _`Clang Source-based Code Coverage`: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+
 .. _`CTest Coverage Step`:
 
 CTest Coverage Step

Help/release/dev/ctest-llvm-cov.rst

@@ -0,0 +1,6 @@
+ctest-llvm-cov
+--------------
+
+* :manual:`ctest(1)` gained a :ref:`CoverageTool <ctest-CoverageTool>`
+  option in :ref:`Dashboard Client` mode to configure tests for
+  coverage collection by ``llvm-cov``.

Help/variable/CTEST_TEST_COVERAGE_TOOL.rst

@@ -0,0 +1,7 @@
+CTEST_TEST_COVERAGE_TOOL
+------------------------
+
+.. versionadded:: 4.4
+
+Specify the CTest :ref:`CoverageTool <ctest-CoverageTool>` setting
+in a :manual:`ctest(1)` :ref:`Dashboard Client` script.

Modules/DartConfiguration.tcl.in

@@ -107,3 +107,6 @@ CurlOptions: @CTEST_CURL_OPTIONS@
 # specify behavior for retrying the submission
 CTestSubmitRetryDelay: @CTEST_SUBMIT_RETRY_DELAY@
 CTestSubmitRetryCount: @CTEST_SUBMIT_RETRY_COUNT@
+
+# Invoke each test with environment variables configuring tool's collection.
+CTestTestCoverageTool: @CTEST_TEST_COVERAGE_TOOL@

Source/CTest/cmCTestRunTest.cxx

@@ -14,7 +14,10 @@
 
 #include <cm/memory>
 #include <cm/optional>
+#include <cm/string_view>
+#include <cmext/string_view>
 
+#include "cmsys/Glob.hxx"
 #include "cmsys/RegularExpression.hxx"
 
 #include "cmCTest.h"
@@ -874,6 +877,35 @@ bool cmCTestRunTest::ForkProcess()
     diff.ApplyTo(env);
   }
 
+  // Inject LLVM Profile name :
+  // https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program
+  if (this->CTest->GetCTestConfiguration("CTestTestCoverageTool") ==
+        "LLVM-COV"_s ||
+      this->TestHandler->TestOptions.CoverageTool == "LLVM-COV"_s) {
+
+    // Isolate the test from any ambient LLVM_PROFILE_FILE
+    env.UnPutEnv("LLVM_PROFILE_FILE");
+    // Value is <Test Dir>/<testName>_<processID>.profraw
+    std::string profileEnv =
+      cmStrCat(this->TestProperties->CTestDirectory, "/",
+               this->TestProperties->Name, "_%p.profraw"_s);
+    cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                       this->Index
+                         << ": Using environment variable LLVM_PROFILE_FILE="_s
+                         << profileEnv << " \n",
+                       this->TestHandler->GetQuiet());
+    env.PutEnv(cmStrCat("LLVM_PROFILE_FILE="_s, profileEnv));
+    // ProcessID -> * to allow for glob to find all
+    // files generated by the test
+    cmSystemTools::ReplaceString(profileEnv, "%p", "*");
+
+    cmsys::Glob glob;
+    glob.FindFiles(profileEnv);
+    for (std::string const& file : glob.GetFiles()) {
+      cmSystemTools::RemoveFile(file);
+    }
+  };
+
   if (this->UseAllocatedResources) {
     this->SetupResourcesEnvironment(env);
   } else {

Source/CTest/cmCTestTestCommand.cxx

@@ -96,6 +96,9 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
   if (!args.ResourceSpecFile.empty()) {
     handler->TestOptions.ResourceSpecFile = args.ResourceSpecFile;
   }
+  if (!args.CoverageTool.empty()) {
+    handler->TestOptions.CoverageTool = args.CoverageTool;
+  }
   if (!args.StopTime.empty()) {
     this->CTest->SetStopTime(args.StopTime);
   }

Source/CTest/cmCTestTestCommand.h

@@ -46,6 +46,7 @@ protected:
     std::string TestLoad;
     std::string ResourceSpecFile;
     std::string OutputJUnit;
+    std::string CoverageTool;
     bool StopOnFailure = false;
   };
 
@@ -74,6 +75,7 @@ protected:
       .Bind("TEST_LOAD"_s, &TestArguments::TestLoad)
       .Bind("RESOURCE_SPEC_FILE"_s, &TestArguments::ResourceSpecFile)
       .Bind("STOP_ON_FAILURE"_s, &TestArguments::StopOnFailure)
+      .Bind("COVERAGE_TOOL"_s, &TestArguments::CoverageTool)
       .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit);
   }
 

Source/CTest/cmCTestTestHandler.h

@@ -56,6 +56,8 @@ struct cmCTestTestOptions
   std::string ResourceSpecFile;
   std::string JUnitXMLFileName;
 
+  std::string CoverageTool;
+
   std::vector<std::string> TestPassthroughArguments;
 };
 

Source/cmCTest.cxx

@@ -3260,6 +3260,7 @@ void cmCTest::SetCMakeVariables(cmMakefile& mf)
 
   // CTest Test Step
   set("CTEST_TEST_TIMEOUT", "TimeOut");
+  set("CTEST_TEST_COVERAGE_TOOL", "CTestTestCoverageTool");
 
   // CTest Coverage Step
   set("CTEST_COVERAGE_COMMAND", "CoverageCommand");

Templates/CTestScript.cmake.in

@@ -48,6 +48,7 @@ set(CTEST_USE_LAUNCHERS "@CTEST_USE_LAUNCHERS@")
 set(CTEST_RESOURCE_SPEC_FILE "@CTEST_RESOURCE_SPEC_FILE@")
 set(CTEST_TEST_LOAD "@CTEST_TEST_LOAD@")
 set(CTEST_TEST_TIMEOUT "@DART_TESTING_TIMEOUT@")
+set(CTEST_TEST_COVERAGE_TOOL "@COVERAGE_TOOL@"
 
 # CTest Coverage Step
 set(CTEST_COVERAGE_COMMAND "@COVERAGE_COMMAND@")

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -260,6 +260,15 @@ set_property(TEST RunCMakeVersion PROPERTY ENVIRONMENT "ENV1=env1;ENV2=env2")
 endfunction()
 run_environment()
 
+# Test setting of CTEST_TEST_COVERAGE_TOOL
+function(run_CTestTestCoverageTool)
+  set(CASE_CTEST_TEST_ARGS "COVERAGE_TOOL" "LLVM-COV")
+  set(CASE_TEST_PREFIX_CODE [[ unset(ENV{LLVM_PROFILE_FILE})]])
+
+  run_ctest(TestCTestTestCoverageTool -VV)
+endfunction()
+run_CTestTestCoverageTool()
+
 # test for OUTPUT_JUNIT
 run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2)
 

Tests/RunCMake/ctest_test/TestCTestTestCoverageTool-stdout.txt

@@ -0,0 +1,3 @@
+1: Using environment variable LLVM_PROFILE_FILE=[^
+]*/Tests/RunCMake/ctest_test/[^
+]*_%p.profraw