ctest: Respect test envvars set by CTEST_PRESET

Note the special attention given to support CTest environment variables
from inside a test preset's `environment` field.

Fixes: #28079

Source/CTest/cmCTestTestCommand.cxx

@@ -2,8 +2,10 @@
    file LICENSE.rst or https://cmake.org/licensing for details.  */
 #include "cmCTestTestCommand.h"
 
+#include <cassert>
 #include <chrono>
 #include <cstdlib>
+#include <map>
 #include <ratio>
 #include <sstream>
 #include <string>
@@ -105,13 +107,49 @@ cmCTestTestCommand::ResolveTestPreset(cmMakefile& mf,
   return cm::optional<ResolvedTestPreset>(std::move(resolved));
 }
 
+bool cmCTestTestCommand::ExecuteHandlerCommand(TestArguments& args,
+                                               cmExecutionStatus& status) const
+{
+  cmMakefile& mf = status.GetMakefile();
+
+  // Resolve the preset once here (rather than letting InitializeHandler()
+  // resolve it again) so its Environment can be applied for the duration of
+  // the whole test run, without parsing the presets file (and emitting errors,
+  // etc.) more than once.
+  this->CachedPresetResolution.emplace(
+    ResolveTestPreset(mf, args.Preset, args.PresetsFile, status));
+
+  cmSystemTools::SaveRestoreEnvironment restoreEnv;
+  if (*this->CachedPresetResolution &&
+      (*this->CachedPresetResolution)->ExpandedPreset) {
+    for (auto const& var :
+         (*this->CachedPresetResolution)->ExpandedPreset->Environment) {
+      if (var.second) {
+        cmSystemTools::PutEnv(cmStrCat(var.first, '=', *var.second));
+      }
+    }
+  }
+
+  // Special case for CTest environment variables specified in a preset, which
+  // are handled by cmCTest much earlier than here, and would otherwise be
+  // ignored.
+  if (!this->CTest->UpdateStateFromEnvironment()) {
+    return false;
+  }
+
+  return cmCTestHandlerCommand::ExecuteHandlerCommand(args, status);
+}
+
 std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
   HandlerArguments& arguments, cmExecutionStatus& status) const
 {
   cmMakefile& mf = status.GetMakefile();
   auto& args = static_cast<TestArguments&>(arguments);
-  auto resolvedPreset =
-    ResolveTestPreset(mf, args.Preset, args.PresetsFile, status);
+
+  assert(this->CachedPresetResolution);
+  cm::optional<ResolvedTestPreset> resolvedPreset =
+    std::move(*this->CachedPresetResolution);
+  this->CachedPresetResolution.reset();
   if (!resolvedPreset) {
     return nullptr;
   }

Source/CTest/cmCTestTestCommand.h

@@ -25,6 +25,17 @@ class cmCTestTestCommand : public cmCTestHandlerCommand
 public:
   using cmCTestHandlerCommand::cmCTestHandlerCommand;
 
+  cmCTestTestCommand(cmCTestTestCommand const& other)
+    : cmCTestHandlerCommand(other)
+  {
+  }
+
+  cmCTestTestCommand& operator=(cmCTestTestCommand const& other)
+  {
+    cmCTestHandlerCommand::operator=(other);
+    return *this;
+  }
+
 protected:
   struct TestArguments : HandlerArguments
   {
@@ -54,6 +65,9 @@ protected:
     std::string PresetsFile;
   };
 
+  bool ExecuteHandlerCommand(TestArguments& args,
+                             cmExecutionStatus& status) const;
+
   template <typename Args>
   static auto MakeTestParser() -> cmArgumentParser<Args>
   {
@@ -101,6 +115,12 @@ private:
     cmMakefile& mf, std::string const& presetArg,
     std::string const& presetsFileArg, cmExecutionStatus& status) const;
 
+  // Set by ExecuteHandlerCommand() (resolves the preset once, up
+  // front) and consumed by InitializeHandler() to avoid parsing the presets
+  // file (and emitting error messages, etc.) more than once.
+  mutable cm::optional<cm::optional<ResolvedTestPreset>>
+    CachedPresetResolution;
+
   virtual std::unique_ptr<cmCTestTestHandler> InitializeActualHandler(
     HandlerArguments& arguments, cmExecutionStatus& status) const;
 

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -869,6 +869,84 @@ block()
     -V)
 endblock()
 
+# Environment variables from a test preset selected by -D CTEST_PRESET are
+# applied when dashboard mode runs the Test step.
+block()
+  set(src "${RunCMake_BINARY_DIR}/TestPresetCLIVarEnvironment")
+  set(bin "${RunCMake_BINARY_DIR}/TestPresetCLIVarEnvironment-build")
+  file(REMOVE_RECURSE "${src}" "${bin}")
+  file(MAKE_DIRECTORY "${src}" "${bin}")
+  configure_file("${RunCMake_SOURCE_DIR}/TestPresetCLIVarEnvironment-CMakePresets.json.in"
+                 "${src}/CMakePresets.json" @ONLY)
+  file(WRITE "${bin}/DartConfiguration.tcl"
+    "BuildDirectory: ${bin}\n"
+    "SourceDirectory: ${src}\n")
+  file(WRITE "${bin}/CTestTestfile.cmake" "
+add_test(print-env \"${CMAKE_COMMAND}\" -E environment)
+set_tests_properties(print-env PROPERTIES PASS_REGULAR_EXPRESSION \"DASHBOARD_PRESET_ENV=from-preset\")
+")
+  set(RunCMake_TEST_SOURCE_DIR "${src}")
+  set(RunCMake_TEST_BINARY_DIR "${bin}")
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(TestPresetCLIVarEnvironment
+    ${CMAKE_CTEST_COMMAND}
+    -M Experimental
+    -D "CTEST_PRESET=my-test-preset"
+    -T Test
+    -V)
+endblock()
+
+# CTEST_PARALLEL_LEVEL set by a test preset's environment is respected.
+block()
+  set(src "${RunCMake_BINARY_DIR}/TestPresetParallelLevelEnvironment")
+  set(bin "${RunCMake_BINARY_DIR}/TestPresetParallelLevelEnvironment-build")
+  file(REMOVE_RECURSE "${src}" "${bin}")
+  file(MAKE_DIRECTORY "${src}" "${bin}")
+  configure_file("${RunCMake_SOURCE_DIR}/TestPresetParallelLevelEnvironment-CMakePresets.json.in"
+                 "${src}/CMakePresets.json" @ONLY)
+  file(WRITE "${bin}/DartConfiguration.tcl"
+    "BuildDirectory: ${bin}\n"
+    "SourceDirectory: ${src}\n")
+  file(WRITE "${bin}/CTestTestfile.cmake" "
+foreach(i RANGE 1 6)
+  add_test(test\${i} \"${CMAKE_COMMAND}\" -E true)
+endforeach()
+")
+  set(RunCMake_TEST_SOURCE_DIR "${src}")
+  set(RunCMake_TEST_BINARY_DIR "${bin}")
+  set(RunCMake_TEST_NO_CLEAN 1)
+  # Spoof a number of processors to make these tests predictable.
+  set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1)
+  run_cmake_command(TestPresetParallelLevelEnvironment
+    ${CMAKE_CTEST_COMMAND}
+    -M Experimental
+    -D "CTEST_PRESET=my-test-preset"
+    -T Test)
+  unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING})
+endblock()
+
+# CTEST_NO_TESTS_ACTION set by a test preset's environment is respected.
+block()
+  set(src "${RunCMake_BINARY_DIR}/TestPresetNoTestsActionEnvironment")
+  set(bin "${RunCMake_BINARY_DIR}/TestPresetNoTestsActionEnvironment-build")
+  file(REMOVE_RECURSE "${src}" "${bin}")
+  file(MAKE_DIRECTORY "${src}" "${bin}")
+  configure_file("${RunCMake_SOURCE_DIR}/TestPresetNoTestsActionEnvironment-CMakePresets.json.in"
+                 "${src}/CMakePresets.json" @ONLY)
+  file(WRITE "${bin}/DartConfiguration.tcl"
+    "BuildDirectory: ${bin}\n"
+    "SourceDirectory: ${src}\n")
+  file(WRITE "${bin}/CTestTestfile.cmake" "")
+  set(RunCMake_TEST_SOURCE_DIR "${src}")
+  set(RunCMake_TEST_BINARY_DIR "${bin}")
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(TestPresetNoTestsActionEnvironment
+    ${CMAKE_CTEST_COMMAND}
+    -M Experimental
+    -D "CTEST_PRESET=my-test-preset"
+    -T Test)
+endblock()
+
 # Test --output-junit
 function(run_output_junit)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/output-junit)

Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-CMakePresets.json.in

@@ -0,0 +1,24 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "my-configure-preset",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "my-test-preset",
+      "configurePreset": "my-configure-preset",
+      "environment": {
+        "DASHBOARD_PRESET_ENV": "from-preset"
+      }
+    }
+  ]
+}

Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-stdout.txt

@@ -0,0 +1 @@
+DASHBOARD_PRESET_ENV=from-preset

Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-CMakePresets.json.in

@@ -0,0 +1,24 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "my-configure-preset",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "my-test-preset",
+      "configurePreset": "my-configure-preset",
+      "environment": {
+        "CTEST_NO_TESTS_ACTION": "error"
+      }
+    }
+  ]
+}

Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!

Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-CMakePresets.json.in

@@ -0,0 +1,24 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "my-configure-preset",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "my-test-preset",
+      "configurePreset": "my-configure-preset",
+      "environment": {
+        "CTEST_PARALLEL_LEVEL": "3"
+      }
+    }
+  ]
+}

Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-stdout.txt

@@ -0,0 +1,6 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-build
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+1/6 Test #[0-9]+: test[0-9]+ \.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.   Passed +[0-9\.]+ sec