From 4e81913645bf719e63b426b67142ab3ba59c1e82 Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Tue, 15 Sep 2026 15:59:34 -0400 Subject: [PATCH 1/6] Tests: Fix unset(ENV{...}) typos --- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 2 +- Tests/RunCMake/Make/RunCMakeTest.cmake | 2 +- Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 36d4fe4741..789231f8b3 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -328,7 +328,7 @@ run_Parallel(env-0) set(ENV{CTEST_PARALLEL_LEVEL} 3) run_Parallel(env-3) unset(ENV{CTEST_PARALLEL_LEVEL}) -unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING}) function(run_TestLoad name load) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad) diff --git a/Tests/RunCMake/Make/RunCMakeTest.cmake b/Tests/RunCMake/Make/RunCMakeTest.cmake index 28301994b0..8e9caee1e7 100644 --- a/Tests/RunCMake/Make/RunCMakeTest.cmake +++ b/Tests/RunCMake/Make/RunCMakeTest.cmake @@ -119,7 +119,7 @@ function(run_CTestJobServer) run_make_rule(CTestJobServer NoTests 2) run_make_rule(CTestJobServer Tests 2) run_make_rule(CTestJobServer Tests 3) - unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) + unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING}) endfunction() # Jobservers are currently only supported by GNU makes, except MSYS2 make diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 59cbb801ad..5c879f4d9c 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -126,7 +126,7 @@ set_property(TEST test6 PROPERTY DEPENDS test1) ]]) run_ctest_test(SerialOrder INCLUDE test) unset(CASE_CMAKELISTS_SUFFIX_CODE) -unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING}) set(CASE_CMAKELISTS_SUFFIX_CODE [[ add_test(NAME skip COMMAND ${CMAKE_COMMAND} -E true) @@ -164,7 +164,7 @@ set(ENV{CTEST_PARALLEL_LEVEL} 3) run_ctest_test(ParallelEnv3 INCLUDE test) unset(ENV{CTEST_PARALLEL_LEVEL}) unset(CASE_CMAKELISTS_SUFFIX_CODE) -unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING}) # Tests for the 'Test Load' feature of ctest # From 81b46d87baa1376e951f48cae99fc6ebb125a530 Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Tue, 15 Sep 2026 15:37:21 -0400 Subject: [PATCH 2/6] cmCTest: Factor out environment variable handling --- Source/cmCTest.cxx | 79 ++++++++++++++++++++++++++-------------------- Source/cmCTest.h | 7 ++++ 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index bd063f5c84..9b9bdf2bdf 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -369,6 +369,48 @@ void cmCTest::SetParallelLevel(cm::optional level) this->Impl->ParallelLevel = level; } +bool cmCTest::UpdateStateFromEnvironment() +{ + // handle CTEST_PARALLEL_LEVEL environment variable + if (!this->Impl->ParallelLevelSetInCli) { + if (cm::optional parallelEnv = + cmSystemTools::GetEnvVar("CTEST_PARALLEL_LEVEL")) { + if (parallelEnv->empty() || + parallelEnv->find_first_not_of(" \t") == std::string::npos) { + // An empty value tells ctest to choose a default. + this->SetParallelLevel(cm::nullopt); + } else { + // A non-empty value must be a non-negative integer. + // Otherwise, ignore it. + unsigned long plevel = 0; + if (cmStrToULong(*parallelEnv, &plevel)) { + this->SetParallelLevel(plevel); + } + } + } + } + + // handle CTEST_NO_TESTS_ACTION environment variable + if (!this->Impl->NoTestsModeSetInCli) { + std::string action; + if (cmSystemTools::GetEnv("CTEST_NO_TESTS_ACTION", action) && + !action.empty()) { + if (action == "error"_s) { + this->Impl->NoTestsMode = cmCTest::NoTests::Error; + } else if (action == "ignore"_s) { + this->Impl->NoTestsMode = cmCTest::NoTests::Ignore; + } else { + cmCTestLog(this, ERROR_MESSAGE, + "Unknown value for CTEST_NO_TESTS_ACTION: '" << action + << '\''); + return false; + } + } + } + + return true; +} + unsigned long cmCTest::GetTestLoad() const { return this->Impl->TestLoad; @@ -2596,41 +2638,8 @@ int cmCTest::Run(std::vector const& args) } } - // handle CTEST_PARALLEL_LEVEL environment variable - if (!this->Impl->ParallelLevelSetInCli) { - if (cm::optional parallelEnv = - cmSystemTools::GetEnvVar("CTEST_PARALLEL_LEVEL")) { - if (parallelEnv->empty() || - parallelEnv->find_first_not_of(" \t") == std::string::npos) { - // An empty value tells ctest to choose a default. - this->SetParallelLevel(cm::nullopt); - } else { - // A non-empty value must be a non-negative integer. - // Otherwise, ignore it. - unsigned long plevel = 0; - if (cmStrToULong(*parallelEnv, &plevel)) { - this->SetParallelLevel(plevel); - } - } - } - } - - // handle CTEST_NO_TESTS_ACTION environment variable - if (!this->Impl->NoTestsModeSetInCli) { - std::string action; - if (cmSystemTools::GetEnv("CTEST_NO_TESTS_ACTION", action) && - !action.empty()) { - if (action == "error"_s) { - this->Impl->NoTestsMode = cmCTest::NoTests::Error; - } else if (action == "ignore"_s) { - this->Impl->NoTestsMode = cmCTest::NoTests::Ignore; - } else { - cmCTestLog(this, ERROR_MESSAGE, - "Unknown value for CTEST_NO_TESTS_ACTION: '" << action - << '\''); - return 1; - } - } + if (!this->UpdateStateFromEnvironment()) { + return 1; } // Passthrough arguments (after --) are only supported in direct test diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 4227e0e2a0..3643444a3a 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -101,6 +101,13 @@ public: cm::optional GetParallelLevel() const; void SetParallelLevel(cm::optional level); + /** + * Check environment variables controlling CTest's behavior and update state + * accordingly, when the corresponding command-line toggles are unset. + * Returns false if there are errors during parsing. + */ + bool UpdateStateFromEnvironment(); + unsigned long GetTestLoad() const; void SetTestLoad(unsigned long); From 277236a99a1726a0db6458d9edc54a0c1fd16777 Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Wed, 9 Sep 2026 10:40:43 -0400 Subject: [PATCH 3/6] cmCTestTestCommand: Factor out preset resolution Prepare for a future commit which will reuse this functionality. Add a structure to hold the resolved fields so that we can more easily distinguish between errors during preset resolution and cases when presets simply aren't used. --- Source/CTest/cmCTestTestCommand.cxx | 94 +++++++++++++++++------------ Source/CTest/cmCTestTestCommand.h | 15 +++++ 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 3e8e319a38..e39fad34ea 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -28,47 +28,48 @@ using TestPreset = cmCMakePresetsGraph::TestPreset; -std::unique_ptr cmCTestTestCommand::InitializeHandler( - HandlerArguments& arguments, cmExecutionStatus& status) const +cm::optional +cmCTestTestCommand::ResolveTestPreset(cmMakefile& mf, + std::string const& presetArg, + std::string const& presetsFileArg, + cmExecutionStatus& status) const { - cmMakefile& mf = status.GetMakefile(); - auto& args = static_cast(arguments); - - std::string const sourceDirectory = - mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY"); + ResolvedTestPreset resolved; + resolved.SourceDirectory = mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY"); // Presets file is set according to the following priority order: // 1) The PRESETS_FILE option to ctest_test() // 2) CTEST_PRESETS_FILE script variable - std::string const rawPresetsFile = !args.PresetsFile.empty() - ? args.PresetsFile + std::string const rawPresetsFile = !presetsFileArg.empty() + ? presetsFileArg : mf.GetSafeDefinition("CTEST_PRESETS_FILE"); - std::string const presetsFile = rawPresetsFile.empty() + resolved.PresetsFile = rawPresetsFile.empty() ? "" - : cmSystemTools::CollapseFullPath(rawPresetsFile, sourceDirectory); + : cmSystemTools::CollapseFullPath(rawPresetsFile, + resolved.SourceDirectory); // Preset name is set according to the following priority order: // 1) The PRESET option to ctest_test() // 2) CTEST_TEST_PRESET script variable // 3) CTEST_PRESET script variable (a warning is emitted if no test preset // exists with this name) - std::string effectivePreset = !args.Preset.empty() ? args.Preset + resolved.EffectivePreset = !presetArg.empty() ? presetArg : cmNonempty(mf.GetDefinition("CTEST_TEST_PRESET")) ? *mf.GetDefinition("CTEST_TEST_PRESET") : ""; - if (effectivePreset.empty()) { + if (resolved.EffectivePreset.empty()) { cmValue v = mf.GetDefinition("CTEST_PRESET"); if (cmNonempty(v)) { std::string presetError; - auto presetCheck = - TestPresetExists(*v, sourceDirectory, presetsFile, presetError); + auto presetCheck = TestPresetExists(*v, resolved.SourceDirectory, + resolved.PresetsFile, presetError); if (presetCheck == PresetCheckResult::ReadError) { status.SetError(cmStrCat('\n', presetError)); - return nullptr; + return cm::nullopt; } if (presetCheck == PresetCheckResult::Found) { - effectivePreset = *v; + resolved.EffectivePreset = *v; } else { cmCTestLog(this->CTest, WARNING, "No test preset named \"" @@ -77,30 +78,45 @@ std::unique_ptr cmCTestTestCommand::InitializeHandler( } } - std::unique_ptr presetsGraph; - TestPreset const* expandedPreset = nullptr; - if (!effectivePreset.empty()) { - presetsGraph = cm::make_unique(); - if (!presetsGraph->ReadProjectPresets(sourceDirectory, presetsFile)) { - status.SetError(cmStrCat("\n Could not read presets from \"", - sourceDirectory, "\":\n ", - presetsGraph->parseState.GetErrorMessage())); - return nullptr; - } - - auto resolveResult = - presetsGraph->ResolvePreset(effectivePreset, presetsGraph->TestPresets); - auto resolveError = cmCMakePresetsGraph::FormatPresetError( - resolveResult.StatusCode, resolveResult.ErrorPresetName, - sourceDirectory); - if (resolveError) { - status.SetError(*resolveError); - return nullptr; - } - - expandedPreset = resolveResult.Preset; + if (resolved.EffectivePreset.empty()) { + return cm::optional(std::move(resolved)); } + resolved.PresetsGraph = cm::make_unique(); + if (!resolved.PresetsGraph->ReadProjectPresets(resolved.SourceDirectory, + resolved.PresetsFile)) { + status.SetError( + cmStrCat("\n Could not read presets from \"", resolved.SourceDirectory, + "\":\n ", resolved.PresetsGraph->parseState.GetErrorMessage())); + return cm::nullopt; + } + + auto resolveResult = resolved.PresetsGraph->ResolvePreset( + resolved.EffectivePreset, resolved.PresetsGraph->TestPresets); + auto resolveError = cmCMakePresetsGraph::FormatPresetError( + resolveResult.StatusCode, resolveResult.ErrorPresetName, + resolved.SourceDirectory); + if (resolveError) { + status.SetError(*resolveError); + return cm::nullopt; + } + resolved.ExpandedPreset = resolveResult.Preset; + + return cm::optional(std::move(resolved)); +} + +std::unique_ptr cmCTestTestCommand::InitializeHandler( + HandlerArguments& arguments, cmExecutionStatus& status) const +{ + cmMakefile& mf = status.GetMakefile(); + auto& args = static_cast(arguments); + auto resolvedPreset = + ResolveTestPreset(mf, args.Preset, args.PresetsFile, status); + if (!resolvedPreset) { + return nullptr; + } + TestPreset const* expandedPreset = resolvedPreset->ExpandedPreset; + cmValue ctestTimeout = mf.GetDefinition("CTEST_TEST_TIMEOUT"); cmDuration timeout; diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index d8782654cd..1e10ded9df 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -13,10 +13,12 @@ #include "cmArgumentParser.h" #include "cmArgumentParserTypes.h" +#include "cmCMakePresetsGraph.h" #include "cmCTestHandlerCommand.h" class cmExecutionStatus; class cmCTestTestHandler; +class cmMakefile; class cmCTestTestCommand : public cmCTestHandlerCommand { @@ -86,6 +88,19 @@ protected: private: std::string GetName() const override { return "ctest_test"; } + struct ResolvedTestPreset + { + std::string SourceDirectory; + std::string PresetsFile; + std::string EffectivePreset; + std::unique_ptr PresetsGraph; + cmCMakePresetsGraph::TestPreset const* ExpandedPreset = nullptr; + }; + + cm::optional ResolveTestPreset( + cmMakefile& mf, std::string const& presetArg, + std::string const& presetsFileArg, cmExecutionStatus& status) const; + virtual std::unique_ptr InitializeActualHandler( HandlerArguments& arguments, cmExecutionStatus& status) const; From f745dd11abef2c2d8f206362698bc6106791d4ca Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Wed, 9 Sep 2026 10:41:12 -0400 Subject: [PATCH 4/6] ctest: Respect test envvars set by CTEST_PRESET Note the special attention given to support CTest environment variables from inside a test preset's `environment` field. Fixes: #28079 --- Source/CTest/cmCTestTestCommand.cxx | 42 +++++++++- Source/CTest/cmCTestTestCommand.h | 20 +++++ .../CTestCommandLine/RunCMakeTest.cmake | 78 +++++++++++++++++++ ...esetCLIVarEnvironment-CMakePresets.json.in | 24 ++++++ .../TestPresetCLIVarEnvironment-stdout.txt | 1 + ...estsActionEnvironment-CMakePresets.json.in | 24 ++++++ ...tPresetNoTestsActionEnvironment-result.txt | 1 + ...tPresetNoTestsActionEnvironment-stderr.txt | 1 + ...allelLevelEnvironment-CMakePresets.json.in | 24 ++++++ ...tPresetParallelLevelEnvironment-stdout.txt | 6 ++ 10 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-CMakePresets.json.in create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-CMakePresets.json.in create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-CMakePresets.json.in create mode 100644 Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-stdout.txt diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index e39fad34ea..bbcc30c7ef 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -2,8 +2,10 @@ file LICENSE.rst or https://cmake.org/licensing for details. */ #include "cmCTestTestCommand.h" +#include #include #include +#include #include #include #include @@ -105,13 +107,49 @@ cmCTestTestCommand::ResolveTestPreset(cmMakefile& mf, return cm::optional(std::move(resolved)); } +bool cmCTestTestCommand::ExecuteHandlerCommand(TestArguments& args, + cmExecutionStatus& status) const +{ + cmMakefile& mf = status.GetMakefile(); + + // Resolve the preset once here (rather than letting InitializeHandler() + // resolve it again) so its Environment can be applied for the duration of + // the whole test run, without parsing the presets file (and emitting errors, + // etc.) more than once. + this->CachedPresetResolution.emplace( + ResolveTestPreset(mf, args.Preset, args.PresetsFile, status)); + + cmSystemTools::SaveRestoreEnvironment restoreEnv; + if (*this->CachedPresetResolution && + (*this->CachedPresetResolution)->ExpandedPreset) { + for (auto const& var : + (*this->CachedPresetResolution)->ExpandedPreset->Environment) { + if (var.second) { + cmSystemTools::PutEnv(cmStrCat(var.first, '=', *var.second)); + } + } + } + + // Special case for CTest environment variables specified in a preset, which + // are handled by cmCTest much earlier than here, and would otherwise be + // ignored. + if (!this->CTest->UpdateStateFromEnvironment()) { + return false; + } + + return cmCTestHandlerCommand::ExecuteHandlerCommand(args, status); +} + std::unique_ptr cmCTestTestCommand::InitializeHandler( HandlerArguments& arguments, cmExecutionStatus& status) const { cmMakefile& mf = status.GetMakefile(); auto& args = static_cast(arguments); - auto resolvedPreset = - ResolveTestPreset(mf, args.Preset, args.PresetsFile, status); + + assert(this->CachedPresetResolution); + cm::optional resolvedPreset = + std::move(*this->CachedPresetResolution); + this->CachedPresetResolution.reset(); if (!resolvedPreset) { return nullptr; } diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 1e10ded9df..c5cab859e3 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -25,6 +25,17 @@ class cmCTestTestCommand : public cmCTestHandlerCommand public: using cmCTestHandlerCommand::cmCTestHandlerCommand; + cmCTestTestCommand(cmCTestTestCommand const& other) + : cmCTestHandlerCommand(other) + { + } + + cmCTestTestCommand& operator=(cmCTestTestCommand const& other) + { + cmCTestHandlerCommand::operator=(other); + return *this; + } + protected: struct TestArguments : HandlerArguments { @@ -54,6 +65,9 @@ protected: std::string PresetsFile; }; + bool ExecuteHandlerCommand(TestArguments& args, + cmExecutionStatus& status) const; + template static auto MakeTestParser() -> cmArgumentParser { @@ -101,6 +115,12 @@ private: cmMakefile& mf, std::string const& presetArg, std::string const& presetsFileArg, cmExecutionStatus& status) const; + // Set by ExecuteHandlerCommand() (resolves the preset once, up + // front) and consumed by InitializeHandler() to avoid parsing the presets + // file (and emitting error messages, etc.) more than once. + mutable cm::optional> + CachedPresetResolution; + virtual std::unique_ptr InitializeActualHandler( HandlerArguments& arguments, cmExecutionStatus& status) const; diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 789231f8b3..25dde30212 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -869,6 +869,84 @@ block() -V) endblock() +# Environment variables from a test preset selected by -D CTEST_PRESET are +# applied when dashboard mode runs the Test step. +block() + set(src "${RunCMake_BINARY_DIR}/TestPresetCLIVarEnvironment") + set(bin "${RunCMake_BINARY_DIR}/TestPresetCLIVarEnvironment-build") + file(REMOVE_RECURSE "${src}" "${bin}") + file(MAKE_DIRECTORY "${src}" "${bin}") + configure_file("${RunCMake_SOURCE_DIR}/TestPresetCLIVarEnvironment-CMakePresets.json.in" + "${src}/CMakePresets.json" @ONLY) + file(WRITE "${bin}/DartConfiguration.tcl" + "BuildDirectory: ${bin}\n" + "SourceDirectory: ${src}\n") + file(WRITE "${bin}/CTestTestfile.cmake" " +add_test(print-env \"${CMAKE_COMMAND}\" -E environment) +set_tests_properties(print-env PROPERTIES PASS_REGULAR_EXPRESSION \"DASHBOARD_PRESET_ENV=from-preset\") +") + set(RunCMake_TEST_SOURCE_DIR "${src}") + set(RunCMake_TEST_BINARY_DIR "${bin}") + set(RunCMake_TEST_NO_CLEAN 1) + run_cmake_command(TestPresetCLIVarEnvironment + ${CMAKE_CTEST_COMMAND} + -M Experimental + -D "CTEST_PRESET=my-test-preset" + -T Test + -V) +endblock() + +# CTEST_PARALLEL_LEVEL set by a test preset's environment is respected. +block() + set(src "${RunCMake_BINARY_DIR}/TestPresetParallelLevelEnvironment") + set(bin "${RunCMake_BINARY_DIR}/TestPresetParallelLevelEnvironment-build") + file(REMOVE_RECURSE "${src}" "${bin}") + file(MAKE_DIRECTORY "${src}" "${bin}") + configure_file("${RunCMake_SOURCE_DIR}/TestPresetParallelLevelEnvironment-CMakePresets.json.in" + "${src}/CMakePresets.json" @ONLY) + file(WRITE "${bin}/DartConfiguration.tcl" + "BuildDirectory: ${bin}\n" + "SourceDirectory: ${src}\n") + file(WRITE "${bin}/CTestTestfile.cmake" " +foreach(i RANGE 1 6) + add_test(test\${i} \"${CMAKE_COMMAND}\" -E true) +endforeach() +") + set(RunCMake_TEST_SOURCE_DIR "${src}") + set(RunCMake_TEST_BINARY_DIR "${bin}") + set(RunCMake_TEST_NO_CLEAN 1) + # Spoof a number of processors to make these tests predictable. + set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) + run_cmake_command(TestPresetParallelLevelEnvironment + ${CMAKE_CTEST_COMMAND} + -M Experimental + -D "CTEST_PRESET=my-test-preset" + -T Test) + unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING}) +endblock() + +# CTEST_NO_TESTS_ACTION set by a test preset's environment is respected. +block() + set(src "${RunCMake_BINARY_DIR}/TestPresetNoTestsActionEnvironment") + set(bin "${RunCMake_BINARY_DIR}/TestPresetNoTestsActionEnvironment-build") + file(REMOVE_RECURSE "${src}" "${bin}") + file(MAKE_DIRECTORY "${src}" "${bin}") + configure_file("${RunCMake_SOURCE_DIR}/TestPresetNoTestsActionEnvironment-CMakePresets.json.in" + "${src}/CMakePresets.json" @ONLY) + file(WRITE "${bin}/DartConfiguration.tcl" + "BuildDirectory: ${bin}\n" + "SourceDirectory: ${src}\n") + file(WRITE "${bin}/CTestTestfile.cmake" "") + set(RunCMake_TEST_SOURCE_DIR "${src}") + set(RunCMake_TEST_BINARY_DIR "${bin}") + set(RunCMake_TEST_NO_CLEAN 1) + run_cmake_command(TestPresetNoTestsActionEnvironment + ${CMAKE_CTEST_COMMAND} + -M Experimental + -D "CTEST_PRESET=my-test-preset" + -T Test) +endblock() + # Test --output-junit function(run_output_junit) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/output-junit) diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-CMakePresets.json.in b/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-CMakePresets.json.in new file mode 100644 index 0000000000..6f2aa3e980 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-CMakePresets.json.in @@ -0,0 +1,24 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "my-configure-preset", + "generator": "@RunCMake_GENERATOR@", + "binaryDir": "${sourceDir}/build" + } + ], + "testPresets": [ + { + "name": "my-test-preset", + "configurePreset": "my-configure-preset", + "environment": { + "DASHBOARD_PRESET_ENV": "from-preset" + } + } + ] +} diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-stdout.txt new file mode 100644 index 0000000000..4b1b619265 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetCLIVarEnvironment-stdout.txt @@ -0,0 +1 @@ +DASHBOARD_PRESET_ENV=from-preset diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-CMakePresets.json.in b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-CMakePresets.json.in new file mode 100644 index 0000000000..002475602e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-CMakePresets.json.in @@ -0,0 +1,24 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "my-configure-preset", + "generator": "@RunCMake_GENERATOR@", + "binaryDir": "${sourceDir}/build" + } + ], + "testPresets": [ + { + "name": "my-test-preset", + "configurePreset": "my-configure-preset", + "environment": { + "CTEST_NO_TESTS_ACTION": "error" + } + } + ] +} diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-result.txt b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-result.txt new file mode 100644 index 0000000000..45a4fb75db --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-result.txt @@ -0,0 +1 @@ +8 diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-stderr.txt b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-stderr.txt new file mode 100644 index 0000000000..eafba1c692 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetNoTestsActionEnvironment-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-CMakePresets.json.in b/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-CMakePresets.json.in new file mode 100644 index 0000000000..bf4ed1edd9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-CMakePresets.json.in @@ -0,0 +1,24 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "my-configure-preset", + "generator": "@RunCMake_GENERATOR@", + "binaryDir": "${sourceDir}/build" + } + ], + "testPresets": [ + { + "name": "my-test-preset", + "configurePreset": "my-configure-preset", + "environment": { + "CTEST_PARALLEL_LEVEL": "3" + } + } + ] +} diff --git a/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-stdout.txt new file mode 100644 index 0000000000..3ec49479b4 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-stdout.txt @@ -0,0 +1,6 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/TestPresetParallelLevelEnvironment-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ \.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\. Passed +[0-9\.]+ sec From a235560e53039cdceda59243affadec9d4c5a066 Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Tue, 15 Sep 2026 15:49:45 -0400 Subject: [PATCH 5/6] Help: Clarify interaction between CTest parallelism toggles --- Help/command/ctest_test.rst | 7 ++++++ Help/envvar/CTEST_PARALLEL_LEVEL.rst | 23 ++++++++++++++++++-- Help/manual/ctest.1.rst | 3 ++- Help/manual/presets/execution-properties.rst | 5 +++++ Help/manual/presets/schema.yaml | 5 +++++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index e5a174cdbf..1697fa93ca 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -115,6 +115,13 @@ The options are: default level of parallelism, or unbounded parallelism, respectively, as documented by the :option:`ctest --parallel` option. + If given, this argument takes precedence over a test preset's + :preset:`testPresets.execution.jobs` field. Both this argument and that + field take precedence over the :envvar:`CTEST_PARALLEL_LEVEL` environment + variable, and over an explicit :option:`ctest --parallel` given on the + command line. See :envvar:`CTEST_PARALLEL_LEVEL` for the full precedence + order. + ``RESOURCE_SPEC_FILE `` .. versionadded:: 3.16 diff --git a/Help/envvar/CTEST_PARALLEL_LEVEL.rst b/Help/envvar/CTEST_PARALLEL_LEVEL.rst index ba8ae4f3b2..f1b38c9d06 100644 --- a/Help/envvar/CTEST_PARALLEL_LEVEL.rst +++ b/Help/envvar/CTEST_PARALLEL_LEVEL.rst @@ -5,12 +5,12 @@ CTEST_PARALLEL_LEVEL Specify the number of tests for CTest to run in parallel. For example, if ``CTEST_PARALLEL_LEVEL`` is set to 8, CTest will run -up to 8 tests concurrently as if ``ctest`` were invoked with the +up to 8 tests concurrently as if :manual:`ctest(1)` were invoked with the :option:`--parallel 8 ` option. .. versionchanged:: 3.29 - The value may be empty, or ``0``, to let ctest use a default level of + The value may be empty, or ``0``, to let CTest use a default level of parallelism, or unbounded parallelism, respectively, as documented by the :option:`ctest --parallel` option. @@ -18,4 +18,23 @@ up to 8 tests concurrently as if ``ctest`` were invoked with the In CMake 3.28 and earlier, an empty or ``0`` value was equivalent to ``1``. +This environment variable is ignored if :option:`ctest --parallel` is given on +the command line, and is overridden by the :preset:`testPresets.execution.jobs` +field of a test preset, if that field is set. + +A test preset may also set this variable itself, using its own +:preset:`testPresets.environment` field. Doing so is equivalent to setting +the variable in the calling process's environment, except that it takes +effect only for the preset's Test step, and only if ``execution.jobs`` is +not also set (which would take precedence, as noted above). + +When a test preset is used by a :command:`ctest_test` command in a +:ref:`CTest Script`, that command's own ``PARALLEL_LEVEL`` argument, if given, +takes precedence over the preset's ``execution.jobs`` field. Together, +``PARALLEL_LEVEL`` and ``execution.jobs`` take precedence not only over this +environment variable, but over an explicit :option:`ctest --parallel` on the +command line as well. This differs from a preset used directly via +:option:`ctest --preset`, where an explicit ``--parallel`` always wins over the +preset's ``execution.jobs`` field. + See :manual:`ctest(1)` for more information on parallel test execution. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index e807425489..88e93c9761 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -187,7 +187,8 @@ The options for running tests are: * Otherwise, if the value is ``0``, parallelism is unbounded. This option may instead be specified by the :envvar:`CTEST_PARALLEL_LEVEL` - environment variable. + environment variable. See the documentation of that variable for how it + interacts with a test preset's :preset:`testPresets.execution.jobs` field. This option can be used with the :prop_test:`PROCESSORS` test property. See the `Label and Subproject Summary`_. diff --git a/Help/manual/presets/execution-properties.rst b/Help/manual/presets/execution-properties.rst index 8e3d4a0f86..a743e566ed 100644 --- a/Help/manual/presets/execution-properties.rst +++ b/Help/manual/presets/execution-properties.rst @@ -20,6 +20,11 @@ :ctest-option:`--parallel` on the command line. If the value is ``0``, it is equivalent to unbounded parallelism. + If set, this field takes precedence over the + :envvar:`CTEST_PARALLEL_LEVEL` environment variable, + including when that variable is instead set via this test + preset's own :preset:`testPresets.environment` field. + .. presets-versionchanged:: 11 This field can also be a string, in which case it must be diff --git a/Help/manual/presets/schema.yaml b/Help/manual/presets/schema.yaml index 7650fa459f..9d1fa8dac6 100644 --- a/Help/manual/presets/schema.yaml +++ b/Help/manual/presets/schema.yaml @@ -1624,6 +1624,11 @@ properties: :ctest-option:`--parallel` on the command line. If the value is ``0``, it is equivalent to unbounded parallelism. + If set, this field takes precedence over the + :envvar:`CTEST_PARALLEL_LEVEL` environment variable, + including when that variable is instead set via this test + preset's own :preset:`testPresets.environment` field. + .. presets-versionchanged:: 11 This field can also be a string, in which case it must be From 27f28db197124a9da4033f0a30b87a758c5dca1f Mon Sep 17 00:00:00 2001 From: Tyler Yankee Date: Wed, 16 Sep 2026 09:30:49 -0400 Subject: [PATCH 6/6] Help: Clarify CTEST_NO_TESTS_ACTION support and interaction --- Help/command/ctest_test.rst | 7 +++++++ Help/envvar/CTEST_NO_TESTS_ACTION.rst | 15 +++++++++++++++ Help/manual/ctest.1.rst | 4 +++- Help/manual/presets/execution-properties.rst | 7 +++++++ Help/manual/presets/schema.yaml | 7 +++++++ 5 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 1697fa93ca..64b94944e5 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -212,6 +212,13 @@ The options are: See also the :variable:`CTEST_TEST_PRESET` and :variable:`CTEST_PRESET` variables. + In general, only preset fields which have equivalent options to + ``ctest_test`` are supported. An exception is the + :preset:`testPresets.environment`, which can hold a value for the + :envvar:`CTEST_NO_TESTS_ACTION` environment variable and will be respected, + despite ``ctest_test`` not having an equivalent option to the + :preset:`testPresets.execution.noTestsAction` presets field. + ``PRESETS_FILE `` .. versionadded:: 4.4 diff --git a/Help/envvar/CTEST_NO_TESTS_ACTION.rst b/Help/envvar/CTEST_NO_TESTS_ACTION.rst index c5f74a74c8..a3dfb36609 100644 --- a/Help/envvar/CTEST_NO_TESTS_ACTION.rst +++ b/Help/envvar/CTEST_NO_TESTS_ACTION.rst @@ -12,3 +12,18 @@ cases when there are no tests to run. Possible values are: ``error``, The :option:`--no-tests=\ ` option to :manual:`ctest ` overrides this environment variable if both are given. + +When :option:`ctest --preset` is used, this environment variable, if set, +takes precedence over the :preset:`testPresets.execution.noTestsAction` +field of the selected test preset. This includes the case where the +variable is instead set via the preset's own +:preset:`testPresets.environment` field, which is equivalent to setting it +in the calling process's environment. An explicit +:option:`--no-tests=\ ` on the command line +takes precedence over all of the above. + +This environment variable, including when set via a test preset's +``environment`` field, is also respected when that same preset is used by +a :command:`ctest_test` command in a :ref:`CTest Script`. However, in that +case, the preset's ``noTestsAction`` field itself has no effect, because +:command:`ctest_test` has no equivalent argument to apply it through. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 88e93c9761..845291e119 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -556,7 +556,9 @@ The options for running tests are: .. versionadded:: 3.26 This option can also be set by setting the :envvar:`CTEST_NO_TESTS_ACTION` - environment variable. + environment variable. See the documentation of that variable for how it + interacts with a test preset's :preset:`testPresets.execution.noTestsAction` + field. .. option:: --collect-instrumentation diff --git a/Help/manual/presets/execution-properties.rst b/Help/manual/presets/execution-properties.rst index a743e566ed..363637372c 100644 --- a/Help/manual/presets/execution-properties.rst +++ b/Help/manual/presets/execution-properties.rst @@ -106,6 +106,13 @@ Equivalent to passing :ctest-option:`--no-tests=ignore` on the command line. + If set, this field is overridden by the + :envvar:`CTEST_NO_TESTS_ACTION` environment variable, + including when that variable is instead set via this test + preset's own :preset:`testPresets.environment` field. + See :envvar:`CTEST_NO_TESTS_ACTION` for the full precedence + order. + .. _`CMakePresets.testPresets.execution.testPassthroughArguments`: ``testPassthroughArguments`` diff --git a/Help/manual/presets/schema.yaml b/Help/manual/presets/schema.yaml index 9d1fa8dac6..3476e3f0f2 100644 --- a/Help/manual/presets/schema.yaml +++ b/Help/manual/presets/schema.yaml @@ -1766,6 +1766,13 @@ properties: ``ignore`` Equivalent to passing :ctest-option:`--no-tests=ignore` on the command line. + + If set, this field is overridden by the + :envvar:`CTEST_NO_TESTS_ACTION` environment variable, + including when that variable is instead set via this test + preset's own :preset:`testPresets.environment` field. + See :envvar:`CTEST_NO_TESTS_ACTION` for the full precedence + order. testPassthroughArguments: since: 12 type: array