CMakePresets: Add testPassthroughArguments to test presets

Add a testPassthroughArguments field to the execution object of test
presets, allowing users to forward arguments to test executables via
preset JSON. This extends the existing ctest -- CLI feature into the
preset system as schema version 12.

Preset arguments are assigned first, then CLI -- arguments are appended
after, so both sources combine naturally.

Fixes: #20470

Help/manual/cmake-presets.7.rst

@@ -979,6 +979,17 @@ that may contain the following fields:
       Equivalent to passing :option:`--no-tests=ignore <ctest --no-tests>`
       on the command line.
 
+.. _`CMakePresets test testPassthroughArguments`:
+
+  ``testPassthroughArguments``
+    An optional array of strings. Each element is forwarded as an argument
+    to every test executable, equivalent to passing arguments after
+    :option:`ctest --` on the command line.
+    This is allowed in preset files specifying version ``12`` or above.
+
+    When both preset and command-line passthrough arguments are specified,
+    preset arguments appear first, followed by command-line arguments.
+
 Package Preset
 ^^^^^^^^^^^^^^
 
@@ -1513,6 +1524,11 @@ they were added and a summary of the new features and changes is given below.
         regardless of whether it is inherited by another preset in a different
         directory.
 
+    * Changes to `Test Presets <Test Preset_>`_
+
+      * The `testPassthroughArguments <CMakePresets test testPassthroughArguments_>`_
+        field was added to forward arguments to test executables.
+
 Schema
 ======
 

Help/manual/presets/schema.json

@@ -193,7 +193,7 @@
         "vendor": { "$ref": "#/definitions/vendor" },
         "configurePresets": { "$ref": "#/definitions/configurePresetsV10" },
         "buildPresets": { "$ref": "#/definitions/buildPresetsV11" },
-        "testPresets": { "$ref": "#/definitions/testPresetsV11" },
+        "testPresets": { "$ref": "#/definitions/testPresetsV12" },
         "packagePresets": { "$ref": "#/definitions/packagePresetsV10" },
         "workflowPresets": { "$ref": "#/definitions/workflowPresetsV10" },
         "include": { "$ref": "#/definitions/include" }
@@ -1211,6 +1211,28 @@
         }
       }
     },
+    "testPresetsExecutionV12": {
+      "type": "object",
+      "description": "An optional object specifying options for test execution.",
+      "allOf": [
+        { "$ref": "#/definitions/testPresetsExecutionObject" },
+        { "$ref": "#/definitions/commentAsProperty" },
+        {
+          "properties": {
+            "repeat": { "$ref": "#/definitions/testPresetsExecutionRepeatV10" },
+            "jobs": { "$ref": "#/definitions/testPresetsExecutionJobsV11" },
+            "testPassthroughArguments": {
+              "type": "array",
+              "description": "An optional array of strings. Each element is forwarded as an argument to every test executable. Equivalent to passing arguments after -- on the ctest command line.",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        }
+      ],
+      "unevaluatedProperties": false
+    },
     "testPresetsExecutionV11": {
       "type": "object",
       "description": "An optional object specifying options for test execution.",
@@ -1413,6 +1435,22 @@
         }
       }
     },
+    "testPresetsAdditionalPropertiesV12": {
+      "properties": {
+        "execution": { "$ref": "#/definitions/testPresetsExecutionV12" },
+        "filter": {
+          "type": "object",
+          "description": "An optional object specifying how to filter the tests to run.",
+          "properties": {
+            "include": { "$ref": "#/definitions/testPresetsFilterIncludeV10" },
+            "exclude": { "$ref": "#/definitions/testPresetsFilterExcludeV10" },
+            "$comment": { "$ref": "#/definitions/$comment" }
+          }
+        },
+        "output": { "$ref": "#/definitions/testPresetsOutputV6" },
+        "condition": { "$ref": "#/definitions/topConditionV10" }
+      }
+    },
     "testPresetsAdditionalPropertiesV11": {
       "properties": {
         "execution": { "$ref": "#/definitions/testPresetsExecutionV11" },
@@ -1555,6 +1593,20 @@
         "name"
       ]
     },
+    "testPresetsV12": {
+      "type": "array",
+      "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 12 and higher.",
+      "items": {
+        "type": "object",
+        "description": "A test preset object",
+        "allOf": [
+          { "$ref": "#/definitions/testPresetsItemsV2" },
+          { "$ref": "#/definitions/testPresetsAdditionalPropertiesV12" },
+          { "$ref": "#/definitions/commentAsProperty" }
+        ],
+        "unevaluatedProperties": false
+      }
+    },
     "testPresetsV11": {
       "type": "array",
       "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 11 and higher.",

Help/release/dev/ctest-passthrough-args.rst

@@ -3,3 +3,8 @@ ctest-passthrough-args
 
 * :manual:`ctest(1)` gained support for forwarding extra arguments to test
   executables using a :option:`-- <ctest -->` separator.
+
+* :manual:`cmake-presets(7)` gained support for a
+  ``testPassthroughArguments`` field in the
+  test preset ``execution`` object for
+  forwarding arguments to test executables via presets.

Source/cmCMakePresetsErrors.cxx

@@ -189,6 +189,12 @@ void JOBS_PROC_UNSUPPORTED(cmJSONState* state)
                   "processor-count-based jobs preset support");
 }
 
+void PASSTHROUGH_ARGS_UNSUPPORTED(cmJSONState* state)
+{
+  state->AddError("File version must be 12 or higher for "
+                  "testPassthroughArguments preset support");
+}
+
 void CYCLIC_INCLUDE(std::string const& file, cmJSONState* state)
 {
   state->AddError(cmStrCat("Cyclic include among preset files: ", file));

Source/cmCMakePresetsErrors.h

@@ -74,6 +74,8 @@ void GRAPHVIZ_FILE_UNSUPPORTED(cmJSONState* state);
 
 void JOBS_PROC_UNSUPPORTED(cmJSONState* state);
 
+void PASSTHROUGH_ARGS_UNSUPPORTED(cmJSONState* state);
+
 void CYCLIC_INCLUDE(std::string const& file, cmJSONState* state);
 
 void TEST_OUTPUT_TRUNCATION_UNSUPPORTED(cmJSONState* state);

Source/cmCMakePresetsGraph.cxx

@@ -1008,6 +1008,8 @@ bool cmCMakePresetsGraph::TestPreset::VisitPresetInherit(
       InheritOptionalValue(execution.Timeout, parentExecution.Timeout);
       InheritOptionalValue(execution.NoTestsAction,
                            parentExecution.NoTestsAction);
+      InheritVector(execution.TestPassthroughArguments,
+                    parentExecution.TestPassthroughArguments);
     } else {
       preset.Execution = parent.Execution;
     }

Source/cmCMakePresetsGraph.h

@@ -299,6 +299,7 @@ public:
       cm::optional<bool> ScheduleRandom;
       cm::optional<int> Timeout;
       cm::optional<NoTestsActionEnum> NoTestsAction;
+      std::vector<std::string> TestPassthroughArguments;
     };
 
     std::string ConfigurePreset;

Source/cmCMakePresetsGraphReadJSON.cxx

@@ -719,6 +719,13 @@ bool cmCMakePresetsGraph::ReadJSONFile(std::string const& filename,
       return false;
     }
 
+    // Support for testPassthroughArguments added in version 12.
+    if (v < 12 && preset.Execution &&
+        !preset.Execution->TestPassthroughArguments.empty()) {
+      cmCMakePresetsErrors::PASSTHROUGH_ARGS_UNSUPPORTED(&this->parseState);
+      return false;
+    }
+
     this->TestPresetOrder.push_back(preset.Name);
   }
 

Source/cmCMakePresetsGraphReadJSONTestPresets.cxx

@@ -353,7 +353,10 @@ auto const TestPresetExecutionHelper =
       .Bind("timeout"_s, &TestPreset::ExecutionOptions::Timeout,
             cmCMakePresetsGraphInternal::PresetOptionalIntHelper, false)
       .Bind("noTestsAction"_s, &TestPreset::ExecutionOptions::NoTestsAction,
-            TestPresetOptionalExecutionNoTestsActionHelper, false));
+            TestPresetOptionalExecutionNoTestsActionHelper, false)
+      .Bind("testPassthroughArguments"_s,
+            &TestPreset::ExecutionOptions::TestPassthroughArguments,
+            cmCMakePresetsGraphInternal::PresetVectorStringHelper, false));
 
 auto const TestPresetFilterHelper =
   JSONHelperBuilder::Optional<TestPreset::FilterOptions>(

Source/cmCTest.cxx

@@ -1800,6 +1800,11 @@ bool cmCTest::SetArgsFromPreset(std::string const& presetName,
           return false;
       }
     }
+
+    // Assign passthrough arguments from preset.
+    // CLI -- args (parsed later in Run()) will be appended after these.
+    this->Impl->TestOptions.TestPassthroughArguments =
+      expandedPreset->Execution->TestPassthroughArguments;
   }
 
   return true;

Tests/RunCMake/CMakePresetsTest/Passthrough-test-basic-stdout.txt

@@ -0,0 +1,3 @@
+1: Test command: [^
+]* "-E" "echo" "base_output" "--extra-flag"
+.*

Tests/RunCMake/CMakePresetsTest/Passthrough-test-execOnly-stdout.txt

@@ -0,0 +1,3 @@
+1: Test command: [^
+]* "-E" "echo" "base_output" "--only-arg"
+.*

Tests/RunCMake/CMakePresetsTest/Passthrough-test-inherit-stdout.txt

@@ -0,0 +1,3 @@
+1: Test command: [^
+]* "-E" "echo" "base_output" "--parent-arg"
+.*

Tests/RunCMake/CMakePresetsTest/Passthrough.cmake

@@ -0,0 +1,2 @@
+enable_testing()
+add_test(echo_test ${CMAKE_COMMAND} -E echo base_output)

Tests/RunCMake/CMakePresetsTest/Passthrough.json.in

@@ -0,0 +1,47 @@
+{
+    "version": 12,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "basic",
+            "configurePreset": "default",
+            "output": {
+                "verbosity": "verbose"
+            },
+            "execution": {
+                "testPassthroughArguments": ["--extra-flag"]
+            }
+        },
+        {
+            "name": "parent",
+            "hidden": true,
+            "execution": {
+                "testPassthroughArguments": ["--parent-arg"]
+            }
+        },
+        {
+            "name": "inherit",
+            "inherits": "parent",
+            "configurePreset": "default",
+            "output": {
+                "verbosity": "verbose"
+            }
+        },
+        {
+            "name": "execOnly",
+            "configurePreset": "default",
+            "output": {
+                "verbosity": "verbose"
+            },
+            "execution": {
+                "testPassthroughArguments": ["--only-arg"]
+            }
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/PassthroughCombined-test-combined-stdout.txt

@@ -0,0 +1,3 @@
+1: Test command: [^
+]* "-E" "echo" "base_output" "--preset-arg" "--cli-arg"
+.*

Tests/RunCMake/CMakePresetsTest/PassthroughCombined.cmake

@@ -0,0 +1,2 @@
+enable_testing()
+add_test(echo_test ${CMAKE_COMMAND} -E echo base_output)

Tests/RunCMake/CMakePresetsTest/PassthroughCombined.json.in

@@ -0,0 +1,22 @@
+{
+    "version": 12,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "combined",
+            "configurePreset": "default",
+            "output": {
+                "verbosity": "verbose"
+            },
+            "execution": {
+                "testPassthroughArguments": ["--preset-arg"]
+            }
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/PassthroughUnsupported-test-x-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/PassthroughUnsupported-test-x-stderr.txt

@@ -0,0 +1,3 @@
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/PassthroughUnsupported:
+File version must be 12 or higher for testPassthroughArguments preset support$

Tests/RunCMake/CMakePresetsTest/PassthroughUnsupported.json.in

@@ -0,0 +1,19 @@
+{
+  "version": 11,
+  "configurePresets": [
+    {
+      "name": "default",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "default",
+      "configurePreset": "default",
+      "execution": {
+        "testPassthroughArguments": ["--extra"]
+      }
+    }
+  ]
+}

Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake

@@ -113,8 +113,13 @@ run_cmake_test_presets(TestOutputTruncationUnsupported "" "" "x")
 run_cmake_test_presets(OutputJUnitUnsupported "" "" "x")
 run_cmake_test_presets(InvalidJobs "" "" "x")
 run_cmake_test_presets(JobsProcUnsupported "" "" "x")
+run_cmake_test_presets(PassthroughUnsupported "" "" "x")
 set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_test_presets(ConfigurePresetUnreachable "" "" "x")
 set(CMakePresetsTest_NO_CONFIGURE 0)
 
+run_cmake_test_presets(Passthrough "default" "" "basic;inherit;execOnly")
+run_cmake_test_presets(PassthroughCombined "default" "" "combined" "--" "--cli-arg")
+set(CMakePresetsTest_NO_CONFIGURE 1)
+
 set(CMakePresetsTest_NO_BUILD 0)