CTest: make test result for insufficient resources configurable

Add the `RESOURCE_ERROR_ACTION` test property to configure the action in
case test resources cannot be satisfied. Currently it supports `FAIL`
(default) and `SKIP`.

Also add the `CTEST_RESOURCE_ERROR_ACTION` variable, which can be used
to set the default behavior for all tests.

Fixes: #28095

Auxiliary/vim/syntax/cmake.vim

@@ -381,6 +381,7 @@ syn keyword cmakeProperty contained
             \ RESOURCE
             \ RESOURCE_GROUPS
             \ RESOURCE_LOCK
+            \ RESOURCE_ERROR_ACTION
             \ RULE_LAUNCH_COMPILE
             \ RULE_LAUNCH_CUSTOM
             \ RULE_LAUNCH_LINK
@@ -2231,6 +2232,7 @@ syn keyword cmakeVariable contained
             \ CTEST_P4_OPTIONS
             \ CTEST_P4_UPDATE_OPTIONS
             \ CTEST_RESOURCE_SPEC_FILE
+            \ CTEST_RESOURCE_ERROR_ACTION
             \ CTEST_RUN_CURRENT_SCRIPT
             \ CTEST_SCP_COMMAND
             \ CTEST_SCRIPT_DIRECTORY

Help/manual/cmake-properties.7.rst

@@ -624,6 +624,7 @@ Properties on Tests
    /prop_test/PROCESSOR_AFFINITY
    /prop_test/PROCESSORS
    /prop_test/REQUIRED_FILES
+   /prop_test/RESOURCE_ERROR_ACTION
    /prop_test/RESOURCE_GROUPS
    /prop_test/RESOURCE_LOCK
    /prop_test/RUN_SERIAL

Help/manual/cmake-variables.7.rst

@@ -797,6 +797,7 @@ Variables for CTest
    /variable/CTEST_P4_UPDATE_OPTIONS
    /variable/CTEST_PRESET
    /variable/CTEST_PRESETS_FILE
+   /variable/CTEST_RESOURCE_ERROR_ACTION
    /variable/CTEST_RESOURCE_SPEC_FILE
    /variable/CTEST_RUN_CURRENT_SCRIPT
    /variable/CTEST_SCRIPT_DIRECTORY

Help/manual/ctest.1.rst

@@ -2067,6 +2067,10 @@ must have resource allocation, then it can return a failing exit code or use
 the :prop_test:`SKIP_RETURN_CODE` or :prop_test:`SKIP_REGULAR_EXPRESSION`
 properties to indicate a skipped test.
 
+The :prop_test:`RESOURCE_ERROR_ACTION` test property can be used to steer the
+behavior in case the resources requested cannot be satisfied. By default the
+test is marked as failed.
+
 .. _`ctest-resource-specification-file`:
 
 Resource Specification File

Help/prop_test/RESOURCE_ERROR_ACTION.rst

@@ -0,0 +1,13 @@
+RESOURCE_ERROR_ACTION
+---------------------
+
+.. versionadded:: 4.5
+
+This property defines the behavior in case the resource(s) requested via the
+:prop_test:`RESOURCE_GROUPS` property cannot be satisfied because the resource
+group is not available at all or not in sufficient amounts.
+
+If the property is unset or set to ``FAIL`` the test will be marked as failed.
+If the property is set to ``SKIP`` the test will be marked as skipped.
+
+This setting takes precedence over :variable:`CTEST_RESOURCE_ERROR_ACTION`.

Help/release/dev/ctest-resource-error-action.rst

@@ -0,0 +1,6 @@
+ctest-resource-error-action
+---------------------------
+
+* The :prop_test:`RESOURCE_ERROR_ACTION` test property and
+  :variable:`CTEST_RESOURCE_ERROR_ACTION` variable were added to
+  configure the test behavior in case of insufficient resources.

Help/variable/CTEST_RESOURCE_ERROR_ACTION.rst

@@ -0,0 +1,6 @@
+CTEST_RESOURCE_ERROR_ACTION
+---------------------------
+
+.. versionadded:: 4.5
+
+Default value for the :prop_test:`RESOURCE_ERROR_ACTION` test property.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -35,6 +35,7 @@
 #include "cmCTestBinPacker.h"
 #include "cmCTestRunTest.h"
 #include "cmCTestTestHandler.h"
+#include "cmCTestTypes.h"
 #include "cmInstrumentationInterrupt.h"
 #include "cmJSONState.h"
 #include "cmListFileCache.h"
@@ -322,8 +323,19 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
       e << "\n";
     }
     e << "Resource spec file:\n\n  " << this->ResourceSpecFile;
-    cmCTestRunTest::StartFailure(std::move(testRun), this->Total, e.str(),
-                                 "Insufficient resources");
+
+    // Handle insufficient resources
+    auto const errorAction =
+      this->Properties[test]->ResourceErrorAction.value_or(
+        this->TestHandler->TestOptions.ResourceErrorAction);
+
+    if (errorAction == cmCTestTypes::ResourceErrorAction::Skip) {
+      cmCTestRunTest::StartFailure(std::move(testRun), this->Total, "",
+                                   "SKIP_INSUFFICIENT_RESOURCES");
+    } else { // "FAIL"
+      cmCTestRunTest::StartFailure(std::move(testRun), this->Total, e.str(),
+                                   "Insufficient resources");
+    }
     return;
   }
 
@@ -1262,6 +1274,12 @@ static Json::Value DumpCTestProperties(
                         raw ? rawProperties["GENERATED_RESOURCE_SPEC_FILE"]
                             : testProperties.GeneratedResourceSpecFile));
   }
+  if (testProperties.ResourceErrorAction) {
+    properties.append(DumpCTestProperty(
+      "RESOURCE_ERROR_ACTION",
+      raw ? rawProperties["RESOURCE_ERROR_ACTION"]
+          : ResourceErrorActionToString(*testProperties.ResourceErrorAction)));
+  }
   if (!testProperties.Labels.empty()) {
     properties.append(DumpCTestProperty(
       "LABELS",

Source/CTest/cmCTestRunTest.cxx

@@ -240,6 +240,10 @@ cmCTestRunTest::EndTestResult cmCTestRunTest::EndTest(size_t completed,
                    this->TestResult.ExceptionStatus);
         this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
     }
+  } else if ("SKIP_INSUFFICIENT_RESOURCES" ==
+             this->TestResult.CompletionStatus) {
+    outputStream << "***Skipped (Resource error)";
+    skipped = true;
   } else if ("Disabled" == this->TestResult.CompletionStatus) {
     outputStream << "***Not Run (Disabled) ";
   } else // cmProcess::State::Error

Source/CTest/cmCTestTestHandler.cxx

@@ -1984,6 +1984,17 @@ bool cmCTestTestHandler::GetListOfTests()
   if (this->TestOptions.ResourceSpecFile.empty() && specFile) {
     this->TestOptions.ResourceSpecFile = *specFile;
   }
+  cmValue specErrorAction = mf.GetDefinition("CTEST_RESOURCE_ERROR_ACTION");
+  if (specErrorAction) {
+    auto action = cmCTestTypes::GetResourceErrorAction(*specErrorAction);
+    if (!action) {
+      cmCTestLog(this->CTest, ERROR_MESSAGE,
+                 "Invalid value for CTEST_RESOURCE_ERROR_ACTION: "
+                   << *specErrorAction << std::endl);
+    } else {
+      this->TestOptions.ResourceErrorAction = *action;
+    }
+  }
 
   if (!this->TestOptions.TestListFile.empty()) {
     this->TestsToRunByName =
@@ -2523,6 +2534,16 @@ bool cmCTestTestHandler::SetTestsProperties(
           } else if (key == "GENERATED_RESOURCE_SPEC_FILE"_s) {
             rt.RawProperties[key] = val;
             rt.GeneratedResourceSpecFile = val;
+          } else if (key == "RESOURCE_ERROR_ACTION"_s) {
+            rt.RawProperties[key] = val;
+            rt.ResourceErrorAction = cmCTestTypes::GetResourceErrorAction(val);
+            if (!rt.ResourceErrorAction) {
+              cmCTestLog(this->CTest, ERROR_MESSAGE,
+                         "Invalid value for CTEST_RESOURCE_ERROR_ACTION: "
+                           << val << std::endl);
+              return false;
+            }
+
           } else if (key == "SKIP_RETURN_CODE"_s) {
             rt.RawProperties[key] = val;
             rt.SkipReturnCode = atoi(val.c_str());

Source/CTest/cmCTestTestHandler.h

@@ -57,6 +57,8 @@ struct cmCTestTestOptions
   std::string TestListFile;
   std::string ExcludeTestListFile;
   std::string ResourceSpecFile;
+  cmCTestTypes::ResourceErrorAction ResourceErrorAction =
+    cmCTestTypes::ResourceErrorAction::Fail;
   std::string JUnitXMLFileName;
 
   std::string CoverageTool;
@@ -225,6 +227,7 @@ public:
     std::string ResourceGroupsRaw;
     std::string GeneratedResourceSpecFile;
     std::string GeneratedResourceSpecFileRaw;
+    cm::optional<cmCTestTypes::ResourceErrorAction> ResourceErrorAction;
     std::string BuildDepends;
     // Private test generator properties used to track backtraces
     cmListFileBacktrace Backtrace;

Source/CTest/cmCTestTypes.cxx

@@ -21,4 +21,26 @@ bool SetTruncationMode(TruncationMode& mode, cm::string_view str)
   return true;
 }
 
+cm::optional<ResourceErrorAction> GetResourceErrorAction(cm::string_view str)
+{
+  if (str == "FAIL") {
+    return cmCTestTypes::ResourceErrorAction::Fail;
+  }
+  if (str == "SKIP") {
+    return cmCTestTypes::ResourceErrorAction::Skip;
+  }
+  return {};
+}
+
+std::string ResourceErrorActionToString(ResourceErrorAction action)
+{
+  switch (action) {
+    case cmCTestTypes::ResourceErrorAction::Fail:
+      return "FAIL";
+    case cmCTestTypes::ResourceErrorAction::Skip:
+      return "SKIP";
+  }
+  return {};
+}
+
 } // namespace cmCTestTypes

Source/CTest/cmCTestTypes.h

@@ -5,6 +5,9 @@
 
 #include "cmConfigure.h" // IWYU pragma: keep
 
+#include <string>
+
+#include <cm/optional>
 #include <cm/string_view>
 
 namespace cmCTestTypes {
@@ -17,6 +20,16 @@ enum class TruncationMode
   Head
 };
 
+// Resource error action
+enum class ResourceErrorAction
+{
+  Fail,
+  Skip
+};
+
 bool SetTruncationMode(TruncationMode& mode, cm::string_view str);
 
+cm::optional<ResourceErrorAction> GetResourceErrorAction(cm::string_view str);
+std::string ResourceErrorActionToString(ResourceErrorAction action);
+
 } // namespace cmCTestTypes

Source/cmCTest.cxx

@@ -3414,6 +3414,7 @@ CTestVarConfigEntry const kCTestVarConfigMap[] = {
   { "CTEST_TEST_TIMEOUT",                  "TimeOut"                     },
   { "CTEST_TEST_COVERAGE_TOOL",            "CTestTestCoverageTool"       },
   { "CTEST_RESOURCE_SPEC_FILE",            "ResourceSpecFile"            },
+  { "CTEST_RESOURCE_ERROR_ACTION",         "ResourceErrorAction"         },
   { "CTEST_TEST_LOAD",                     "TestLoad"                    },
   // Coverage step
   { "CTEST_COVERAGE_COMMAND",              "CoverageCommand"             },

Source/cmLocalGenerator.cxx

@@ -386,6 +386,12 @@ void cmLocalGenerator::GenerateTestFiles()
   if (!resourceSpecFile.empty()) {
     fout << "set(CTEST_RESOURCE_SPEC_FILE \"" << resourceSpecFile << "\")\n";
   }
+  std::string resourceErrorAction =
+    this->Makefile->GetSafeDefinition("CTEST_RESOURCE_ERROR_ACTION");
+  if (!resourceErrorAction.empty()) {
+    fout << "set(CTEST_RESOURCE_ERROR_ACTION \"" << resourceErrorAction
+         << "\")\n";
+  }
 
   auto writeTestIncludeFile = [this, &fout, &configurationTypes,
                                &config](std::string const& entry) {

Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake

@@ -196,6 +196,7 @@ endfunction()
 
 run_ctest_resource_dynamic(dynamic-resource -VV)
 run_ctest_resource_dynamic(dynamic-resource-notenough)
+run_ctest_resource_dynamic(dynamic-resource-notenough-skip)
 run_ctest_resource_dynamic(dynamic-resource-nofile)
 run_ctest_resource_dynamic(dynamic-resource-multiple-generators)
 run_ctest_resource_dynamic(dynamic-resource-no-setup-fixture)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-skip-ctest-s-stdout.txt

@@ -0,0 +1,4 @@
+    Start 2: GenerateSpecFile
+1/2 Test #2: GenerateSpecFile \.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.   Passed +[0-9]+\.[0-9]+ sec
+    Start 1: RealTest
+2/2 Test #1: RealTest \.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.\*\*\*Skipped \(Resource error\) +[0-9]+\.[0-9]+ sec

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-skip.cmake

@@ -0,0 +1,11 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:2"
+  RESOURCE_ERROR_ACTION "SKIP"
+  )