From 572e4b5d2af4d2b9c5e8a57d9c6da5a94eb73b60 Mon Sep 17 00:00:00 2001 From: Arha Gatram Date: Thu, 23 Jul 2026 15:50:43 -0700 Subject: [PATCH] ctest: Add --show-only=json-v1-raw option This option changes the JSON value field for test properties to use the unparsed string value. Fixes: #27958 --- Help/manual/ctest.1.rst | 13 +- Help/manual/presets/execution-properties.rst | 2 + Help/manual/presets/schema.json | 5 +- Help/manual/presets/schema.yaml | 6 +- Help/release/dev/ctest-json-test-prop-raw.rst | 6 + Source/CTest/cmCTestMultiProcessHandler.cxx | 152 ++++++++++---- Source/CTest/cmCTestTestHandler.cxx | 40 ++++ Source/CTest/cmCTestTestHandler.h | 31 +++ Source/cmCTest.cxx | 13 ++ Source/cmCTest.h | 2 + Source/ctest.cxx | 3 +- .../CTestCommandLine/RunCMakeTest.cmake | 9 + .../show-only_json-v1-raw-check.cmake | 1 + .../show-only_json-v1-raw_check.py | 193 ++++++++++++++++++ .../show-only_json-v1_check.py | 47 +++-- 15 files changed, 455 insertions(+), 68 deletions(-) create mode 100644 Help/release/dev/ctest-json-test-prop-raw.rst create mode 100644 Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw-check.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw_check.py diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index b72536f85f..c32a7522f9 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -260,6 +260,13 @@ The options for running tests are: Dump the test information in JSON format. See `Show as JSON Object Model`_. + ``json-v1-raw`` + + .. versionadded:: 4.5 + + Dump the test information in JSON format, but leaves test property + values as raw strings. See `Show as JSON Object Model`_. + .. option:: -L , --label-regex Run tests with labels matching regular expression as described under @@ -1895,9 +1902,9 @@ Show as JSON Object Model .. versionadded:: 3.14 -When the ``--show-only=json-v1`` command line option is given, the test -information is output in JSON format. Version 1.0 of the JSON object -model is defined as follows: +When the ``--show-only=json-v1`` or ``--show-only=json-v1-raw`` command +line option is given, the test information is output in JSON format. +Version 1.0 of the JSON object model is defined as follows: ``kind`` The string "ctestInfo". diff --git a/Help/manual/presets/execution-properties.rst b/Help/manual/presets/execution-properties.rst index 8e3d4a0f86..a158d0ca4f 100644 --- a/Help/manual/presets/execution-properties.rst +++ b/Help/manual/presets/execution-properties.rst @@ -55,6 +55,8 @@ * ``json-v1`` + * ``json-v1-raw`` + .. _`CMakePresets.testPresets.execution.repeat`: ``repeat`` diff --git a/Help/manual/presets/schema.json b/Help/manual/presets/schema.json index 34a51d3272..5ebce72f2c 100644 --- a/Help/manual/presets/schema.json +++ b/Help/manual/presets/schema.json @@ -3391,9 +3391,10 @@ "type": "string", "enum": [ "human", - "json-v1" + "json-v1", + "json-v1-raw" ], - "description": "An optional string. Equivalent to passing --show-only on the command line. Value must be \"human\" or \"json-v1\"." + "description": "An optional string. Equivalent to passing --show-only on the command line. Value must be \"human\", \"json-v1\", or \"json-v1-raw\"." }, "testPresets.execution.repeat@v2..v9": { "type": "object", diff --git a/Help/manual/presets/schema.yaml b/Help/manual/presets/schema.yaml index 631acd042e..4376e54f24 100644 --- a/Help/manual/presets/schema.yaml +++ b/Help/manual/presets/schema.yaml @@ -1657,9 +1657,11 @@ properties: enum: - human - json-v1 + - json-v1-raw description: An optional string. Equivalent to passing --show-only on the - command line. Value must be "human" or "json-v1". + command line. Value must be "human", "json-v1", or + "json-v1-raw". sphinxDescription: | An optional string. Equivalent to passing :ctest-option:`--show-only` on the command line. The string @@ -1668,6 +1670,8 @@ properties: * ``human`` * ``json-v1`` + + * ``json-v1-raw`` repeat: type: object description: diff --git a/Help/release/dev/ctest-json-test-prop-raw.rst b/Help/release/dev/ctest-json-test-prop-raw.rst new file mode 100644 index 0000000000..a531fb5e92 --- /dev/null +++ b/Help/release/dev/ctest-json-test-prop-raw.rst @@ -0,0 +1,6 @@ +ctest-json-test-prop-raw +------------------------ + +* The CTest option ``--show-only=[format]`` now has a ``json-v1-raw`` format. + When this format is used, the ``value`` for each test property is the + original unparsed string value specified in CMake. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 3ad8855791..fbe9a4c71d 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -1170,134 +1170,197 @@ static Json::Value DumpCTestProperty(std::string const& name, return property; } +static Json::Value DumpCTestPropertyRaw(std::string const& name, + std::string const& value) +{ + Json::Value property = Json::objectValue; + property["name"] = name; + property["value"] = value; + return property; +} + static Json::Value DumpCTestProperties( - cmCTestTestHandler::cmCTestTestProperties& testProperties) + cmCTestTestHandler::cmCTestTestProperties& testProperties, bool raw) { Json::Value properties = Json::arrayValue; + std::unordered_map& rawProperties = + testProperties.RawProperties; if (!testProperties.AttachOnFail.empty()) { - properties.append(DumpCTestProperty( - "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail))); + properties.append( + DumpCTestProperty("ATTACHED_FILES_ON_FAIL", + raw ? rawProperties["ATTACHED_FILES_ON_FAIL"] + : DumpToJsonArray(testProperties.AttachOnFail))); } if (!testProperties.AttachedFiles.empty()) { - properties.append(DumpCTestProperty( - "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles))); + properties.append( + DumpCTestProperty("ATTACHED_FILES", + raw ? rawProperties["ATTACHED_FILES"] + : DumpToJsonArray(testProperties.AttachedFiles))); } if (testProperties.Cost != 0.0f) { - properties.append( - DumpCTestProperty("COST", static_cast(testProperties.Cost))); + properties.append(DumpCTestProperty( + "COST", + raw ? rawProperties["COST"] + : Json::Value(static_cast(testProperties.Cost)))); } if (!testProperties.Depends.empty()) { properties.append( - DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends))); + DumpCTestProperty("DEPENDS", + raw ? rawProperties["DEPENDS"] + : DumpToJsonArray(testProperties.Depends))); } if (testProperties.Disabled) { - properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled)); + properties.append(DumpCTestProperty( + "DISABLED", + raw ? rawProperties["DISABLED"] : Json::Value(testProperties.Disabled))); } if (!testProperties.Environment.empty()) { - properties.append(DumpCTestProperty( - "ENVIRONMENT", DumpToJsonArray(testProperties.Environment))); + properties.append( + DumpCTestProperty("ENVIRONMENT", + raw ? rawProperties["ENVIRONMENT"] + : DumpToJsonArray(testProperties.Environment))); } if (!testProperties.EnvironmentModification.empty()) { properties.append(DumpCTestProperty( "ENVIRONMENT_MODIFICATION", - DumpToJsonArray(testProperties.EnvironmentModification))); + raw ? rawProperties["ENVIRONMENT_MODIFICATION"] + : DumpToJsonArray(testProperties.EnvironmentModification))); } if (!testProperties.ErrorRegularExpressions.empty()) { properties.append(DumpCTestProperty( "FAIL_REGULAR_EXPRESSION", - DumpRegExToJsonArray(testProperties.ErrorRegularExpressions))); + raw ? rawProperties["FAIL_REGULAR_EXPRESSION"] + : DumpRegExToJsonArray(testProperties.ErrorRegularExpressions))); } if (!testProperties.SkipRegularExpressions.empty()) { properties.append(DumpCTestProperty( "SKIP_REGULAR_EXPRESSION", - DumpRegExToJsonArray(testProperties.SkipRegularExpressions))); + raw ? rawProperties["SKIP_REGULAR_EXPRESSION"] + : DumpRegExToJsonArray(testProperties.SkipRegularExpressions))); } if (!testProperties.FixturesCleanup.empty()) { properties.append(DumpCTestProperty( - "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup))); + "FIXTURES_CLEANUP", + raw ? rawProperties["FIXTURES_CLEANUP"] + : DumpToJsonArray(testProperties.FixturesCleanup))); } if (!testProperties.FixturesRequired.empty()) { properties.append(DumpCTestProperty( - "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired))); + "FIXTURES_REQUIRED", + raw ? rawProperties["FIXTURES_REQUIRED"] + : DumpToJsonArray(testProperties.FixturesRequired))); } if (!testProperties.FixturesSetup.empty()) { - properties.append(DumpCTestProperty( - "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup))); + properties.append( + DumpCTestProperty("FIXTURES_SETUP", + raw ? rawProperties["FIXTURES_SETUP"] + : DumpToJsonArray(testProperties.FixturesSetup))); } if (!testProperties.GeneratedResourceSpecFile.empty()) { properties.append( DumpCTestProperty("GENERATED_RESOURCE_SPEC_FILE", - testProperties.GeneratedResourceSpecFile)); + raw ? rawProperties["GENERATED_RESOURCE_SPEC_FILE"] + : testProperties.GeneratedResourceSpecFile)); } if (!testProperties.Labels.empty()) { - properties.append( - DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels))); + properties.append(DumpCTestProperty( + "LABELS", + raw ? rawProperties["LABELS"] : DumpToJsonArray(testProperties.Labels))); } if (!testProperties.Measurements.empty()) { properties.append(DumpCTestProperty( - "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements))); + "MEASUREMENT", + raw ? rawProperties["MEASUREMENT"] + : DumpMeasurementToJsonArray(testProperties.Measurements))); } if (!testProperties.RequiredRegularExpressions.empty()) { properties.append(DumpCTestProperty( "PASS_REGULAR_EXPRESSION", - DumpRegExToJsonArray(testProperties.RequiredRegularExpressions))); + raw ? rawProperties["PASS_REGULAR_EXPRESSION"] + : DumpRegExToJsonArray(testProperties.RequiredRegularExpressions))); } if (!testProperties.ResourceGroups.empty()) { properties.append(DumpCTestProperty( "RESOURCE_GROUPS", - DumpResourceGroupsToJsonArray(testProperties.ResourceGroups))); + raw ? rawProperties["RESOURCE_GROUPS"] + : DumpResourceGroupsToJsonArray(testProperties.ResourceGroups))); } if (testProperties.WantAffinity) { properties.append( - DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity)); + DumpCTestProperty("PROCESSOR_AFFINITY", + raw ? rawProperties["PROCESSOR_AFFINITY"] + : Json::Value(testProperties.WantAffinity))); } if (testProperties.Processors != 1) { properties.append( - DumpCTestProperty("PROCESSORS", testProperties.Processors)); + DumpCTestProperty("PROCESSORS", + raw ? rawProperties["PROCESSORS"] + : Json::Value(testProperties.Processors))); } if (!testProperties.RequiredFiles.empty()) { - properties.append(DumpCTestProperty( - "REQUIRED_FILES", DumpToJsonArray(testProperties.RequiredFiles))); + properties.append( + DumpCTestProperty("REQUIRED_FILES", + raw ? rawProperties["REQUIRED_FILES"] + : DumpToJsonArray(testProperties.RequiredFiles))); } if (!testProperties.ProjectResources.empty()) { properties.append(DumpCTestProperty( - "RESOURCE_LOCK", DumpToJsonArray(testProperties.ProjectResources))); + "RESOURCE_LOCK", + raw ? rawProperties["RESOURCE_LOCK"] + : DumpToJsonArray(testProperties.ProjectResources))); } if (testProperties.RunSerial) { properties.append( - DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial)); + DumpCTestProperty("RUN_SERIAL", + raw ? rawProperties["RUN_SERIAL"] + : Json::Value(testProperties.RunSerial))); } if (testProperties.SkipReturnCode != -1) { properties.append( - DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode)); + DumpCTestProperty("SKIP_RETURN_CODE", + raw ? rawProperties["SKIP_RETURN_CODE"] + : Json::Value(testProperties.SkipReturnCode))); } if (testProperties.Timeout) { properties.append( - DumpCTestProperty("TIMEOUT", testProperties.Timeout->count())); + DumpCTestProperty("TIMEOUT", + raw ? rawProperties["TIMEOUT"] + : Json::Value(testProperties.Timeout->count()))); } if (testProperties.TimeoutSignal) { - properties.append(DumpCTestProperty("TIMEOUT_SIGNAL_NAME", - testProperties.TimeoutSignal->Name)); + properties.append( + DumpCTestProperty("TIMEOUT_SIGNAL_NAME", + raw ? rawProperties["TIMEOUT_SIGNAL_NAME"] + : testProperties.TimeoutSignal->Name)); } if (testProperties.TimeoutGracePeriod) { - properties.append( - DumpCTestProperty("TIMEOUT_SIGNAL_GRACE_PERIOD", - testProperties.TimeoutGracePeriod->count())); + properties.append(DumpCTestProperty( + "TIMEOUT_SIGNAL_GRACE_PERIOD", + raw ? rawProperties["TIMEOUT_SIGNAL_GRACE_PERIOD"] + : Json::Value(testProperties.TimeoutGracePeriod->count()))); } if (!testProperties.TimeoutRegularExpressions.empty()) { - properties.append(DumpCTestProperty( - "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties))); + properties.append( + DumpCTestProperty("TIMEOUT_AFTER_MATCH", + raw ? rawProperties["TIMEOUT_AFTER_MATCH"] + : DumpTimeoutAfterMatch(testProperties))); } if (testProperties.WillFail) { - properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail)); + properties.append( + DumpCTestProperty("WILL_FAIL", + raw ? rawProperties["WILL_FAIL"] + : Json::Value(testProperties.WillFail))); } if (!testProperties.Directory.empty()) { - properties.append( - DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory)); + properties.append(DumpCTestProperty( + "WORKING_DIRECTORY", + raw ? rawProperties["WORKING_DIRECTORY"] : testProperties.Directory)); } if (!testProperties.CustomProperties.empty()) { for (auto const& it : testProperties.CustomProperties) { - properties.append(DumpCTestProperty(it.first, it.second)); + Json::Value property = raw ? DumpCTestPropertyRaw(it.first, it.second) + : DumpCTestProperty(it.first, it.second); + properties.append(std::move(property)); } } return properties; @@ -1409,7 +1472,8 @@ static Json::Value DumpCTestInfo( } testInfo["command"] = DumpToJsonArray(commandAndArgs); } - Json::Value properties = DumpCTestProperties(testProperties); + Json::Value properties = DumpCTestProperties( + testProperties, testRun.GetCTest()->GetOutputAsJsonRaw()); if (!properties.empty()) { testInfo["properties"] = properties; } diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index a5cd760d1c..00c46f9ff4 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -338,6 +338,14 @@ bool cmCTestSetDirectoryPropertiesCommand::InitialPass( return this->TestHandler->SetDirectoryProperties(args); } +void AppendRawProperty(std::string& property, std::string const& value) +{ + if (!property.empty() && !value.empty()) { + property += ';'; + } + property += value; +} + // get the next number in a string with numbers separated by , // pos is the start of the search and pos2 is the end of the search // pos becomes pos2 after a call to GetNextNumber. @@ -2392,32 +2400,42 @@ bool cmCTestTestHandler::SetTestsProperties( } } } else if (key == "WILL_FAIL"_s) { + rt.RawProperties[key] = val; rt.WillFail = cmIsOn(val); } else if (key == "DISABLED"_s) { + rt.RawProperties[key] = val; rt.Disabled = cmIsOn(val); } else if (key == "ATTACHED_FILES"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.AttachedFiles); } else if (key == "ATTACHED_FILES_ON_FAIL"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.AttachOnFail); } else if (key == "RESOURCE_LOCK"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; rt.ProjectResources.insert(lval.begin(), lval.end()); } else if (key == "FIXTURES_SETUP"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; rt.FixturesSetup.insert(lval.begin(), lval.end()); } else if (key == "FIXTURES_CLEANUP"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; rt.FixturesCleanup.insert(lval.begin(), lval.end()); } else if (key == "FIXTURES_REQUIRED"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; rt.FixturesRequired.insert(lval.begin(), lval.end()); } else if (key == "TIMEOUT"_s) { + rt.RawProperties[key] = val; rt.Timeout = cmDuration(atof(val.c_str())); } else if (key == "TIMEOUT_SIGNAL_NAME"_s) { + rt.RawProperties[key] = val; #ifdef _WIN32 rt.AppendError("TIMEOUT_SIGNAL_NAME is not supported on Windows."); #else @@ -2443,6 +2461,7 @@ bool cmCTestTestHandler::SetTestsProperties( } #endif } else if (key == "TIMEOUT_SIGNAL_GRACE_PERIOD"_s) { + rt.RawProperties[key] = val; #ifdef _WIN32 rt.AppendError( "TIMEOUT_SIGNAL_GRACE_PERIOD is not supported on Windows."); @@ -2465,46 +2484,60 @@ bool cmCTestTestHandler::SetTestsProperties( } #endif } else if (key == "COST"_s) { + rt.RawProperties[key] = val; rt.Cost = static_cast(atof(val.c_str())); } else if (key == "REQUIRED_FILES"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.RequiredFiles); } else if (key == "RUN_SERIAL"_s) { + rt.RawProperties[key] = val; rt.RunSerial = cmIsOn(val); } else if (key == "FAIL_REGULAR_EXPRESSION"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; for (std::string const& cr : lval) { rt.ErrorRegularExpressions.emplace_back(cr, cr); } } else if (key == "SKIP_REGULAR_EXPRESSION"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; for (std::string const& cr : lval) { rt.SkipRegularExpressions.emplace_back(cr, cr); } } else if (key == "PROCESSORS"_s) { + rt.RawProperties[key] = val; rt.Processors = atoi(val.c_str()); if (rt.Processors < 1) { rt.Processors = 1; } } else if (key == "PROCESSOR_AFFINITY"_s) { + rt.RawProperties[key] = val; rt.WantAffinity = cmIsOn(val); } else if (key == "RESOURCE_GROUPS"_s) { + AppendRawProperty(rt.RawProperties[key], val); if (!ParseResourceGroupsProperty(val, rt.ResourceGroups)) { return false; } } else if (key == "GENERATED_RESOURCE_SPEC_FILE"_s) { + rt.RawProperties[key] = val; rt.GeneratedResourceSpecFile = val; } else if (key == "SKIP_RETURN_CODE"_s) { + rt.RawProperties[key] = val; rt.SkipReturnCode = atoi(val.c_str()); if (rt.SkipReturnCode < 0 || rt.SkipReturnCode > 255) { rt.SkipReturnCode = -1; } } else if (key == "DEPENDS"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.Depends); } else if (key == "ENVIRONMENT"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.Environment); } else if (key == "ENVIRONMENT_MODIFICATION"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmExpandList(val, rt.EnvironmentModification); } else if (key == "LABELS"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList Labels{ val }; rt.Labels.insert(rt.Labels.end(), Labels.begin(), Labels.end()); // sort the array @@ -2513,6 +2546,7 @@ bool cmCTestTestHandler::SetTestsProperties( auto new_end = std::unique(rt.Labels.begin(), rt.Labels.end()); rt.Labels.erase(new_end, rt.Labels.end()); } else if (key == "MEASUREMENT"_s) { + AppendRawProperty(rt.RawProperties[key], val); size_t pos = val.find_first_of('='); if (pos != std::string::npos) { std::string mKey = val.substr(0, pos); @@ -2522,13 +2556,16 @@ bool cmCTestTestHandler::SetTestsProperties( rt.Measurements[val] = "1"; } } else if (key == "PASS_REGULAR_EXPRESSION"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList lval{ val }; for (std::string const& cr : lval) { rt.RequiredRegularExpressions.emplace_back(cr, cr); } } else if (key == "WORKING_DIRECTORY"_s) { + rt.RawProperties[key] = val; rt.Directory = val; } else if (key == "TIMEOUT_AFTER_MATCH"_s) { + AppendRawProperty(rt.RawProperties[key], val); cmList propArgs{ val }; if (propArgs.size() != 2) { cmCTestLog(this->CTest, WARNING, @@ -2582,6 +2619,8 @@ bool cmCTestTestHandler::SetDirectoryProperties( std::string cwd = cmSystemTools::GetLogicalWorkingDirectory(); if (cwd == rt.Directory) { if (key == "LABELS"_s) { + AppendRawProperty(rt.RawProperties[key], val); + cmList DirectoryLabels{ val }; rt.Labels.insert(rt.Labels.end(), DirectoryLabels.begin(), DirectoryLabels.end()); @@ -2622,6 +2661,7 @@ bool cmCTestTestHandler::AddTest(std::vector const& args) test.Args = args; test.CTestDirectory = cmSystemTools::GetLogicalWorkingDirectory(); test.Directory = test.CTestDirectory; + test.RawProperties["WORKING_DIRECTORY"] = test.CTestDirectory; cmCTestOptionalLog(this->CTest, DEBUG, "Set test directory: " << test.Directory << std::endl, this->Quiet); diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 926cad3132..b7c3a1f3b2 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -150,50 +151,80 @@ public: std::string Name; // working directory for test, overridden by WORKING_DIRECTORY property std::string Directory; + std::string DirectoryRaw; // Original directory of test creation std::string CTestDirectory; std::vector Args; std::vector RequiredFiles; + std::string RequiredFilesRaw; std::vector Depends; + std::string DependsRaw; std::vector AttachedFiles; + std::string AttachedFilesRaw; std::vector AttachOnFail; + std::string AttachOnFailRaw; std::vector> ErrorRegularExpressions; + std::string ErrorRegularExpressionsRaw; std::vector> RequiredRegularExpressions; + std::string RequiredRegularExpressionsRaw; std::vector> SkipRegularExpressions; + std::string SkipRegularExpressionsRaw; std::vector> TimeoutRegularExpressions; + std::string TimeoutRegularExpressionsRaw; std::map Measurements; + std::string MeasurementsRaw; std::map CustomProperties; + std::unordered_map RawProperties; bool IsInBasedOnREOptions = true; bool WillFail = false; + std::string WillFailRaw; bool Disabled = false; + std::string DisabledRaw; float Cost = 0; + std::string CostRaw; int PreviousRuns = 0; bool RunSerial = false; + std::string RunSerialRaw; cm::optional Timeout; + std::string TimeoutRaw; cm::optional TimeoutSignal; + std::string TimeoutSignalRaw; cm::optional TimeoutGracePeriod; + std::string TimeoutGracePeriodRaw; cmDuration AlternateTimeout; int Index = 0; // Requested number of process slots int Processors = 1; + std::string ProcessorsRaw; bool WantAffinity = false; + std::string WantAffinityRaw; std::vector Affinity; // return code of test which will mark test as "not run" int SkipReturnCode = -1; + std::string SkipReturnCodeRaw; std::vector Environment; + std::string EnvironmentRaw; std::vector EnvironmentModification; + std::string EnvironmentModificationRaw; std::vector Labels; + std::string LabelsRaw; std::set ProjectResources; // RESOURCE_LOCK + std::string ProjectResourcesRaw; std::set FixturesSetup; + std::string FixturesSetupRaw; std::set FixturesCleanup; + std::string FixturesCleanupRaw; std::set FixturesRequired; + std::string FixturesRequiredRaw; std::set RequireSuccessDepends; std::vector> ResourceGroups; + std::string ResourceGroupsRaw; std::string GeneratedResourceSpecFile; + std::string GeneratedResourceSpecFileRaw; std::string BuildDepends; // Private test generator properties used to track backtraces cmListFileBacktrace Backtrace; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 25963bf61d..6dd9c30629 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -134,6 +134,7 @@ struct cmCTest::Private bool ShowOnly = false; bool OutputAsJson = false; int OutputAsJsonVersion = 1; + bool OutputAsJsonRaw = false; // TODO: The ctest configuration should be a hierarchy of // configuration option sources: command-line, script, ini file. @@ -2355,6 +2356,13 @@ int cmCTest::Run(std::vector const& args) this->Impl->Quiet = true; this->Impl->OutputAsJson = true; this->Impl->OutputAsJsonVersion = 1; + } else if (format == "json-v1-raw") { + // Force quiet mode so the only output + // is the json object model. + this->Impl->Quiet = true; + this->Impl->OutputAsJson = true; + this->Impl->OutputAsJsonVersion = 1; + this->Impl->OutputAsJsonRaw = true; } else if (format == "human") { } else if (!format.empty()) { cmSystemTools::Error( @@ -3104,6 +3112,11 @@ int cmCTest::GetOutputAsJsonVersion() return this->Impl->OutputAsJsonVersion; } +bool cmCTest::GetOutputAsJsonRaw() const +{ + return this->Impl->OutputAsJsonRaw; +} + bool cmCTest::ShouldUseHTTP10() const { return this->Impl->UseHTTP10; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 9bdedf5b98..392dd973c0 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -183,6 +183,8 @@ public: int GetOutputAsJsonVersion(); + bool GetOutputAsJsonRaw() const; + bool ShouldUseHTTP10() const; bool ShouldPrintLabels() const; diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 10c0d8a784..654df77967 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -60,7 +60,8 @@ cmDocumentationEntry const cmDocumentationOptions[] = { { "-N,--show-only[=format]", "Disable actual execution of tests. The optional 'format' defines the " "format of the test information and can be 'human' for the current text " - "format or 'json-v1' for json format. Defaults to 'human'." }, + "format, 'json-v1' for json format, or 'json-v1-raw' where the json " + "format stores the raw test property values. Defaults to 'human'." }, { "-L , --label-regex ", "Run tests with labels matching regular expression. " "With multiple -L, run tests where each " diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index ed407b85f6..bd1ce18035 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -554,6 +554,7 @@ function(run_ShowOnly) add_test(ShowOnly \"${CMAKE_COMMAND}\" -E echo) set_tests_properties(ShowOnly PROPERTIES GENERATED_RESOURCE_SPEC_FILE \"/Path/Does/Not/Exist\" + LABELS TestLabel RESOURCE_GROUPS \"2,threads:2,gpus:4;gpus:2,threads:4\" REQUIRED_FILES RequiredFileDoesNotExist _BACKTRACE_TRIPLES \"file1;1;add_test;file0;;\" @@ -564,11 +565,19 @@ function(run_ShowOnly) USER_DEFINED_A \"User defined property A value\" USER_DEFINED_B \"User defined property B value\" ) + set_tests_properties(ShowOnly PROPERTIES + REQUIRED_FILES AnotherRequiredFileDoesNotExist + ) + set_directory_properties(PROPERTIES + LABELS DirectoryLabel + ) add_test(ShowOnlyNotAvailable NOT_AVAILABLE) ") run_cmake_command(show-only_human ${CMAKE_CTEST_COMMAND} --show-only=human) run_cmake_command(show-only_bad ${CMAKE_CTEST_COMMAND} --show-only=bad) run_cmake_command(show-only_json-v1 ${CMAKE_CTEST_COMMAND} --show-only=json-v1) + run_cmake_command(show-only_json-v1-raw + ${CMAKE_CTEST_COMMAND} --show-only=json-v1-raw) endfunction() run_ShowOnly() diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw-check.cmake b/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw-check.cmake new file mode 100644 index 0000000000..936344f7e6 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw-check.cmake @@ -0,0 +1 @@ +show_only_json_check_python(1-raw) diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw_check.py b/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw_check.py new file mode 100644 index 0000000000..4404879732 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/show-only_json-v1-raw_check.py @@ -0,0 +1,193 @@ +import sys + +from show_only_json_check import * + +def check_kind(k): + assert is_string(k) + assert k == "ctestInfo" + +def check_version(v): + assert is_dict(v) + assert sorted(v.keys()) == ["major", "minor"] + assert is_int(v["major"]) + assert is_int(v["minor"]) + assert v["major"] == 1 + assert v["minor"] == 0 + +def check_backtracegraph(b): + assert is_dict(b) + assert sorted(b.keys()) == ["commands", "files", "nodes"] + check_backtracegraph_commands(b["commands"]) + check_backtracegraph_files(b["files"]) + check_backtracegraph_nodes(b["nodes"]) + +def check_backtracegraph_commands(c): + assert is_list(c) + assert len(c) == 1 + assert is_string(c[0]) + assert c[0] == "add_test" + +def check_backtracegraph_files(f): + assert is_list(f) + assert len(f) == 2 + assert is_string(f[0]) + assert is_string(f[1]) + assert f[0] == "file1" + assert f[1] == "file0" + +def check_backtracegraph_nodes(n): + assert is_list(n) + assert len(n) == 2 + node = n[0] + assert is_dict(node) + assert sorted(node.keys()) == ["file"] + assert is_int(node["file"]) + assert node["file"] == 1 + node = n[1] + assert is_dict(node) + assert sorted(node.keys()) == ["command", "file", "line", "parent"] + assert is_int(node["command"]) + assert is_int(node["file"]) + assert is_int(node["line"]) + assert is_int(node["parent"]) + assert node["command"] == 0 + assert node["file"] == 0 + assert node["line"] == 1 + assert node["parent"] == 0 + +def check_command(c): + assert is_list(c) + assert len(c) == 3 + assert is_string(c[0]) + check_re(c[0], r"/cmake(\.exe)?$") + assert is_string(c[1]) + assert c[1] == "-E" + assert is_string(c[2]) + assert c[2] == "echo" + +def check_generated_resource_spec_file_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "GENERATED_RESOURCE_SPEC_FILE" + assert p["value"] == "/Path/Does/Not/Exist" + +def check_labels_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "LABELS" + assert p["value"] == "TestLabel;DirectoryLabel" + +def check_reqfiles_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "REQUIRED_FILES" + assert p["value"] == \ + "RequiredFileDoesNotExist;AnotherRequiredFileDoesNotExist" + +def check_timeout_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "TIMEOUT" + assert p["value"] == "1234.5" + +def check_timeout_signal_name_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "TIMEOUT_SIGNAL_NAME" + assert p["value"] == "SIGINT" + +def check_timeout_signal_grace_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "TIMEOUT_SIGNAL_GRACE_PERIOD" + assert p["value"] == "2.1" + +def check_willfail_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "WILL_FAIL" + assert p["value"] == "true" + +def check_resource_groups_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "RESOURCE_GROUPS" + assert p["value"] == "2,threads:2,gpus:4;gpus:2,threads:4" + +def check_workingdir_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "WORKING_DIRECTORY" + assert p["value"].endswith("Tests/RunCMake/CTestCommandLine/ShowOnly") + +def check_defined_properties(p_list): + for property_id, p in zip(["A", "B"], p_list): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_string(p["value"]) + assert p["name"] == "USER_DEFINED_" + property_id + assert p["value"] == "User defined property " + property_id + " value" + +def check_properties(p): + assert is_list(p) + if sys.platform in ("win32"): + assert len(p) == 9 + check_generated_resource_spec_file_property(p[0]) + check_labels_property(p[1]) + check_resource_groups_property(p[2]) + check_reqfiles_property(p[3]) + check_timeout_property(p[4]) + check_willfail_property(p[5]) + check_workingdir_property(p[6]) + check_defined_properties(p[7:9]) + else: + assert len(p) == 11 + check_generated_resource_spec_file_property(p[0]) + check_labels_property(p[1]) + check_resource_groups_property(p[2]) + check_reqfiles_property(p[3]) + check_timeout_property(p[4]) + check_timeout_signal_name_property(p[5]) + check_timeout_signal_grace_property(p[6]) + check_willfail_property(p[7]) + check_workingdir_property(p[8]) + check_defined_properties(p[9:11]) + +def check_tests(t): + assert is_list(t) + assert len(t) == 1 + test = t[0] + assert is_dict(test) + assert sorted(test.keys()) == ["backtrace", "command", "name", "properties"] + assert is_int(test["backtrace"]) + assert test["backtrace"] == 1 + check_command(test["command"]) + assert is_string(test["name"]) + assert test["name"] == "ShowOnly" + check_properties(test["properties"]) + +assert is_dict(ctest_json) +assert sorted(ctest_json.keys()) == ["backtraceGraph", "kind", "tests", "version"] +check_backtracegraph(ctest_json["backtraceGraph"]) +check_kind(ctest_json["kind"]) +check_version(ctest_json["version"]) +check_tests(ctest_json["tests"]) diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py b/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py index 504bd3fcaa..2e0decc5aa 100644 --- a/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py +++ b/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py @@ -73,14 +73,25 @@ def check_generated_resource_spec_file_property(p): assert p["name"] == "GENERATED_RESOURCE_SPEC_FILE" assert p["value"] == "/Path/Does/Not/Exist" +def check_labels_property(p): + assert is_dict(p) + assert sorted(p.keys()) == ["name", "value"] + assert is_string(p["name"]) + assert is_list(p["value"]) + assert p["name"] == "LABELS" + assert len(p["value"]) == 2 + assert p["value"][0] == "DirectoryLabel" + assert p["value"][1] == "TestLabel" + def check_reqfiles_property(p): assert is_dict(p) assert sorted(p.keys()) == ["name", "value"] assert is_string(p["name"]) assert is_list(p["value"]) assert p["name"] == "REQUIRED_FILES" - assert len(p["value"]) == 1 + assert len(p["value"]) == 2 assert p["value"][0] == "RequiredFileDoesNotExist" + assert p["value"][1] == "AnotherRequiredFileDoesNotExist" def check_timeout_property(p): assert is_dict(p) @@ -190,25 +201,27 @@ def check_defined_properties(p_list): def check_properties(p): assert is_list(p) if sys.platform in ("win32"): - assert len(p) == 8 + assert len(p) == 9 check_generated_resource_spec_file_property(p[0]) - check_resource_groups_property(p[1]) - check_reqfiles_property(p[2]) - check_timeout_property(p[3]) - check_willfail_property(p[4]) - check_workingdir_property(p[5]) - check_defined_properties(p[6:7]) + check_labels_property(p[1]) + check_resource_groups_property(p[2]) + check_reqfiles_property(p[3]) + check_timeout_property(p[4]) + check_willfail_property(p[5]) + check_workingdir_property(p[6]) + check_defined_properties(p[7:9]) else: - assert len(p) == 10 + assert len(p) == 11 check_generated_resource_spec_file_property(p[0]) - check_resource_groups_property(p[1]) - check_reqfiles_property(p[2]) - check_timeout_property(p[3]) - check_timeout_signal_name_property(p[4]) - check_timeout_signal_grace_property(p[5]) - check_willfail_property(p[6]) - check_workingdir_property(p[7]) - check_defined_properties(p[8:9]) + check_labels_property(p[1]) + check_resource_groups_property(p[2]) + check_reqfiles_property(p[3]) + check_timeout_property(p[4]) + check_timeout_signal_name_property(p[5]) + check_timeout_signal_grace_property(p[6]) + check_willfail_property(p[7]) + check_workingdir_property(p[8]) + check_defined_properties(p[9:11]) def check_tests(t): assert is_list(t)