Merge topic 'sarif-exe-info'

afcf9ac341 SARIF: Report information about processes invoked in the SARIF log

Acked-by: Kitware Robot <kwrobot@kitware.com>
Tested-by: buildbot <buildbot@kitware.com>
Merge-request: !12439

Source/cmCMakeSarifLogger.cxx

@@ -8,9 +8,9 @@
 #include <utility>
 #include <vector>
 
+#include <cm/optional>
 #include <cm/string_view>
 
-#include "cmsys/FStream.hxx"
 #include "cmsys/String.h"
 
 #include "cmDiagnostics.h"
@@ -18,19 +18,14 @@
 #include "cmMessageType.h"
 #include "cmMessenger.h"
 #include "cmSarif.h"
-#include "cmState.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
-#include "cmValue.h"
+#include "cmTimestamp.h"
 #include "cmVersionConfig.h"
-#include "cmake.h"
 
 // CMake-specific SARIF helpers
 namespace {
 
-constexpr char const* CMakeSarifOutputFlag = "CMAKE_EXPORT_SARIF";
-constexpr char const* DefaultSarifFile = ".cmake/sarif/cmake.sarif";
-
 /// @brief Express the location of a `cmListFileContext` in SARIF
 /// @param[in] uriBaseIds A list of logical base directory names and their path
 ///
@@ -44,8 +39,7 @@ constexpr char const* DefaultSarifFile = ".cmake/sarif/cmake.sarif";
 /// map. Bases are tried in order.
 cmSarif::Location LocationFromContext(
   cmListFileContext const& lfc,
-  std::vector<std::pair<cm::string_view, cm::string_view>> const&
-    uriBaseIds = {})
+  std::vector<std::pair<std::string, std::string>> const& uriBaseIds = {})
 {
   cmSarif::Location location;
   location.Physical.Artifact.Uri = lfc.FilePath;
@@ -55,10 +49,10 @@ cmSarif::Location LocationFromContext(
   // provided.
   for (auto const& baseUri : uriBaseIds) {
     std::string relative = cmSystemTools::RelativeIfUnder(
-      std::string(baseUri.second), location.Physical.Artifact.Uri);
+      baseUri.second, location.Physical.Artifact.Uri);
     if (relative != location.Physical.Artifact.Uri) {
       location.Physical.Artifact.Uri = relative;
-      location.Physical.Artifact.UriBaseId = std::string(baseUri.first);
+      location.Physical.Artifact.UriBaseId = baseUri.first;
     }
   }
 
@@ -83,8 +77,7 @@ cmSarif::Location LocationFromContext(
 
 cm::optional<cmSarif::Location> LastLocation(
   cmListFileBacktrace backtrace,
-  std::vector<std::pair<cm::string_view, cm::string_view>> const&
-    uriBaseIds = {})
+  std::vector<std::pair<std::string, std::string>> const& uriBaseIds = {})
 {
   if (backtrace.Empty()) {
     return {};
@@ -94,8 +87,7 @@ cm::optional<cmSarif::Location> LastLocation(
 
 cm::optional<cmSarif::Stack> StackFromBacktrace(
   cmListFileBacktrace bt,
-  std::vector<std::pair<cm::string_view, cm::string_view>> const&
-    uriBaseIds = {})
+  std::vector<std::pair<std::string, std::string>> const& uriBaseIds = {})
 {
   if (bt.Empty()) {
     return {};
@@ -205,90 +197,48 @@ cmSarif::ResultSeverityLevel SarifLevelFromMessageType(MessageType type)
 
 } // namespace
 
-cmCMakeSarifLogger::cmCMakeSarifLogger(cmake& cm)
-  : CM(cm)
-{
-  if (this->CM.GetState()->GetRole() == cmState::Role::Project) {
-    cm.MarkCliAsUsed(CMakeSarifOutputFlag);
-  }
-}
-
 cmCMakeSarifLogger::~cmCMakeSarifLogger()
 {
   this->GenerateForRun();
 }
 
-cm::optional<std::string> cmCMakeSarifLogger::FileOutputPath() const
+void cmCMakeSarifLogger::SetOutputPath(std::string const& path)
 {
-  // If a SARIF path was specified via CLI, use it. Otherwise, check whether
-  // logging is enabled via the project cache variable and use the default
-  // path if so.
-  if (cm::optional<std::string> specifiedPath = this->CM.GetSarifFilePath()) {
-    return specifiedPath;
-  }
-  if (this->CM.GetState()->GetRole() == cmState::Role::Project &&
-      this->CM.GetCacheDefinition(CMakeSarifOutputFlag).IsOn()) {
-    return cmStrCat(this->CM.GetHomeOutputDirectory(), '/', DefaultSarifFile);
-  }
-  return cm::nullopt;
+  this->FilePath = path;
 }
 
-bool cmCMakeSarifLogger::WriteFile(std::string const& path,
-                                   bool createParentDirectories) const
+void cmCMakeSarifLogger::AddBaseDirectory(cm::string_view name,
+                                          cm::string_view path)
 {
-  if (createParentDirectories) {
-    if (!cmSystemTools::MakeDirectory(cmSystemTools::GetFilenamePath(path))
-           .IsSuccess()) {
-      return false;
-    }
-  }
+  this->UriBaseIds.emplace_back(std::string(name), std::string(path));
+  this->CMakeRun.OriginalUriBaseIds.emplace(
+    std::string(name),
+    cmSarif::ArtifactLocation{ cmStrCat("file://", path, "/"), "" });
+}
 
-  cmsys::ofstream outputFile(path);
-  if (!outputFile.good()) {
-    return false;
-  }
-
-  // Run object to build
-  cmSarif::Run run;
-  run.Tool = CreateCMakeTool();
+void cmCMakeSarifLogger::RecordDiagnostics(
+  std::vector<cmMessenger::Message> const& messages)
+{
+  this->CMakeRun.Tool = CreateCMakeTool();
 
   // Helper to add rules to the run as encountered in results and get their
   // index for reporting
-  std::unordered_map<std::string, std::size_t> ruleIndices;
   auto use_rule = [&](MessageType type, cmDiagnosticCategory category) {
-    std::string category_name = RuleIdForMessageType(type, category);
-    auto result = ruleIndices.emplace(category_name, 0);
-    if (result.second) {
-      result.first->second = run.Tool.Driver.Rules.size();
-      run.Tool.Driver.Rules.emplace_back(
-        ReportingDescriptorForMessageType(type, category));
+    auto category_name = RuleIdForMessageType(type, category);
+    auto ruleIt = this->RuleIndices.find(category_name);
+    if (ruleIt != this->RuleIndices.end()) {
+      return std::make_pair(category_name, ruleIt->second);
     }
-    return *result.first;
+
+    this->CMakeRun.Tool.Driver.Rules.emplace_back(
+      ReportingDescriptorForMessageType(type, category));
+    this->RuleIndices.emplace(category_name,
+                              this->CMakeRun.Tool.Driver.Rules.size() - 1);
+    return std::make_pair(category_name,
+                          this->CMakeRun.Tool.Driver.Rules.size() - 1);
   };
 
-  // Make a prioritized list of base directories applicable in this context.
-  // This is used for normalizing the paths of related locations.
-  std::vector<std::pair<cm::string_view, cm::string_view>> uriBaseIds;
-
-  std::string const& binDir = this->CM.GetHomeOutputDirectory();
-  if (!binDir.empty()) {
-    uriBaseIds.emplace_back("CMAKE_BINARY_DIR", binDir);
-  }
-
-  std::string const& homeDir = this->CM.GetHomeDirectory();
-  if (!homeDir.empty()) {
-    uriBaseIds.emplace_back("CMAKE_SOURCE_DIR", homeDir);
-  }
-
-  // Log the base directories for this run.
-  for (auto const& base : uriBaseIds) {
-    run.OriginalUriBaseIds.emplace(
-      std::string(base.first),
-      cmSarif::ArtifactLocation{ cmStrCat("file://", base.second, "/"), "" });
-  }
-
-  cmMessenger const& messenger = *this->CM.GetMessenger();
-  for (auto const& message : messenger.GetDisplayedMessages()) {
+  for (auto const& message : messages) {
     // SARIF should only emit diagnostic messages, not general messages/logs
     switch (message.Type) {
       case MessageType::MESSAGE:
@@ -306,29 +256,55 @@ bool cmCMakeSarifLogger::WriteFile(std::string const& path,
     result.RuleId = ruleInfo.first;
     result.RuleIndex = ruleInfo.second;
     result.Message = cmSarif::Message{ message.Text };
-    result.Location = LastLocation(message.Backtrace, uriBaseIds);
+    result.Location = LastLocation(message.Backtrace, this->UriBaseIds);
     if (cm::optional<cmSarif::Stack> stack =
-          StackFromBacktrace(message.Backtrace, uriBaseIds)) {
+          StackFromBacktrace(message.Backtrace, this->UriBaseIds)) {
       result.Stacks.emplace_back(std::move(*stack));
     }
     result.Level = SarifLevelFromMessageType(message.Type);
 
-    run.Results.emplace_back(std::move(result));
+    this->CMakeRun.Results.emplace_back(std::move(result));
+  }
+}
+
+void cmCMakeSarifLogger::RecordInvocation(
+  int ac, char const* const* av, int exitCode,
+  std::chrono::system_clock::time_point startTime,
+  std::chrono::system_clock::time_point endTime)
+{
+  cmTimestamp timestamp;
+
+  cmSarif::Invocation invocation;
+  invocation.Arguments.assign(av, av + ac);
+  invocation.ExecutableLocation.Uri = cmSystemTools::GetCMakeCommand();
+  invocation.StartTimeUtc = timestamp.CreateTimestampFromTimeT(
+    std::chrono::system_clock::to_time_t(startTime), "", true);
+  invocation.EndTimeUtc = timestamp.CreateTimestampFromTimeT(
+    std::chrono::system_clock::to_time_t(endTime), "", true);
+  invocation.ExitCode = exitCode;
+  invocation.ExecutionSuccessful = (exitCode == 0);
+
+  this->CMakeRun.Invocations.emplace_back(std::move(invocation));
+}
+
+bool cmCMakeSarifLogger::WriteFile(std::string const& path) const
+{
+  std::string const dir = cmSystemTools::GetFilenamePath(path);
+  if (!cmSystemTools::FileIsDirectory(dir)) {
+    return false;
   }
 
-  return cmSarif::WriteLog(path, run);
+  return cmSarif::WriteLog(path, this->CMakeRun);
 }
 
 void cmCMakeSarifLogger::GenerateForRun() const
 {
-  cm::optional<std::string> path = this->FileOutputPath();
-  if (!path) {
+  if (this->FilePath.empty()) {
     return;
   }
 
-  // If using the default path within the build dir, ensure parents are created
-  bool const createParents = !this->CM.GetSarifFilePath().has_value();
-  if (!this->WriteFile(*path, createParents)) {
-    cmSystemTools::Error(cmStrCat("Failed to write SARIF log to ", *path));
+  if (!this->WriteFile(this->FilePath)) {
+    cmSystemTools::Error(
+      cmStrCat("Failed to write SARIF log to ", this->FilePath));
   }
 }

Source/cmCMakeSarifLogger.h

@@ -2,11 +2,17 @@
     file LICENSE.rst or https://cmake.org/licensing for details.  */
 #pragma once
 
+#include <chrono>
+#include <cstddef>
 #include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
-#include <cm/optional>
+#include <cm/string_view>
 
-class cmake;
+#include "cmMessenger.h"
+#include "cmSarif.h"
 
 /// @brief Manages SARIF logging for a CMake run
 ///
@@ -15,15 +21,37 @@ class cmake;
 class cmCMakeSarifLogger final
 {
 public:
-  cmCMakeSarifLogger(cmake& cm);
+  cmCMakeSarifLogger() = default;
   ~cmCMakeSarifLogger();
 
+  /// @brief Enable SARIF file generation at the given path when the logger is
+  /// destroyed.
+  void SetOutputPath(std::string const& path);
+
+  /// @brief Add a logical base directory used to emit relative paths.
+  ///
+  /// Artifact locations under a given path will be expressed according to
+  /// the named base directory. If multiple base directories are added to one
+  /// log, they are tested in order.
+  void AddBaseDirectory(cm::string_view name, cm::string_view path);
+
+  /// @brief Save CMake diagnostic messages to the SARIF log.
+  void RecordDiagnostics(std::vector<cmMessenger::Message> const& messages);
+
+  /// @brief Save information about the CMake process invocation.
+  void RecordInvocation(int ac, char const* const* av, int exitCode,
+                        std::chrono::system_clock::time_point startTime,
+                        std::chrono::system_clock::time_point endTime);
+
   void GenerateForRun() const;
 
 private:
-  bool WriteFile(std::string const& path,
-                 bool createParentDirectories = false) const;
-  cm::optional<std::string> FileOutputPath() const;
+  bool WriteFile(std::string const& path) const;
 
-  cmake const& CM;
+  std::string FilePath;
+  cmSarif::Run CMakeRun;
+  // Index of a diagnostic category appearing in the current run log.
+  std::unordered_map<std::string, std::size_t> RuleIndices;
+  // Maps logical name to actual path for base directories
+  std::vector<std::pair<std::string, std::string>> UriBaseIds;
 };

Source/cmSarif.cxx

@@ -178,6 +178,27 @@ Json::Value GetJson(Tool const& tool)
   return toolJson;
 }
 
+Json::Value GetJson(Invocation const& invocation)
+{
+  Json::Value obj(Json::objectValue);
+  Json::Value arguments(Json::arrayValue);
+  for (auto const& argument : invocation.Arguments) {
+    arguments.append(argument);
+  }
+  obj["arguments"] = arguments;
+  obj["executableLocation"] = cmSarif::GetJson(invocation.ExecutableLocation);
+  obj["startTimeUtc"] = invocation.StartTimeUtc;
+  obj["endTimeUtc"] = invocation.EndTimeUtc;
+  if (invocation.ExitCode) {
+    obj["exitCode"] = *invocation.ExitCode;
+  }
+  if (invocation.ExitSignalNumber) {
+    obj["exitSignalNumber"] = *invocation.ExitSignalNumber;
+  }
+  obj["executionSuccessful"] = invocation.ExecutionSuccessful;
+  return obj;
+}
+
 Json::Value GetJson(Run const& run)
 {
   Json::Value runJson(Json::objectValue);
@@ -191,6 +212,14 @@ Json::Value GetJson(Run const& run)
     runJson["originalUriBaseIds"] = uriBaseIds;
   }
 
+  if (!run.Invocations.empty()) {
+    Json::Value invocations(Json::arrayValue);
+    for (auto const& invocation : run.Invocations) {
+      invocations.append(cmSarif::GetJson(invocation));
+    }
+    runJson["invocations"] = invocations;
+  }
+
   Json::Value results(Json::arrayValue);
   for (auto const& result : run.Results) {
     results.append(cmSarif::GetJson(result));

Source/cmSarif.h

@@ -167,9 +167,23 @@ struct Tool
 
 Json::Value GetJson(Tool const& tool);
 
+struct Invocation
+{
+  std::vector<std::string> Arguments;
+  ArtifactLocation ExecutableLocation;
+  std::string StartTimeUtc;
+  std::string EndTimeUtc;
+  cm::optional<int> ExitCode;
+  cm::optional<int> ExitSignalNumber;
+  bool ExecutionSuccessful;
+};
+
+Json::Value GetJson(Invocation const& invocation);
+
 struct Run
 {
   cmSarif::Tool Tool;
+  std::vector<Invocation> Invocations;
   std::vector<Result> Results;
   std::unordered_map<std::string, ArtifactLocation> OriginalUriBaseIds;
 };

Source/cmake.cxx

@@ -84,7 +84,6 @@
 #  include <cm3p/json/writer.h>
 
 #  include "cmCMakePresetsArgs.h"
-#  include "cmCMakeSarifLogger.h"
 #  include "cmConfigureLog.h"
 #  include "cmFileAPI.h"
 #  include "cmGraphVizWriter.h"
@@ -1366,16 +1365,6 @@ void cmake::SetArgs(std::vector<std::string> const& args)
         state->SetIgnoreLinkWarningAsError(true);
         return true;
       } },
-#ifndef CMAKE_BOOTSTRAP
-    CommandArgument{ "--sarif-output", "No file specified for --sarif-output",
-                     CommandArgument::Values::One,
-                     [](std::string const& value, cmake* state) -> bool {
-                       state->SarifFilePath =
-                         cmSystemTools::ToNormalizedPathOnDisk(value);
-                       state->SarifFileOutput = true;
-                       return true;
-                     } },
-#endif
     CommandArgument{ "--debugger", CommandArgument::Values::Zero,
                      [](std::string const&, cmake* state) -> bool {
 #ifdef CMake_ENABLE_DEBUGGER
@@ -3137,8 +3126,9 @@ int cmake::Run(std::vector<std::string> const& args, bool noconfigure)
   }
 
 #ifndef CMAKE_BOOTSTRAP
-  // Configure the SARIF log for the current run
-  cmCMakeSarifLogger sarifLogger(*this);
+  if (this->State->GetRole() == cmState::Role::Project) {
+    this->MarkCliAsUsed("CMAKE_EXPORT_SARIF");
+  }
 
   this->VariableWatch->AddWatch("CMAKE_WARN_DEPRECATED", cmDeprecatedWatch);
   this->VariableWatch->AddWatch("CMAKE_ERROR_DEPRECATED", cmDeprecatedWatch);

Source/cmake.h

@@ -29,6 +29,7 @@
 #include "cmState.h"
 #include "cmStateSnapshot.h"
 #include "cmStateTypes.h"
+#include "cmStringAlgorithms.h"
 #include "cmValue.h"
 
 #if !defined(CMAKE_BOOTSTRAP)
@@ -599,11 +600,16 @@ public:
   cmMessenger* GetMessenger() const { return this->Messenger.get(); }
 
 #ifndef CMAKE_BOOTSTRAP
-  /// Get the SARIF file path if set manually for this run
-  cm::optional<std::string> GetSarifFilePath() const
+  /// Get the file path for SARIF logging if enabled by cache variable.
+  cm::optional<std::string> GetProjectSarifFile() const
   {
-    return (this->SarifFileOutput ? cm::make_optional(this->SarifFilePath)
-                                  : cm::nullopt);
+    if (this->State->GetRole() == cmState::Role::Project) {
+      if (this->GetCacheDefinition("CMAKE_EXPORT_SARIF").IsOn()) {
+        return cmStrCat(this->GetHomeOutputDirectory(),
+                        "/.cmake/sarif/cmake.sarif"_s);
+      }
+    }
+    return cm::nullopt;
   }
 #endif
 
@@ -850,11 +856,6 @@ private:
                        cmDiagnosticCategory category,
                        cmDiagnosticAction desiredAction, bool recurse);
 
-#ifndef CMAKE_BOOTSTRAP
-  bool SarifFileOutput = false;
-  std::string SarifFilePath;
-#endif
-
   std::vector<std::string> TraceOnlyThisSources;
 
   std::set<std::string> DebugFindPkgs;

Source/cmakemain.cxx

@@ -5,6 +5,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <chrono>
 #include <climits>
 #include <cstring>
 #include <functional>
@@ -33,6 +34,7 @@
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmMessageMetadata.h"
+#include "cmMessenger.h"
 #include "cmState.h"
 #include "cmStateTypes.h"
 #include "cmStdIoConsole.h"
@@ -46,6 +48,7 @@
 
 #ifndef CMAKE_BOOTSTRAP
 #  include "cmCMakePresetsArgs.h"
+#  include "cmCMakeSarifLogger.h"
 #  include "cmDocumentation.h"
 #endif
 
@@ -138,6 +141,8 @@ cmDocumentationEntry const cmDocumentationOptions[] = {
     "--profiling-format." }
 };
 
+cmCMakeSarifLogger cmSarifLogger;
+
 #endif
 
 int do_command(int ac, char const* const* av,
@@ -264,6 +269,9 @@ int do_cmake(int ac, char const* const* av)
   // (Regex) Filter on the cached variable(s) to print.
   std::string filter_var_name;
   bool view_only = false;
+#ifndef CMAKE_BOOTSTRAP
+  cm::optional<std::string> sarif_output_path;
+#endif
   cmState::Role role = cmState::Role::Project;
   std::vector<std::string> parsedArgs;
 
@@ -330,6 +338,15 @@ int do_cmake(int ac, char const* const* av)
                      } },
   };
 
+#ifndef CMAKE_BOOTSTRAP
+  arguments.emplace_back(
+    "--sarif-output", "No file specified for --sarif-output",
+    CommandArgument::Values::One, [&](std::string const& value) -> bool {
+      sarif_output_path = cmSystemTools::ToNormalizedPathOnDisk(value);
+      return true;
+    });
+#endif
+
   std::vector<std::string> inputArgs;
   inputArgs.reserve(ac);
   cm::append(inputArgs, av, av + ac);
@@ -410,6 +427,31 @@ int do_cmake(int ac, char const* const* av)
     }
   }
 
+#ifndef CMAKE_BOOTSTRAP
+  // If SARIF wasn't enabled on the CLI, see if it is enabled for the project.
+  if (!sarif_output_path && (sarif_output_path = cm.GetProjectSarifFile())) {
+    // Create parent directories when writing to the default location
+    cmSystemTools::MakeDirectory(
+      cmSystemTools::GetFilenamePath(*sarif_output_path));
+  }
+
+  if (sarif_output_path) {
+    cmSarifLogger.SetOutputPath(*sarif_output_path);
+
+    std::string const& binDir = cm.GetHomeOutputDirectory();
+    if (!binDir.empty()) {
+      cmSarifLogger.AddBaseDirectory("CMAKE_BINARY_DIR", binDir);
+    }
+
+    std::string const& homeDir = cm.GetHomeDirectory();
+    if (!homeDir.empty()) {
+      cmSarifLogger.AddBaseDirectory("CMAKE_SOURCE_DIR", homeDir);
+    }
+
+    cmSarifLogger.RecordDiagnostics(cm.GetMessenger()->GetDisplayedMessages());
+  }
+#endif
+
   // Always return a non-negative value (except exit code from SCRIPT_MODE).
   // Windows tools do not always interpret negative return values as errors.
   if (res != 0) {
@@ -1175,6 +1217,11 @@ int do_open(int ac, char const* const* av)
 
 int main(int ac, char const* const* av)
 {
+#ifndef CMAKE_BOOTSTRAP
+  std::chrono::system_clock::time_point wall_start_time =
+    std::chrono::system_clock::now();
+#endif
+
   cm::optional<cm::StdIo::Console> console = cm::StdIo::Console();
 
   cmsys::Encoding::CommandLineArguments args =
@@ -1212,5 +1259,9 @@ int main(int ac, char const* const* av)
   if (uv_loop_t* loop = uv_default_loop()) {
     uv_loop_close(loop);
   }
+#ifndef CMAKE_BOOTSTRAP
+  cmSarifLogger.RecordInvocation(ac, av, ret, wall_start_time,
+                                 std::chrono::system_clock::now());
+#endif
   return ret;
 }

Tests/RunCMake/SarifOutput/check-sarif.cmake

@@ -8,6 +8,11 @@ macro(check_sarif_output sarif_output_file expected_sarif_output_file)
   endif()
   file(READ "${sarif_output_file}" actual_output)
 
+  # The tool invocation data is specific to one run of CMake and should not be
+  # checked against the fixture. Extract it and perform some dynamic checks.
+  string(JSON sarif_tool_invocation GET "${actual_output}" runs 0 invocations 0)
+  string(JSON actual_output REMOVE "${actual_output}" runs 0 invocations)
+
   # Make sure the expected output file exists before reading it
   if (NOT EXISTS "${expected_sarif_output_file}")
     message(FATAL_ERROR "Expected SARIF output file not found: ${expected_sarif_output_file}")
@@ -16,4 +21,49 @@ macro(check_sarif_output sarif_output_file expected_sarif_output_file)
 
   # Check the actual output against the expected output
   check_json("${actual_output}" "${expected_output}")
+
+  # Check the claimed path to the CMake executable used in the test.
+  string(JSON sarif_invocation_exe GET "${sarif_tool_invocation}" executableLocation uri)
+  if (NOT sarif_invocation_exe STREQUAL "${CMAKE_COMMAND}")
+    string(APPEND RunCMake_TEST_FAILED
+      "Tool executable path in SARIF does not match actual CMake driver:"
+      "\n ${sarif_invocation_exe}\n expected:\n ${CMAKE_COMMAND}")
+  endif()
+
+  # Check that the timestamps are somewhat reasonable. Do not allow the epoch,
+  # and make sure the end time is not before the start time.
+  string(JSON sarif_invocation_start GET "${sarif_tool_invocation}" startTimeUtc)
+  string(JSON sarif_invocation_end GET "${sarif_tool_invocation}" endTimeUtc)
+  if (sarif_invocation_start MATCHES "^1970-01-01T00:00:00Z?$")
+    string(APPEND RunCMake_TEST_FAILED
+      "SARIF startTimeUtc for CMake invocation looks like the epoch:\n ${sarif_invocation_start}\n")
+  endif()
+  if (sarif_invocation_end MATCHES "^1970-01-01T00:00:00Z?$")
+    string(APPEND RunCMake_TEST_FAILED
+      "SARIF endTimeUtc for CMake invocation looks like the epoch:\n ${sarif_invocation_end}\n")
+  endif()
+  if (sarif_invocation_end STRLESS sarif_invocation_start)
+    string(APPEND RunCMake_TEST_FAILED
+      "SARIF invocation endTimeUtc is before startTimeUtc:\n"
+      " startTimeUtc:\n ${sarif_invocation_start}\n endTimeUtc:\n  ${sarif_invocation_end}\n")
+  endif()
+
+  # Check that the reported exit code matches the expectation for this test.
+  string(JSON sarif_invocation_exit_code GET "${sarif_tool_invocation}" exitCode)
+  if (NOT sarif_invocation_exit_code EQUAL "${expect_result}")
+    string(APPEND RunCMake_TEST_FAILED
+      "SARIF invocation exitCode does not match:\n"
+      " expected:\n ${expect_result}\n actual:\n ${sarif_invocation_exit_code}\n")
+  endif()
+
+  string(JSON sarif_invocation_success GET "${sarif_tool_invocation}" executionSuccessful)
+  set(sarif_invocation_expected_success "ON")
+  if (NOT "${expect_result}" EQUAL 0)
+    set(sarif_invocation_expected_success "OFF")
+  endif()
+  if (NOT sarif_invocation_success STREQUAL sarif_invocation_expected_success)
+    string(APPEND RunCMake_TEST_FAILED
+      "SARIF invocation executionSuccessful does not match:\n"
+      " expected:\n ${sarif_invocation_expected_success}\n actual:\n ${sarif_invocation_success}\n")
+  endif()
 endmacro()