instrumentation: Add JSON schema for query files

Issue: #27504

Help/manual/cmake-instrumentation.7.rst

@@ -265,6 +265,9 @@ be present in all snippet files. User written ``callbacks`` should be able to
 handle the presence of this optional data, since it may be requested by an
 unrelated query.
 
+The JSON format is described in machine-readable form by
+:download:`this JSON schema </manual/instrumentation/query-v1-schema.json>`.
+
 Example:
 
 .. code-block:: json

Help/manual/instrumentation/query-v1-schema.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "http://json-schema.org/2020-12/schema#",
+  "type": "object",
+  "required": ["version"],
+  "properties": {
+    "version": {
+      "type": "integer",
+      "description": "The data version of snippet file to generate.",
+      "enum": [
+        1
+      ]
+    },
+    "callbacks": {
+      "type": "array",
+      "description": "A list of command-line strings for callbacks to handle collected instrumentation data. Whenever these callbacks are executed, the full path to a v1 Index File is appended to the arguments included in the string.",
+      "items": {
+        "type": "string"
+      }
+    },
+    "hooks": {
+      "type": "array",
+      "description": "A list of strings specifying when indexing should occur automatically. These are the intervals when instrumentation data should be collated and user callbacks should be invoked to handle the data.",
+      "items": {
+        "enum": [
+          "postGenerate",
+          "preBuild",
+          "postBuild",
+          "preCMakeBuild",
+          "postCMakeBuild",
+          "postCMakeInstall",
+          "postCMakeWorkflow",
+          "postCTest"
+        ],
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "options": {
+      "type": "array",
+      "description": "A list of strings used to enable certain optional behavior, including the collection of certain additional data.",
+      "uniqueItems": true,
+      "items": {
+        "enum": [
+          "staticSystemInformation",
+          "dynamicSystemInformation",
+          "cdashSubmit",
+          "cdashVerbose",
+          "trace"
+        ],
+        "type": "string"
+      }
+    }
+  }
+}

Tests/RunCMake/CMakeLists.txt

@@ -429,7 +429,8 @@ add_RunCMake_test(FileAPI -DPython_EXECUTABLE=${Python_EXECUTABLE}
                           -DCMAKE_CXX_SIMULATE_ID=${CMAKE_CXX_SIMULATE_ID}
                           -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 if(CMAKE_GENERATOR MATCHES "Make|Ninja|FASTBuild")
-  add_RunCMake_test(Instrumentation)
+  add_RunCMake_test(Instrumentation -DPython_EXECUTABLE=${Python_EXECUTABLE}
+                                    -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 endif()
 add_RunCMake_test(ConfigDir)
 if(CMake_TEST_FindPython2)

Tests/RunCMake/Instrumentation/RunCMakeTest.cmake

@@ -22,6 +22,7 @@ function(instrument test)
     "PRESERVE_DATA"
     "NO_CONFIGURE"
     "FAIL"
+    "BAD_QUERY"
   )
   cmake_parse_arguments(ARGS "${OPTIONS}" "CHECK_SCRIPT;CONFIGURE_ARG" "" ${ARGN})
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test})
@@ -39,6 +40,13 @@ function(instrument test)
     file(REMOVE_RECURSE ${RunCMake_TEST_BINARY_DIR})
   endif()
 
+  if (ARGS_BAD_QUERY)
+    set(schema_validate_result 1)
+  else()
+    set(schema_validate_result 0)
+  endif()
+  set(schema_validate_result ${schema_validate_result} PARENT_SCOPE)
+
   # Set hook command
   set(static_query_hook_arg 0)
   if (ARGS_STATIC_QUERY)
@@ -176,15 +184,29 @@ function(instrument test)
 endfunction()
 
 # Bad Queries
-instrument(bad-option)
-instrument(bad-hook)
-instrument(empty)
-instrument(bad-version)
+instrument(bad-option BAD_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(bad-hook BAD_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(empty BAD_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(bad-version BAD_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
 
 # Verify Hooks Run and Index File
-instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY)
-instrument(hooks-2 BUILD INSTALL TEST)
-instrument(hooks-no-callbacks MANUAL_HOOK)
+instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(hooks-2 BUILD INSTALL TEST
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(hooks-no-callbacks MANUAL_HOOK
+  CHECK_SCRIPT check-query-dir.cmake
+)
 
 # Check data file contents for optional query data
 instrument(no-query

Tests/RunCMake/Instrumentation/check-data-dir.cmake

@@ -1,5 +1,6 @@
 include(${CMAKE_CURRENT_LIST_DIR}/verify-snippet.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/check-query-dir.cmake)
 
 file(GLOB snippets LIST_DIRECTORIES false ${v1}/data/*)
 if (NOT snippets)

Tests/RunCMake/Instrumentation/check-query-dir.cmake

@@ -0,0 +1,12 @@
+include(${CMAKE_CURRENT_LIST_DIR}/verify-snippet.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/validate_schema.cmake)
+file(GLOB_RECURSE queries LIST_DIRECTORIES false ${v1}/query/*)
+
+foreach(query ${queries})
+  validate_schema(
+    "${query}"
+    "${CMAKE_CURRENT_LIST_DIR}/../../../Help/manual/instrumentation/query-v1-schema.json"
+    "${schema_validate_result}"
+  )
+endforeach()

Tests/RunCMake/Instrumentation/json.cmake

@@ -1,3 +1,5 @@
+include_guard()
+
 # Read the JSON `filename` into `outvar`.
 function(read_json filename outvar)
   file(READ "${filename}" ${outvar})

Tests/RunCMake/Instrumentation/validate_schema.cmake

@@ -0,0 +1,15 @@
+function(validate_schema json_file schema_file expected_result)
+  if (Python_EXECUTABLE AND CMake_TEST_JSON_SCHEMA)
+    execute_process(
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/validate_schema.py" "${json_file}" "${schema_file}"
+      RESULT_VARIABLE result
+      OUTPUT_VARIABLE output
+      ERROR_VARIABLE output
+    )
+    if(NOT result MATCHES ${expected_result})
+      string(REPLACE "\n" "\n  " output "${output}")
+      string(APPEND RunCMake_TEST_FAILED "Failed to validate version JSON schema for file: ${json_file}\nOutput:\n${output}\nResult: ${result}\nExpected: ${expected_result}")
+    endif()
+    return(PROPAGATE RunCMake_TEST_FAILED)
+  endif()
+endfunction()

Tests/RunCMake/Instrumentation/validate_schema.py

@@ -0,0 +1,15 @@
+import json
+import jsonschema
+import sys
+
+with open(sys.argv[1], "r", encoding="utf-8-sig") as f:
+    contents = json.load(f)
+
+with open(sys.argv[2], "r", encoding="utf-8") as f:
+    schema = json.load(f)
+
+try:
+    jsonschema.validate(contents, schema)
+except jsonschema.ValidationError as e:
+    print(e)
+    sys.exit(1)

Tests/RunCMake/Instrumentation/verify-snippet.cmake

@@ -1,4 +1,5 @@
 # Performs generic (non-project specific) validation of v1 Snippet File Contents
+include_guard()
 
 include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)