From 7cca8540ca98f29e1e6bd9f8294e4071f0b75791 Mon Sep 17 00:00:00 2001 From: Daksh Mamodiya Date: Fri, 7 Aug 2026 17:39:46 +0200 Subject: [PATCH] string(JSON): Add ARRAY_SPLIT mode for linear array iteration Iterating a JSON array with GET re-parses the entire string on every call, so walking N elements is O(N^2). ARRAY_SPLIT parses the array once and returns its elements as a CMake list, letting each element be queried individually in linear total time. Each element is re-serialized as compact JSON and encoded to survive CMake list parsing: '[' and ']' inside strings are emitted as \u005B and \u005D, and ';' is escaped as '\;'. Every element therefore stays valid, re-queryable JSON. Fixes: #27985 --- Help/command/string.rst | 36 +++ Help/release/dev/string-json-array-split.rst | 7 + Source/cmStringCommand.cxx | 65 +++++- Tests/RunCMake/string/JSON.cmake | 209 ++++++++++++++++++ .../string/JSONArraySplitNoArray-result.txt | 1 + .../string/JSONArraySplitNoArray-stderr.txt | 5 + .../string/JSONArraySplitNoArray.cmake | 1 + .../RunCMake/string/JSONWrongMode-stderr.txt | 4 +- Tests/RunCMake/string/RunCMakeTest.cmake | 1 + 9 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 Help/release/dev/string-json-array-split.rst create mode 100644 Tests/RunCMake/string/JSONArraySplitNoArray-result.txt create mode 100644 Tests/RunCMake/string/JSONArraySplitNoArray-stderr.txt create mode 100644 Tests/RunCMake/string/JSONArraySplitNoArray.cmake diff --git a/Help/command/string.rst b/Help/command/string.rst index 3fe9a7ef09..35754f8ab3 100644 --- a/Help/command/string.rst +++ b/Help/command/string.rst @@ -48,6 +48,9 @@ Synopsis string(JSON [ERROR_VARIABLE ] {`GET `__ | `GET_RAW `__ | `TYPE `__ | `LENGTH `__} [ ...]) + string(JSON [ERROR_VARIABLE ] + `ARRAY_SPLIT `__ + [ ...]) string(JSON [ERROR_VARIABLE ] `REMOVE `__ [ ...]) @@ -607,6 +610,39 @@ string is passed as a single argument even if it contains semicolons. given by the list of ```` arguments. Requires an element of array or object type. +.. signature:: + string(JSON [ERROR_VARIABLE ] + ARRAY_SPLIT [ ...]) + :target: JSON-ARRAY_SPLIT + + .. versionadded:: 4.5 + + Split the array in ```` at the location given by the list of + ```` arguments into a :ref:`semicolon-separated list `, one entry per array element. Requires an element of array + type. + + This enables iterating an array with a single parse of ````, + instead of re-parsing the whole string for each element: + + .. code-block:: cmake + + string(JSON elements ARRAY_SPLIT "${json}") + foreach(element IN LISTS elements) + string(JSON value GET "${element}" some_member) + endforeach() + + Each entry is the corresponding element re-serialized as compact JSON. It is + *semantically* equal to the source element (as compared by + :cref:`EQUAL `), but not necessarily byte-for-byte identical: + insignificant whitespace is removed and object members may be reordered. + + The result is a regular CMake list and should be consumed with list-aware + constructs such as :command:`foreach(IN LISTS) ` or the + :command:`list` command. Elements may contain characters that are encoded to + keep the list well-formed, so operating on the raw ```` value with + plain string operations may expose that internal encoding. + .. signature:: string(JSON [ERROR_VARIABLE ] REMOVE [ ...]) diff --git a/Help/release/dev/string-json-array-split.rst b/Help/release/dev/string-json-array-split.rst new file mode 100644 index 0000000000..bde81a08f7 --- /dev/null +++ b/Help/release/dev/string-json-array-split.rst @@ -0,0 +1,7 @@ +string-json-array-split +----------------------- + +* The :command:`string(JSON)` command gained an ``ARRAY_SPLIT`` mode that + splits a JSON array into a list of its elements with a single parse, + enabling linear-time array iteration instead of re-parsing the whole + JSON string for each element. diff --git a/Source/cmStringCommand.cxx b/Source/cmStringCommand.cxx index acda83e42d..e175fe4eff 100644 --- a/Source/cmStringCommand.cxx +++ b/Source/cmStringCommand.cxx @@ -926,6 +926,41 @@ std::string WriteJson(Json::Value const& value) return Json::writeString(writer, value); } +// Rewrite a compact JSON value so it survives CMake's list grammar as one +// element. CMake list parsing splits on ';' only at bracket-nesting depth +// zero and un-escapes only '\;', never '[' or ']'. So inside string tokens +// emit '[' and ']' as the JSON escapes \u005B and \u005D (the list nesting +// counter never sees a literal bracket), and escape every ';' as '\;'. Both +// stay valid JSON that string(JSON) decodes back to the original characters. +std::string EncodeJsonListElement(std::string const& element) +{ + std::string result; + result.reserve(element.size()); + bool inString = false; + bool escaped = false; + for (char const c : element) { + if (escaped) { + result += c; + escaped = false; + } else if (c == '\\') { + escaped = true; + result += c; + } else if (c == '"') { + inString = !inString; + result += c; + } else if (inString && c == '[') { + result += "\\u005B"; + } else if (inString && c == ']') { + result += "\\u005D"; + } else if (c == ';') { + result += "\\;"; + } else { + result += c; + } + } + return result; +} + bool JsonPartialMatch(Json::Value const& pattern, Json::Value const& actual) { if (pattern.type() != actual.type()) { @@ -1001,13 +1036,13 @@ bool HandleJSONCommand(std::vector const& arguments, auto const& mode = args.PopFront("missing mode argument"_s); if (mode != "GET"_s && mode != "GET_RAW"_s && mode != "TYPE"_s && - mode != "MEMBER"_s && mode != "LENGTH"_s && mode != "REMOVE"_s && - mode != "SET"_s && mode != "EQUAL"_s && mode != "STRING_ENCODE"_s && - mode != "PARTIAL_EQUAL"_s) { + mode != "MEMBER"_s && mode != "LENGTH"_s && mode != "ARRAY_SPLIT"_s && + mode != "REMOVE"_s && mode != "SET"_s && mode != "EQUAL"_s && + mode != "STRING_ENCODE"_s && mode != "PARTIAL_EQUAL"_s) { throw json_error(cmStrCat( "got an invalid mode '"_s, mode, - "', expected one of GET, GET_RAW, TYPE, MEMBER, LENGTH, REMOVE, SET, " - " EQUAL, PARTIAL_EQUAL, STRING_ENCODE"_s)); + "', expected one of GET, GET_RAW, TYPE, MEMBER, LENGTH, ARRAY_SPLIT, " + "REMOVE, SET, EQUAL, PARTIAL_EQUAL, STRING_ENCODE"_s)); } auto const& jsonstr = args.PopFront("missing json string argument"_s); @@ -1064,6 +1099,26 @@ bool HandleJSONCommand(std::vector const& arguments, cmAlphaNum sizeStr{ value.size() }; makefile.AddDefinition(*outputVariable, sizeStr.View()); + } else if (mode == "ARRAY_SPLIT"_s) { + auto const& value = ResolvePath(json, args); + if (!value.isArray()) { + throw json_error(cmStrCat("ARRAY_SPLIT needs to be called with an " + "element of type ARRAY, got "_s, + JsonTypeToString(value.type())), + args); + } + + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; + writer["commentStyle"] = "None"; + std::vector elements; + elements.reserve(value.size()); + for (Json::Value const& element : value) { + elements.push_back( + EncodeJsonListElement(Json::writeString(writer, element))); + } + makefile.AddDefinition(*outputVariable, cmList::to_string(elements)); + } else if (mode == "REMOVE"_s) { auto const& toRemove = args.PopBack("missing member or index to remove"_s); diff --git a/Tests/RunCMake/string/JSON.cmake b/Tests/RunCMake/string/JSON.cmake index f836310a5e..7f8910c1fb 100644 --- a/Tests/RunCMake/string/JSON.cmake +++ b/Tests/RunCMake/string/JSON.cmake @@ -711,3 +711,212 @@ string(JSON result PARTIAL_EQUAL if(result) message(SEND_ERROR "Expected OFF got ${result} for duplicate array mismatch in object") endif() + +# Test ARRAY_SPLIT + +# Split the top-level array in , assert the element count matches LENGTH +# and each element is semantically equal to the corresponding source element, +# then return the split list in (if given). Elements are always read +# back through a list-aware reader so the encoded list is exercised end-to-end. +function(assert_array_split json) + string(JSON _split ERROR_VARIABLE _err ARRAY_SPLIT "${json}") + if(_err) + message(SEND_ERROR "Unexpected ARRAY_SPLIT error: ${_err}\n for: ${json}") + return() + endif() + string(JSON _len LENGTH "${json}") + list(LENGTH _split _n) + if(NOT _n EQUAL _len) + message(SEND_ERROR "ARRAY_SPLIT gave ${_n} elements, expected ${_len}\n for: ${json}") + return() + endif() + set(_i 0) + foreach(_element IN LISTS _split) + string(JSON _src GET_RAW "${json}" ${_i}) + string(JSON _eq ERROR_VARIABLE _eqerr EQUAL "${_element}" "${_src}") + if(_eqerr) + message(SEND_ERROR "ARRAY_SPLIT element ${_i} is not valid JSON: ${_eqerr}\n element: ${_element}") + elseif(NOT _eq) + message(SEND_ERROR "ARRAY_SPLIT element ${_i}\n ${_element}\n not equal to source\n ${_src}") + endif() + math(EXPR _i "${_i} + 1") + endforeach() + if(ARGC GREATER 1) + set(${ARGV1} "${_split}" PARENT_SCOPE) + endif() +endfunction() + +# Basic array of objects: cardinality, order, and member access. +assert_array_split([=[ +[ + { "A": 1, "B": "one" }, + { "A": 2, "B": "two" }, + { "A": 3, "B": "three" } +] +]=] elements) +list(GET elements 0 element0) +string(JSON a GET "${element0}" A) +assert_strequal("${a}" 1) +string(JSON b GET "${element0}" B) +assert_strequal("${b}" one) +list(GET elements 2 element2) +string(JSON b GET "${element2}" B) +assert_strequal("${b}" three) + +# Empty array, top-level and via a path. +assert_array_split("[]" empty) +list(LENGTH empty emptyLen) +assert_strequal("${emptyLen}" 0) +string(JSON pathEmpty ERROR_VARIABLE error ARRAY_SPLIT [=[{"data":[]}]=] data) +if(error) + message(SEND_ERROR "Unexpected error: ${error}") +endif() +list(LENGTH pathEmpty pathEmptyLen) +assert_strequal("${pathEmptyLen}" 0) + +# Path-located array. +set(doc [=[{ "outer": { "data": [10, 20, 30] } }]=]) +string(JSON pathSplit ERROR_VARIABLE error ARRAY_SPLIT "${doc}" outer data) +if(error) + message(SEND_ERROR "Unexpected error: ${error}") +endif() +list(LENGTH pathSplit pathSplitLen) +assert_strequal("${pathSplitLen}" 3) +list(GET pathSplit 1 pathElement1) +assert_strequal("${pathElement1}" 20) + +# Scalars: cardinality preserved (no empty-element collapsing) and types kept. +assert_array_split([=[[null, false, 0, ""]]=] scalars) +list(LENGTH scalars scalarsLen) +assert_strequal("${scalarsLen}" 4) +list(GET scalars 0 scalar0) +string(JSON scalarType TYPE "${scalar0}") +assert_strequal("${scalarType}" NULL) +list(GET scalars 1 scalar1) +string(JSON scalarType TYPE "${scalar1}") +assert_strequal("${scalarType}" BOOLEAN) +list(GET scalars 2 scalar2) +string(JSON scalarType TYPE "${scalar2}") +assert_strequal("${scalarType}" NUMBER) +list(GET scalars 3 scalar3) +string(JSON scalarType TYPE "${scalar3}") +assert_strequal("${scalarType}" STRING) + +# Numbers: integer, real, exponent, and large integer round-trip. +assert_array_split([=[[1, 1000.0, 1e3, 1234567890]]=]) + +# Nested arrays and objects: structural brackets left intact. +assert_array_split([=[[[1,2],[3,4]]]=] nested) +list(GET nested 0 nested0) +string(JSON nestedType TYPE "${nested0}") +assert_strequal("${nestedType}" ARRAY) +string(JSON nestedVal GET "${nested0}" 1) +assert_strequal("${nestedVal}" 2) +assert_array_split([=[[{"x":1},{"y":2}]]=] objects) +list(GET objects 1 objects1) +string(JSON objectsType TYPE "${objects1}") +assert_strequal("${objectsType}" OBJECT) +string(JSON objectsVal GET "${objects1}" y) +assert_strequal("${objectsVal}" 2) + +# Whitespace and object-member-order canonicalization compare EQUAL. +assert_array_split([=[[ { "b" : 1 , "a" : 2 } ]]=]) + +# Adversarial: ';' inside a string value must not split the element. +assert_array_split([=[[{"cmd":"a;b"},{"d":4}]]=] semicolon) +list(LENGTH semicolon semicolonLen) +assert_strequal("${semicolonLen}" 2) +list(GET semicolon 0 semicolon0) +string(JSON semicolonVal GET "${semicolon0}" cmd) +assert_strequal("${semicolonVal}" "a;b") + +# Adversarial: an unbalanced '[' or ']' inside a string must not merge elements. +assert_array_split([=[[{"a":"["},{"b":2}]]=] openBracket) +list(LENGTH openBracket openBracketLen) +assert_strequal("${openBracketLen}" 2) +list(GET openBracket 0 openBracket0) +string(JSON openBracketVal GET "${openBracket0}" a) +assert_strequal("${openBracketVal}" "[") +assert_array_split([=[[{"a":"]"},{"b":2}]]=] closeBracket) +list(LENGTH closeBracket closeBracketLen) +assert_strequal("${closeBracketLen}" 2) +list(GET closeBracket 0 closeBracket0) +string(JSON closeBracketVal GET "${closeBracket0}" a) +assert_strequal("${closeBracketVal}" "]") + +# Adversarial: several brackets in one string value. +assert_array_split([=[[{"s":"[[]]["},{"z":0}]]=] brackets) +list(GET brackets 0 brackets0) +string(JSON bracketsVal GET "${brackets0}" s) +assert_strequal("${bracketsVal}" "[[]][") + +# Adversarial: a bracket in an object *key* (keys are strings too). +assert_array_split([=[[{"[k]":1},{"y":2}]]=] keyBracket) +list(GET keyBracket 0 keyBracket0) +string(JSON keyBracketVal GET "${keyBracket0}" "[k]") +assert_strequal("${keyBracketVal}" 1) +string(JSON keyBracketName MEMBER "${keyBracket0}" 0) +assert_strequal("${keyBracketName}" "[k]") + +# Adversarial: ';' adjacent to backslashes (value is a\;b). +assert_array_split([==[[{"cmd":"a\\;b"},{"d":4}]]==] semiBackslash) +list(GET semiBackslash 0 semiBackslash0) +string(JSON semiBackslashVal GET "${semiBackslash0}" cmd) +assert_strequal("${semiBackslashVal}" [==[a\;b]==]) + +# Adversarial: an escaped quote before a bracket (value is x"[y). +assert_array_split([==[[{"a":"x\"[y"},{"b":2}]]==] quoteBracket) +list(GET quoteBracket 0 quoteBracket0) +string(JSON quoteBracketVal GET "${quoteBracket0}" a) +assert_strequal("${quoteBracketVal}" [==[x"[y]==]) + +# Adversarial: a string value ending in an escaped backslash (value is x\). +assert_array_split([==[[{"a":"x\\"},{"b":2}]]==] endBackslash) +list(GET endBackslash 0 endBackslash0) +string(JSON endBackslashVal GET "${endBackslash0}" a) +assert_strequal("${endBackslashVal}" [==[x\]==]) + +# Adversarial: literal "\u005B" text (six characters) must stay unchanged. +assert_array_split([==[[{"a":"\\u005B"},{"b":2}]]==] literalEscape) +list(GET literalEscape 0 literalEscape0) +string(JSON literalEscapeVal GET "${literalEscape0}" a) +assert_strequal("${literalEscapeVal}" [==[\u005B]==]) + +# Unicode survives compact re-serialization (reuse the unicode fixture). +file(READ ${CMAKE_CURRENT_LIST_DIR}/json/unicode.json unicode) +assert_array_split("[${unicode}]" unicodeSplit) +list(GET unicodeSplit 0 unicodeElement) +string(JSON unicodeVal GET "${unicodeElement}" datalinkescape) +string(JSON unicodeSrc GET "${unicode}" datalinkescape) +assert_strequal("${unicodeVal}" "${unicodeSrc}") + +# Error handling with ERROR_VARIABLE: non-array target must mirror the NOTFOUND +# form that LENGTH produces for the same input, and report an ARRAY message. + +# Non-array (string) at top level. +string(JSON asResult ERROR_VARIABLE asError ARRAY_SPLIT "\"text\"") +string(JSON lenResult ERROR_VARIABLE lenError LENGTH "\"text\"") +assert_strequal("${asResult}" "${lenResult}") +assert_strequal("${asError}" "ARRAY_SPLIT needs to be called with an element of type ARRAY, got STRING") + +# Object at top level is also rejected (LENGTH would accept it, so no cross-check +# on the message, but the NOTFOUND form matches the string case above). +string(JSON objResult ERROR_VARIABLE objError ARRAY_SPLIT "{}") +assert_strequal("${objResult}" "${asResult}") +assert_strequal("${objError}" "ARRAY_SPLIT needs to be called with an element of type ARRAY, got OBJECT") + +# Non-array (string) at a path. +string(JSON pathResult ERROR_VARIABLE pathError ARRAY_SPLIT [=[{"data":"text"}]=] data) +string(JSON pathLenResult ERROR_VARIABLE pathLenError LENGTH [=[{"data":"text"}]=] data) +assert_strequal("${pathResult}" "${pathLenResult}") +assert_strequal("${pathResult}" "data-NOTFOUND") +assert_strequal("${pathError}" "ARRAY_SPLIT needs to be called with an element of type ARRAY, got STRING") + +# Non-existent path mirrors LENGTH's -NOTFOUND and reports an error. +string(JSON missResult ERROR_VARIABLE missError ARRAY_SPLIT [=[{"a":1}]=] b) +string(JSON missLenResult ERROR_VARIABLE missLenError LENGTH [=[{"a":1}]=] b) +assert_strequal("${missResult}" "${missLenResult}") +assert_strequal("${missResult}" "b-NOTFOUND") +if(NOT missError) + message(SEND_ERROR "Expected an error for a non-existent ARRAY_SPLIT path") +endif() diff --git a/Tests/RunCMake/string/JSONArraySplitNoArray-result.txt b/Tests/RunCMake/string/JSONArraySplitNoArray-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/string/JSONArraySplitNoArray-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/string/JSONArraySplitNoArray-stderr.txt b/Tests/RunCMake/string/JSONArraySplitNoArray-stderr.txt new file mode 100644 index 0000000000..cddcde0e3c --- /dev/null +++ b/Tests/RunCMake/string/JSONArraySplitNoArray-stderr.txt @@ -0,0 +1,5 @@ +CMake Error at JSONArraySplitNoArray\.cmake:1 \(string\): + string sub-command JSON ARRAY_SPLIT needs to be called with an element of + type ARRAY, got NUMBER\. +Call Stack \(most recent call first\): + CMakeLists\.txt:[0-9]+ \(include\) diff --git a/Tests/RunCMake/string/JSONArraySplitNoArray.cmake b/Tests/RunCMake/string/JSONArraySplitNoArray.cmake new file mode 100644 index 0000000000..84af3406cc --- /dev/null +++ b/Tests/RunCMake/string/JSONArraySplitNoArray.cmake @@ -0,0 +1 @@ +string(JSON var ARRAY_SPLIT "5") diff --git a/Tests/RunCMake/string/JSONWrongMode-stderr.txt b/Tests/RunCMake/string/JSONWrongMode-stderr.txt index 41ecc12bfc..e3d5b8a2e8 100644 --- a/Tests/RunCMake/string/JSONWrongMode-stderr.txt +++ b/Tests/RunCMake/string/JSONWrongMode-stderr.txt @@ -1,6 +1,6 @@ CMake Error at JSONWrongMode\.cmake:1 \(string\): string sub-command JSON got an invalid mode 'FOO', expected one of GET, - GET_RAW, TYPE, MEMBER, LENGTH, REMOVE, SET, EQUAL, PARTIAL_EQUAL, - STRING_ENCODE\. + GET_RAW, TYPE, MEMBER, LENGTH, ARRAY_SPLIT, REMOVE, SET, EQUAL, + PARTIAL_EQUAL, STRING_ENCODE\. Call Stack \(most recent call first\): CMakeLists\.txt:[0-9]+ \(include\) diff --git a/Tests/RunCMake/string/RunCMakeTest.cmake b/Tests/RunCMake/string/RunCMakeTest.cmake index 80e5319479..39714d6262 100644 --- a/Tests/RunCMake/string/RunCMakeTest.cmake +++ b/Tests/RunCMake/string/RunCMakeTest.cmake @@ -7,6 +7,7 @@ run_cmake(JSONWrongMode) run_cmake(JSONOneArg) run_cmake(JSONNoArgs) run_cmake(JSONBadJson) +run_cmake(JSONArraySplitNoArray) run_cmake(Append) run_cmake(AppendNoArgs)