From 079c86e0a48acc5fe1b0a3d904efa52a3674b2f2 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Sun, 15 Mar 2026 19:25:55 +0100 Subject: [PATCH] ExternalData: Add relocatable state metadata Issue: #27689 --- Help/release/dev/ExternalData-state-dir.rst | 6 ++ Modules/ExternalData.cmake | 59 ++++++++++++++++--- Modules/ExternalData_config.cmake.in | 1 + Tests/Module/ExternalData/CMakeLists.txt | 1 + .../ExternalData/DataStateRoot/CMakeLists.txt | 10 ++++ .../ExternalData/DataStateRoot/Data.dat.md5 | 1 + .../DataStateRoot/DataStateRootCheck.cmake | 9 +++ 7 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 Help/release/dev/ExternalData-state-dir.rst create mode 100644 Tests/Module/ExternalData/DataStateRoot/CMakeLists.txt create mode 100644 Tests/Module/ExternalData/DataStateRoot/Data.dat.md5 create mode 100644 Tests/Module/ExternalData/DataStateRoot/DataStateRootCheck.cmake diff --git a/Help/release/dev/ExternalData-state-dir.rst b/Help/release/dev/ExternalData-state-dir.rst new file mode 100644 index 0000000000..df61d4dcb3 --- /dev/null +++ b/Help/release/dev/ExternalData-state-dir.rst @@ -0,0 +1,6 @@ +ExternalData-state-dir +---------------------- + +* The :module:`ExternalData` module gained an + :variable:`ExternalData_STATE_ROOT` variable to store hash records outside + :variable:`ExternalData_BINARY_ROOT`. diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 02d8bd9467..6dc9ee5d2d 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -169,6 +169,16 @@ calling any of the functions provided by this module. some platforms. The ``ExternalData_NO_SYMLINKS`` variable may be set to disable use of symbolic links and enable use of copies instead. +.. variable:: ExternalData_STATE_ROOT + + .. versionadded:: 4.4 + + The ``ExternalData_STATE_ROOT`` variable may be set to place module-managed + metadata outside ``ExternalData_BINARY_ROOT``. When set, ``ExternalData`` + stores its hash records under this directory while still materializing real + data files under ``ExternalData_BINARY_ROOT``. By default, hash records + continue to be placed next to the materialized data. + .. variable:: ExternalData_OBJECT_STORES The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local @@ -541,7 +551,7 @@ function(ExternalData_add_target target) list(GET tuple 2 exts) string(REPLACE "+" ";" exts_list "${exts}") list(GET exts_list 0 first_ext) - set(stamp "-hash-stamp") + _ExternalData_compute_stamp_file("${file}" stamp_file) if(NOT DEFINED "_ExternalData_FILE_${file}") set("_ExternalData_FILE_${file}" 1) get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}") @@ -554,10 +564,11 @@ function(ExternalData_add_target target) # List the real file as a second output in case it is a broken link. # The files must be listed in this order so CMake can hide from the # make tool that a symlink target may not be newer than the input. - OUTPUT "${file}${stamp}" "${file}" + OUTPUT "${stamp_file}" "${file}" # Run the data fetch/update script. COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR} -Dfile=${file} -Dname=${name} -Dexts=${exts} + -Dstamp_file=${stamp_file} -DExternalData_ACTION=fetch -DExternalData_SHOW_PROGRESS=${_ExternalData_add_target_SHOW_PROGRESS} -DExternalData_CONFIG=${config} @@ -566,7 +577,7 @@ function(ExternalData_add_target target) MAIN_DEPENDENCY "${name}${first_ext}" ) endif() - list(APPEND files "${file}${stamp}") + list(APPEND files "${stamp_file}") endif() endforeach() @@ -636,12 +647,40 @@ function(_ExternalData_exact_regex regex_var string) endfunction() function(_ExternalData_atomic_write file content) + get_filename_component(dir "${file}" DIRECTORY) + file(MAKE_DIRECTORY "${dir}") _ExternalData_random(random) set(tmp "${file}.tmp${random}") file(WRITE "${tmp}" "${content}") file(RENAME "${tmp}" "${file}") endfunction() +function(_ExternalData_compute_stamp_file file var_stamp) + if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "") + _ExternalData_compute_state_relative_path("${file}" relfile) + set(stamp_file "${ExternalData_STATE_ROOT}/${relfile}-hash-stamp") + else() + set(stamp_file "${file}-hash-stamp") + endif() + set(${var_stamp} "${stamp_file}" PARENT_SCOPE) +endfunction() + +function(_ExternalData_compute_state_relative_path file var_relfile) + if(NOT ExternalData_BINARY_ROOT) + set(top_bin "${CMAKE_BINARY_DIR}") + else() + set(top_bin "${ExternalData_BINARY_ROOT}") + endif() + file(RELATIVE_PATH relfile "${top_bin}" "${file}") + if(IS_ABSOLUTE "${relfile}" OR "${relfile}" MATCHES "^\\.\\./") + message(FATAL_ERROR + "File path is not under ExternalData_BINARY_ROOT:\n" + " file = ${file}\n" + " ExternalData_BINARY_ROOT = ${top_bin}") + endif() + set(${var_relfile} "${relfile}" PARENT_SCOPE) +endfunction() + function(_ExternalData_link_content name var_ext) if("${ExternalData_LINK_CONTENT}" MATCHES "^(${_ExternalData_REGEX_ALGO})$") set(algo "${ExternalData_LINK_CONTENT}") @@ -1235,10 +1274,16 @@ if("${ExternalData_ACTION}" STREQUAL "fetch") message(FATAL_ERROR "${errorMsg}") endif() # Check if file already corresponds to the object. - set(stamp "-hash-stamp") + if(NOT DEFINED stamp_file OR "${stamp_file}" STREQUAL "") + if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "") + message(FATAL_ERROR + "stamp_file must be provided when ExternalData_STATE_ROOT is set") + endif() + set(stamp_file "${file}-hash-stamp") + endif() set(file_up_to_date 0) - if(EXISTS "${file}" AND EXISTS "${file}${stamp}") - file(READ "${file}${stamp}" f_hash) + if(EXISTS "${file}" AND EXISTS "${stamp_file}") + file(READ "${stamp_file}" f_hash) string(STRIP "${f_hash}" f_hash) if("${f_hash}" STREQUAL "${hash}") set(file_up_to_date 1) @@ -1253,7 +1298,7 @@ if("${ExternalData_ACTION}" STREQUAL "fetch") endif() # Atomically update the hash/timestamp file to record the object referenced. - _ExternalData_atomic_write("${file}${stamp}" "${hash}\n") + _ExternalData_atomic_write("${stamp_file}" "${hash}\n") elseif("${ExternalData_ACTION}" STREQUAL "local") foreach(v file name) if(NOT DEFINED "${v}") diff --git a/Modules/ExternalData_config.cmake.in b/Modules/ExternalData_config.cmake.in index 18be6b31ac..66785f0d25 100644 --- a/Modules/ExternalData_config.cmake.in +++ b/Modules/ExternalData_config.cmake.in @@ -3,4 +3,5 @@ set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@") set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@") set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@") set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@") +set(ExternalData_STATE_ROOT "@ExternalData_STATE_ROOT@") @_ExternalData_CONFIG_CODE@ diff --git a/Tests/Module/ExternalData/CMakeLists.txt b/Tests/Module/ExternalData/CMakeLists.txt index c09c693005..bda1149a2c 100644 --- a/Tests/Module/ExternalData/CMakeLists.txt +++ b/Tests/Module/ExternalData/CMakeLists.txt @@ -57,3 +57,4 @@ add_subdirectory(Data3) add_subdirectory(Data4) add_subdirectory(Data5) add_subdirectory(DataNoSymlinks) +add_subdirectory(DataStateRoot) diff --git a/Tests/Module/ExternalData/DataStateRoot/CMakeLists.txt b/Tests/Module/ExternalData/DataStateRoot/CMakeLists.txt new file mode 100644 index 0000000000..eb8595f44a --- /dev/null +++ b/Tests/Module/ExternalData/DataStateRoot/CMakeLists.txt @@ -0,0 +1,10 @@ +set(ExternalData_STATE_ROOT "${CMAKE_CURRENT_BINARY_DIR}/State") +ExternalData_Add_Test(DataStateRoot + NAME DataStateRootCheck + COMMAND ${CMAKE_COMMAND} + -D Data=DATA{Data.dat} + -D AdjacentHash=${ExternalData_BINARY_ROOT}/DataStateRoot/Data.dat-hash-stamp + -D StateHash=${ExternalData_STATE_ROOT}/DataStateRoot/Data.dat-hash-stamp + -P ${CMAKE_CURRENT_SOURCE_DIR}/DataStateRootCheck.cmake + ) +ExternalData_Add_Target(DataStateRoot) diff --git a/Tests/Module/ExternalData/DataStateRoot/Data.dat.md5 b/Tests/Module/ExternalData/DataStateRoot/Data.dat.md5 new file mode 100644 index 0000000000..70e39bd5d1 --- /dev/null +++ b/Tests/Module/ExternalData/DataStateRoot/Data.dat.md5 @@ -0,0 +1 @@ +8c018830e3efa5caf3c7415028335a57 diff --git a/Tests/Module/ExternalData/DataStateRoot/DataStateRootCheck.cmake b/Tests/Module/ExternalData/DataStateRoot/DataStateRootCheck.cmake new file mode 100644 index 0000000000..869c87705c --- /dev/null +++ b/Tests/Module/ExternalData/DataStateRoot/DataStateRootCheck.cmake @@ -0,0 +1,9 @@ +if(NOT EXISTS "${Data}") + message(SEND_ERROR "Input file:\n ${Data}\n" "does not exist!") +endif() +if(EXISTS "${AdjacentHash}") + message(SEND_ERROR "Adjacent hash record exists but should not:\n ${AdjacentHash}") +endif() +if(NOT EXISTS "${StateHash}") + message(SEND_ERROR "State-root hash record is missing:\n ${StateHash}") +endif()