ExternalData: Add relocatable state metadata

Issue: #27689
This commit is contained in:
Arkadiusz Lachowicz
2026-03-15 19:27:16 +01:00
parent 9b30acb6ba
commit 079c86e0a4
7 changed files with 80 additions and 7 deletions
@@ -0,0 +1,6 @@
ExternalData-state-dir
----------------------
* The :module:`ExternalData` module gained an
:variable:`ExternalData_STATE_ROOT` variable to store hash records outside
:variable:`ExternalData_BINARY_ROOT`.
+52 -7
View File
@@ -169,6 +169,16 @@ calling any of the functions provided by this module.
some platforms. The ``ExternalData_NO_SYMLINKS`` variable may be set
to disable use of symbolic links and enable use of copies instead.
.. variable:: ExternalData_STATE_ROOT
.. versionadded:: 4.4
The ``ExternalData_STATE_ROOT`` variable may be set to place module-managed
metadata outside ``ExternalData_BINARY_ROOT``. When set, ``ExternalData``
stores its hash records under this directory while still materializing real
data files under ``ExternalData_BINARY_ROOT``. By default, hash records
continue to be placed next to the materialized data.
.. variable:: ExternalData_OBJECT_STORES
The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local
@@ -541,7 +551,7 @@ function(ExternalData_add_target target)
list(GET tuple 2 exts)
string(REPLACE "+" ";" exts_list "${exts}")
list(GET exts_list 0 first_ext)
set(stamp "-hash-stamp")
_ExternalData_compute_stamp_file("${file}" stamp_file)
if(NOT DEFINED "_ExternalData_FILE_${file}")
set("_ExternalData_FILE_${file}" 1)
get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
@@ -554,10 +564,11 @@ function(ExternalData_add_target target)
# List the real file as a second output in case it is a broken link.
# The files must be listed in this order so CMake can hide from the
# make tool that a symlink target may not be newer than the input.
OUTPUT "${file}${stamp}" "${file}"
OUTPUT "${stamp_file}" "${file}"
# Run the data fetch/update script.
COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
-Dfile=${file} -Dname=${name} -Dexts=${exts}
-Dstamp_file=${stamp_file}
-DExternalData_ACTION=fetch
-DExternalData_SHOW_PROGRESS=${_ExternalData_add_target_SHOW_PROGRESS}
-DExternalData_CONFIG=${config}
@@ -566,7 +577,7 @@ function(ExternalData_add_target target)
MAIN_DEPENDENCY "${name}${first_ext}"
)
endif()
list(APPEND files "${file}${stamp}")
list(APPEND files "${stamp_file}")
endif()
endforeach()
@@ -636,12 +647,40 @@ function(_ExternalData_exact_regex regex_var string)
endfunction()
function(_ExternalData_atomic_write file content)
get_filename_component(dir "${file}" DIRECTORY)
file(MAKE_DIRECTORY "${dir}")
_ExternalData_random(random)
set(tmp "${file}.tmp${random}")
file(WRITE "${tmp}" "${content}")
file(RENAME "${tmp}" "${file}")
endfunction()
function(_ExternalData_compute_stamp_file file var_stamp)
if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "")
_ExternalData_compute_state_relative_path("${file}" relfile)
set(stamp_file "${ExternalData_STATE_ROOT}/${relfile}-hash-stamp")
else()
set(stamp_file "${file}-hash-stamp")
endif()
set(${var_stamp} "${stamp_file}" PARENT_SCOPE)
endfunction()
function(_ExternalData_compute_state_relative_path file var_relfile)
if(NOT ExternalData_BINARY_ROOT)
set(top_bin "${CMAKE_BINARY_DIR}")
else()
set(top_bin "${ExternalData_BINARY_ROOT}")
endif()
file(RELATIVE_PATH relfile "${top_bin}" "${file}")
if(IS_ABSOLUTE "${relfile}" OR "${relfile}" MATCHES "^\\.\\./")
message(FATAL_ERROR
"File path is not under ExternalData_BINARY_ROOT:\n"
" file = ${file}\n"
" ExternalData_BINARY_ROOT = ${top_bin}")
endif()
set(${var_relfile} "${relfile}" PARENT_SCOPE)
endfunction()
function(_ExternalData_link_content name var_ext)
if("${ExternalData_LINK_CONTENT}" MATCHES "^(${_ExternalData_REGEX_ALGO})$")
set(algo "${ExternalData_LINK_CONTENT}")
@@ -1235,10 +1274,16 @@ if("${ExternalData_ACTION}" STREQUAL "fetch")
message(FATAL_ERROR "${errorMsg}")
endif()
# Check if file already corresponds to the object.
set(stamp "-hash-stamp")
if(NOT DEFINED stamp_file OR "${stamp_file}" STREQUAL "")
if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "")
message(FATAL_ERROR
"stamp_file must be provided when ExternalData_STATE_ROOT is set")
endif()
set(stamp_file "${file}-hash-stamp")
endif()
set(file_up_to_date 0)
if(EXISTS "${file}" AND EXISTS "${file}${stamp}")
file(READ "${file}${stamp}" f_hash)
if(EXISTS "${file}" AND EXISTS "${stamp_file}")
file(READ "${stamp_file}" f_hash)
string(STRIP "${f_hash}" f_hash)
if("${f_hash}" STREQUAL "${hash}")
set(file_up_to_date 1)
@@ -1253,7 +1298,7 @@ if("${ExternalData_ACTION}" STREQUAL "fetch")
endif()
# Atomically update the hash/timestamp file to record the object referenced.
_ExternalData_atomic_write("${file}${stamp}" "${hash}\n")
_ExternalData_atomic_write("${stamp_file}" "${hash}\n")
elseif("${ExternalData_ACTION}" STREQUAL "local")
foreach(v file name)
if(NOT DEFINED "${v}")
+1
View File
@@ -3,4 +3,5 @@ set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@")
set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@")
set(ExternalData_STATE_ROOT "@ExternalData_STATE_ROOT@")
@_ExternalData_CONFIG_CODE@
+1
View File
@@ -57,3 +57,4 @@ add_subdirectory(Data3)
add_subdirectory(Data4)
add_subdirectory(Data5)
add_subdirectory(DataNoSymlinks)
add_subdirectory(DataStateRoot)
@@ -0,0 +1,10 @@
set(ExternalData_STATE_ROOT "${CMAKE_CURRENT_BINARY_DIR}/State")
ExternalData_Add_Test(DataStateRoot
NAME DataStateRootCheck
COMMAND ${CMAKE_COMMAND}
-D Data=DATA{Data.dat}
-D AdjacentHash=${ExternalData_BINARY_ROOT}/DataStateRoot/Data.dat-hash-stamp
-D StateHash=${ExternalData_STATE_ROOT}/DataStateRoot/Data.dat-hash-stamp
-P ${CMAKE_CURRENT_SOURCE_DIR}/DataStateRootCheck.cmake
)
ExternalData_Add_Target(DataStateRoot)
@@ -0,0 +1 @@
8c018830e3efa5caf3c7415028335a57
@@ -0,0 +1,9 @@
if(NOT EXISTS "${Data}")
message(SEND_ERROR "Input file:\n ${Data}\n" "does not exist!")
endif()
if(EXISTS "${AdjacentHash}")
message(SEND_ERROR "Adjacent hash record exists but should not:\n ${AdjacentHash}")
endif()
if(NOT EXISTS "${StateHash}")
message(SEND_ERROR "State-root hash record is missing:\n ${StateHash}")
endif()