From 0db2ea97047ceb78f8764ca1196c8257da65c247 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 17 Mar 2026 16:11:12 +0100 Subject: [PATCH] ExternalData: Add configurable link materialization mode Fixes: #27700 --- Help/release/dev/ExternalData-link-mode.rst | 7 + Modules/ExternalData.cmake | 136 +++++++++++++++--- Modules/ExternalData_config.cmake.in | 1 + .../ExternalData/BadLinkMode-result.txt | 1 + .../ExternalData/BadLinkMode-stderr.txt | 2 + .../BadLinkModeNoSymlinks-result.txt | 1 + .../BadLinkModeNoSymlinks-stderr.txt | 2 + .../ExternalData/GoodLinkModeList-result.txt | 1 + .../RunCMake/ExternalData/RunCMakeTest.cmake | 27 ++++ 9 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 Help/release/dev/ExternalData-link-mode.rst create mode 100644 Tests/RunCMake/ExternalData/BadLinkMode-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadLinkMode-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/GoodLinkModeList-result.txt diff --git a/Help/release/dev/ExternalData-link-mode.rst b/Help/release/dev/ExternalData-link-mode.rst new file mode 100644 index 0000000000..f341e5843d --- /dev/null +++ b/Help/release/dev/ExternalData-link-mode.rst @@ -0,0 +1,7 @@ +ExternalData-link-mode +---------------------- + +* The :module:`ExternalData` module gained an + :variable:`ExternalData_LINK_MODE` variable to select an ordered list + of file materialization modes for hard links, symbolic links, and + copies. diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 6dc9ee5d2d..c40df1af1e 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -179,6 +179,36 @@ calling any of the functions provided by this module. data files under ``ExternalData_BINARY_ROOT``. By default, hash records continue to be placed next to the materialized data. +.. variable:: ExternalData_LINK_MODE + + .. versionadded:: 4.4 + + The ``ExternalData_LINK_MODE`` variable selects how real data files are + exposed under ``ExternalData_BINARY_ROOT``. + + The value may be one of: + + ``auto`` + Try each of the supported modes until one succeeds. + The default order is ``hardlink -> symlink -> copy`` on Windows + and ``symlink -> hardlink -> copy`` on other platforms. + + ``hardlink`` + Materialize paths as hard links to the real files. + + ``symlink`` + Materialize paths as symbolic links to the real files. + + ``copy`` + Materialize paths as copies of the real files. + + Or, the value may be a :ref:`semicolon-separated list ` + of ``hardlink``, ``symlink``, and ``copy`` to try the modes in the order + specified until one succeeds. + + When ``ExternalData_NO_SYMLINKS`` is set, the ``symlink`` mode is removed + from automatic selection and cannot be requested explicitly. + .. variable:: ExternalData_OBJECT_STORES The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local @@ -1010,33 +1040,103 @@ if(NOT ExternalData_URL_TEMPLATES AND NOT ExternalData_OBJECT_STORES) "Neither ExternalData_URL_TEMPLATES nor ExternalData_OBJECT_STORES is set!") endif() +function(_ExternalData_compute_link_target src dst var_tgt) + get_filename_component(dst_dir "${dst}" PATH) + set(tgt "${src}") + if(relative_top) + # Use relative path if files are close enough. + file(RELATIVE_PATH relsrc "${relative_top}" "${src}") + file(RELATIVE_PATH reldst "${relative_top}" "${dst}") + if(NOT IS_ABSOLUTE "${relsrc}" AND NOT "${relsrc}" MATCHES "^\\.\\./" AND + NOT IS_ABSOLUTE "${reldst}" AND NOT "${reldst}" MATCHES "^\\.\\./") + file(RELATIVE_PATH tgt "${dst_dir}" "${src}") + endif() + endif() + set(${var_tgt} "${tgt}" PARENT_SCOPE) +endfunction() + +function(_ExternalData_try_link_mode mode src dst var_result) + if(mode STREQUAL "copy") + file(COPY_FILE "${src}" "${dst}" RESULT result INPUT_MAY_BE_RECENT) + elseif(mode STREQUAL "hardlink") + file(CREATE_LINK "${src}" "${dst}" RESULT result) + elseif(mode STREQUAL "symlink") + _ExternalData_compute_link_target("${src}" "${dst}" tgt) + file(CREATE_LINK "${tgt}" "${dst}" RESULT result SYMBOLIC) + else() + set(result "Unsupported ExternalData_LINK_MODE `${mode}`") + endif() + set(${var_result} "${result}" PARENT_SCOPE) +endfunction() + +function(_ExternalData_get_auto_link_modes var_modes) + if(CMAKE_HOST_WIN32) + set(modes hardlink) + if(NOT ExternalData_NO_SYMLINKS) + list(APPEND modes symlink) + endif() + list(APPEND modes copy) + else() + if(NOT ExternalData_NO_SYMLINKS) + list(APPEND modes symlink) + endif() + list(APPEND modes hardlink copy) + endif() + set(${var_modes} "${modes}" PARENT_SCOPE) +endfunction() + +function(_ExternalData_get_requested_link_modes var_modes) + if(DEFINED ExternalData_LINK_MODE AND NOT "${ExternalData_LINK_MODE}" STREQUAL "") + set(requested_modes "${ExternalData_LINK_MODE}") + else() + set(requested_modes auto) + endif() + string(TOLOWER "${requested_modes}" requested_modes_lower) + + if(requested_modes_lower STREQUAL "auto") + _ExternalData_get_auto_link_modes(modes) + else() + set(modes) + foreach(mode IN LISTS requested_modes) + string(TOLOWER "${mode}" mode_lower) + if(mode_lower STREQUAL "copy" OR mode_lower STREQUAL "hardlink") + list(APPEND modes "${mode_lower}") + elseif(mode_lower STREQUAL "symlink") + if(ExternalData_NO_SYMLINKS) + message(FATAL_ERROR + "ExternalData_LINK_MODE with `symlink` conflicts with NO_SYMLINKS") + endif() + list(APPEND modes symlink) + else() + message(FATAL_ERROR + "ExternalData_LINK_MODE must be `auto` or list: hardlink, symlink, copy") + endif() + endforeach() + endif() + + set(${var_modes} "${modes}" PARENT_SCOPE) +endfunction() + function(_ExternalData_link_or_copy src dst) # Create a temporary file first. get_filename_component(dst_dir "${dst}" PATH) file(MAKE_DIRECTORY "${dst_dir}") _ExternalData_random(random) set(tmp "${dst}.tmp${random}") - if(UNIX AND NOT ExternalData_NO_SYMLINKS) - # Create a symbolic link. - set(tgt "${src}") - if(relative_top) - # Use relative path if files are close enough. - file(RELATIVE_PATH relsrc "${relative_top}" "${src}") - file(RELATIVE_PATH relfile "${relative_top}" "${dst}") - if(NOT IS_ABSOLUTE "${relsrc}" AND NOT "${relsrc}" MATCHES "^\\.\\./" AND - NOT IS_ABSOLUTE "${reldst}" AND NOT "${reldst}" MATCHES "^\\.\\./") - file(RELATIVE_PATH tgt "${dst_dir}" "${src}") - endif() + _ExternalData_get_requested_link_modes(_ExternalData_link_modes) + + unset(result) + foreach(_ExternalData_mode IN LISTS _ExternalData_link_modes) + _ExternalData_try_link_mode("${_ExternalData_mode}" "${src}" "${tmp}" result) + if(NOT result) + break() endif() - # Create link (falling back to copying if there's a problem). - file(CREATE_LINK "${tgt}" "${tmp}" RESULT result COPY_ON_ERROR SYMBOLIC) - else() - # Create a copy. - file(COPY_FILE "${src}" "${tmp}" RESULT result INPUT_MAY_BE_RECENT) - endif() + file(REMOVE "${tmp}") + endforeach() + if(result) file(REMOVE "${tmp}") - message(FATAL_ERROR "Failed to create:\n \"${tmp}\"\nfrom:\n \"${obj}\"\nwith error:\n ${result}") + message(FATAL_ERROR "Failed to create:\n \"${tmp}\"\nfrom:\n \"${src}\"\nwith error:\n ${result}") endif() # Atomically create/replace the real destination. diff --git a/Modules/ExternalData_config.cmake.in b/Modules/ExternalData_config.cmake.in index 66785f0d25..69cf0f58a5 100644 --- a/Modules/ExternalData_config.cmake.in +++ b/Modules/ExternalData_config.cmake.in @@ -4,4 +4,5 @@ set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@") set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@") set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@") set(ExternalData_STATE_ROOT "@ExternalData_STATE_ROOT@") +set(ExternalData_LINK_MODE "@ExternalData_LINK_MODE@") @_ExternalData_CONFIG_CODE@ diff --git a/Tests/RunCMake/ExternalData/BadLinkMode-result.txt b/Tests/RunCMake/ExternalData/BadLinkMode-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadLinkMode-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadLinkMode-stderr.txt b/Tests/RunCMake/ExternalData/BadLinkMode-stderr.txt new file mode 100644 index 0000000000..29c02a2596 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadLinkMode-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Modules/ExternalData\.cmake:[0-9]+ \(message\): + ExternalData_LINK_MODE must be `auto` or list: hardlink, symlink, copy diff --git a/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-result.txt b/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-stderr.txt b/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-stderr.txt new file mode 100644 index 0000000000..51f3634d91 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadLinkModeNoSymlinks-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Modules/ExternalData\.cmake:[0-9]+ \(message\): + ExternalData_LINK_MODE with `symlink` conflicts with NO_SYMLINKS diff --git a/Tests/RunCMake/ExternalData/GoodLinkModeList-result.txt b/Tests/RunCMake/ExternalData/GoodLinkModeList-result.txt new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/Tests/RunCMake/ExternalData/GoodLinkModeList-result.txt @@ -0,0 +1 @@ +0 diff --git a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake index b4cc95e6f5..53c71526a7 100644 --- a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake @@ -1,5 +1,25 @@ include(RunCMake) +function(run_externaldata_local test) + run_cmake_command(${test} + ${CMAKE_COMMAND} + -DExternalData_ACTION=local + -DExternalData_OBJECT_STORES=${RunCMake_BINARY_DIR}/${test}-store + ${ARGN} + -Dfile=${RunCMake_BINARY_DIR}/${test}-output.txt + -Dname=${RunCMake_SOURCE_DIR}/CMakeLists.txt + -P ${RunCMake_SOURCE_DIR}/../../../Modules/ExternalData.cmake + ) +endfunction() + +function(run_externaldata_local_link_mode test link_mode) + string(REPLACE ";" "\\\\;" link_mode_arg "${link_mode}") + run_externaldata_local(${test} + "-DExternalData_LINK_MODE=${link_mode_arg}" + ${ARGN} + ) +endfunction() + run_cmake(BadAlgoMap1) run_cmake(BadAlgoMap2) run_cmake(BadArguments) @@ -8,6 +28,13 @@ run_cmake(BadCustom2) run_cmake(BadCustom3) run_cmake(BadCustom4) run_cmake(BadHashAlgo1) +run_externaldata_local_link_mode(BadLinkMode "invalid;copy" + ) +run_externaldata_local_link_mode(BadLinkModeNoSymlinks "symlink;copy" + -DExternalData_NO_SYMLINKS=1 + ) +run_externaldata_local_link_mode(GoodLinkModeList "copy;hardlink" + ) run_cmake(BadOption1) run_cmake(BadOption2) run_cmake(BadRecurse1)