diff --git a/Help/release/dev/ExternalProject-git-clone-retry.rst b/Help/release/dev/ExternalProject-git-clone-retry.rst new file mode 100644 index 0000000000..fafe1a6730 --- /dev/null +++ b/Help/release/dev/ExternalProject-git-clone-retry.rst @@ -0,0 +1,7 @@ +ExternalProject-git-clone-retry +------------------------------- + +* The :module:`ExternalProject` and :module:`FetchContent` modules gained + :variable:`CMAKE_EP_GIT_CLONE_RETRY_COUNT` and + :variable:`CMAKE_EP_GIT_CLONE_RETRY_DELAY` variables to control + ``git clone`` retries. diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index fefcd27159..1045259611 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -446,6 +446,31 @@ is used. unintended changes or failed builds resulting from conflicts during rebase operations. +The following variables can be set to control the retry behavior for +``git clone`` operations: + +.. variable:: CMAKE_EP_GIT_CLONE_RETRY_COUNT + + .. versionadded:: 4.4 + + Specifies the number of times to retry a ``git clone`` operation if it + fails. The default is ``2`` retries. The first attempt is not counted + as a retry, so the total number of attempts is the retry count plus one. + This variable should not be set by a project, it is intended for the user + to set. It is primarily useful when dealing with intermittent network + issues or rate limiting from git hosting services. + +.. variable:: CMAKE_EP_GIT_CLONE_RETRY_DELAY + + .. versionadded:: 4.4 + + Specifies the delay in seconds between retry attempts for git clone + operations. The default is ``0`` seconds (no delay between retries). + This variable should not be set by a project, it is intended for the + user to set. Setting a delay can be helpful when dealing with rate + limiting from git hosting services, where immediate retries would + also be rejected. + Subversion ~~~~~~~~~~ diff --git a/Modules/ExternalProject/gitclone.cmake.in b/Modules/ExternalProject/gitclone.cmake.in index 449bcb30c7..19a7eb447b 100644 --- a/Modules/ExternalProject/gitclone.cmake.in +++ b/Modules/ExternalProject/gitclone.cmake.in @@ -30,10 +30,15 @@ if(error_code) message(FATAL_ERROR "Failed to remove directory: '@source_dir@'") endif() -# try the clone 3 times in case there is an odd git clone issue +# try the clone 1 + N times in case there is an odd git clone issue set(error_code 1) set(number_of_tries 0) -while(error_code AND number_of_tries LESS 3) +math(EXPR max_tries "1 + @git_clone_retries@") +while(error_code AND number_of_tries LESS ${max_tries}) + if(number_of_tries GREATER 0 AND @git_clone_retry_delay@ GREATER 0) + message(STATUS "Retry #${number_of_tries}, waiting @git_clone_retry_delay@ seconds before next attempt...") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep @git_clone_retry_delay@) + endif() execute_process( COMMAND "@git_EXECUTABLE@" clone @git_clone_options@ "@git_repository@" "@src_name@" @@ -47,7 +52,7 @@ if(number_of_tries GREATER 1) message(NOTICE "Had to git clone more than once: ${number_of_tries} times.") endif() if(error_code) - message(FATAL_ERROR "Failed to clone repository: '@git_repository@'") + message(FATAL_ERROR "Failed to clone repository:\n '@git_repository@'") endif() execute_process( diff --git a/Modules/ExternalProject/shared_internal_commands.cmake b/Modules/ExternalProject/shared_internal_commands.cmake index b7077ef07e..64ad3e3ddc 100644 --- a/Modules/ExternalProject/shared_internal_commands.cmake +++ b/Modules/ExternalProject/shared_internal_commands.cmake @@ -397,6 +397,8 @@ function(_ep_write_gitclone_script gitclone_stampfile tls_version tls_verify + git_clone_retries + git_clone_retry_delay ) if(NOT Git_VERSION VERSION_LESS 1.8.5) @@ -979,6 +981,16 @@ CMP0097=${_EP_CMP0097} get_filename_component(src_name "${source_dir}" NAME) get_filename_component(work_dir "${source_dir}" PATH) + # Get retry configuration from variables with defaults + set(git_clone_retries "${CMAKE_EP_GIT_CLONE_RETRY_COUNT}") + if("${git_clone_retries}" STREQUAL "") + set(git_clone_retries 2) + endif() + set(git_clone_retry_delay "${CMAKE_EP_GIT_CLONE_RETRY_DELAY}") + if("${git_clone_retry_delay}" STREQUAL "") + set(git_clone_retry_delay 0) + endif() + # Since git clone doesn't succeed if the non-empty source_dir exists, # create a cmake script to invoke as download command. # The script will delete the source directory and then call git clone. @@ -1003,6 +1015,8 @@ CMP0097=${_EP_CMP0097} ${stamp_dir}/${name}-gitclone-lastrun.txt "${tls_version}" "${tls_verify}" + "${git_clone_retries}" + "${git_clone_retry_delay}" ) set(comment "Performing download step (git clone) for '${name}'") set(cmd ${CMAKE_COMMAND} diff --git a/Modules/FetchContent.cmake b/Modules/FetchContent.cmake index 80152f61c9..12d5f3bbce 100644 --- a/Modules/FetchContent.cmake +++ b/Modules/FetchContent.cmake @@ -1866,6 +1866,8 @@ ExternalProject_Add_Step(${contentName}-populate copyfile set(__FETCHCONTENT_CACHED_INFO "") set(__passthrough_vars CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY + CMAKE_EP_GIT_CLONE_RETRY_COUNT + CMAKE_EP_GIT_CLONE_RETRY_DELAY CMAKE_TLS_VERSION CMAKE_TLS_VERIFY CMAKE_TLS_CAINFO diff --git a/Tests/RunCMake/ExternalProject/GitCloneRetry-build-result.txt b/Tests/RunCMake/ExternalProject/GitCloneRetry-build-result.txt new file mode 100644 index 0000000000..d197c913c2 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/GitCloneRetry-build-result.txt @@ -0,0 +1 @@ +[^0] diff --git a/Tests/RunCMake/ExternalProject/GitCloneRetry-build-stdout.txt b/Tests/RunCMake/ExternalProject/GitCloneRetry-build-stdout.txt new file mode 100644 index 0000000000..be2ff68d25 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/GitCloneRetry-build-stdout.txt @@ -0,0 +1,9 @@ + *-- Retry #1, waiting 1 seconds before next attempt\.\.\..* + *-- Retry #2, waiting 1 seconds before next attempt\.\.\..* + *-- Retry #3, waiting 1 seconds before next attempt\.\.\..* + *Had to git clone more than once: 4 times\. + *CMake Error at [^ +]*TestGitRetry-prefix/tmp/TestGitRetry-gitclone.cmake:[0-9]+ \(message\): + +Failed to clone repository:[ + ]* + +'[^']*/Tests/RunCMake/ExternalProject/GitCloneRetry-build/TestGitRetry.git' diff --git a/Tests/RunCMake/ExternalProject/GitCloneRetry.cmake b/Tests/RunCMake/ExternalProject/GitCloneRetry.cmake new file mode 100644 index 0000000000..16c9dabd33 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/GitCloneRetry.cmake @@ -0,0 +1,13 @@ +include(ExternalProject) + +set(CMAKE_EP_GIT_CLONE_RETRY_COUNT 3) +set(CMAKE_EP_GIT_CLONE_RETRY_DELAY 1) + +# The GIT_REPOSITORY path does not exist, so git clone will always fail, +# which is what triggers the retries. +ExternalProject_Add(TestGitRetry + GIT_REPOSITORY "${CMAKE_CURRENT_BINARY_DIR}/TestGitRetry.git" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" +) diff --git a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake index 3848d77886..3e87efb914 100644 --- a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake @@ -258,6 +258,14 @@ if(GIT_EXECUTABLE) run_cmake(TLSVersionBadArg) run_cmake(TLSVersionBadVar) run_cmake(TLSVersionBadEnv) + + block() + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/GitCloneRetry-build) + run_cmake(GitCloneRetry) + set(RunCMake_TEST_NO_CLEAN 1) + set(RunCMake_TEST_OUTPUT_MERGE TRUE) + run_cmake_command(GitCloneRetry-build ${CMAKE_COMMAND} --build . --target TestGitRetry) + endblock() endif() set(RunCMake_TEST_OUTPUT_MERGE 1) diff --git a/Tests/RunCMake/FetchContent/VarPassthroughs.cmake b/Tests/RunCMake/FetchContent/VarPassthroughs.cmake index 284f5b8f16..f3bc7d0b06 100644 --- a/Tests/RunCMake/FetchContent/VarPassthroughs.cmake +++ b/Tests/RunCMake/FetchContent/VarPassthroughs.cmake @@ -8,6 +8,8 @@ set(CMAKE_TLS_CAINFO CCCC) set(CMAKE_NETRC DDDD) set(CMAKE_NETRC_FILE EEEE) set(CMAKE_TLS_VERSION FFFF) +set(CMAKE_EP_GIT_CLONE_RETRY_COUNT GGGG) +set(CMAKE_EP_GIT_CLONE_RETRY_DELAY HHHH) FetchContent_Declare(PassThrough DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo "Download command executed" @@ -43,3 +45,11 @@ endif() if(NOT contents MATCHES "CMAKE_NETRC_FILE \\[==\\[EEEE\\]==\\]") message(FATAL_ERROR "Missing CMAKE_NETRC_FILE") endif() + +if(NOT contents MATCHES "CMAKE_EP_GIT_CLONE_RETRY_COUNT \\[==\\[GGGG\\]==\\]") + message(FATAL_ERROR "Missing CMAKE_EP_GIT_CLONE_RETRY_COUNT") +endif() + +if(NOT contents MATCHES "CMAKE_EP_GIT_CLONE_RETRY_DELAY \\[==\\[HHHH\\]==\\]") + message(FATAL_ERROR "Missing CMAKE_EP_GIT_CLONE_RETRY_DELAY") +endif()