Merge pull request #21531 from fengyuentau:add_3rdparty_mirrors

fengyuentau · web-flow · commit f7ac724c5d2b · 2022-03-18T09:50:21.000Z
Support downloading 3rdparty resources from Gitcode &amp; Gitlab-style mirrors

* replace github.com with gitcode.net for ocv_download

* replace raw.githubusercontent.com with gitcode.net for ocv_download

* rename funtions and remove some comments

* add options for custom mirrors, which simply replace domain github.com &amp; githubusercontent.com

* run ocv_init_download once; replace DL_URL with mirrored one when calling ocv_download

* fix for empty download links when not using mirror

* fix bugs: set(.. .. PARENT_SCOPE) for ocv_init_download; correct macro names for replace github archives and raw githubusercontent

* adjusted mirror swapping impl: replace with mirrored link before each ocv_download; update md5sum for archives

* fix a bug: macro invoked with incorrect arguments by non-set vars

* enclose if statement

* workable impl

* shorten the var names of two key options

* scalable implementation of downloading from mirror and using custom mirror

* improve ocv_init_download help message

* fix the different extracted directory name in case of ADE &amp; TBB which are downloaded from release page

* improve help message printing

* Download ADE &amp; TBB using commit ids instead of from release pages

* support custom mirrors on downloading archives

* improve hints

* add missing parentheses

* reset ocv_download calls

* mirror support implementation using ocv_cmake_hook &amp; ocv_cmake_hook_append

* move ocv_init_download into cmake/OpenCVDownload.cmake

* move ocv_cmake_hook before checking CMake cache

* improve hints when not fetching as git repo

* add WORKING_DIRECTORY  in execute_process in ocv_init_download

* use OPENCV_DOWNLOAD_MIRROR_ID

* add custom.cmake for custom mirror

* detect github origin

* fix broken var name

* download from github by default if custom tbb is set

* add checksum checks for gitcode.cmake before replacing urls and checksums

* add checksum checks for custom.cmake before replacing urls and checkusms

* use description specify instead of set for messages in custom.cmake; use warning message for warnings

* updates and fixes
diff --git a/cmake/OpenCVDownload.cmake b/cmake/OpenCVDownload.cmake
@@ -37,6 +37,50 @@ file(WRITE "${OPENCV_DOWNLOAD_LOG}" "#use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n")
 file(REMOVE "${OPENCV_DOWNLOAD_WITH_CURL}")
 file(REMOVE "${OPENCV_DOWNLOAD_WITH_WGET}")
 
+ocv_check_environment_variables(OPENCV_DOWNLOAD_MIRROR_ID)
+
+function(ocv_init_download_mirror)
+  if(NOT DEFINED OPENCV_DOWNLOAD_MIRROR_ID)
+    # Run `git remote get-url origin` to get remote source
+    execute_process(
+      COMMAND
+        git remote get-url origin
+      WORKING_DIRECTORY
+        ${CMAKE_SOURCE_DIR}
+      RESULT_VARIABLE
+        RESULT_STATUS
+      OUTPUT_VARIABLE
+        OCV_GIT_ORIGIN_URL_OUTPUT
+      ERROR_QUIET
+    )
+    # if non-git, OCV_GIT_ORIGIN_URL_OUTPUT is empty
+    if(NOT OCV_GIT_ORIGIN_URL_OUTPUT)
+      message(STATUS "ocv_init_download: OpenCV source tree is not fetched as git repository. 3rdparty resources will be downloaded from github.com by default.")
+      return()
+    else()
+      # Check if git origin is github.com
+      string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "github.com" _found_github)
+      if(NOT ${_found_github} EQUAL -1)
+        set(OPENCV_DOWNLOAD_MIRROR_ID "github" CACHE STRING "")
+      endif()
+      # Check if git origin is gitcode.net
+      string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "gitcode.net" _found_gitcode)
+      if(NOT ${_found_gitcode} EQUAL -1)
+        set(OPENCV_DOWNLOAD_MIRROR_ID "gitcode" CACHE STRING "")
+      endif()
+    endif()
+  endif()
+
+  if(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "gitcode" OR OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "custom")
+    message(STATUS "ocv_init_download: Using ${OPENCV_DOWNLOAD_MIRROR_ID}-hosted mirror to download 3rdparty components.")
+    ocv_cmake_hook_append(OPENCV_DOWNLOAD_PRE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/mirrors/${OPENCV_DOWNLOAD_MIRROR_ID}.cmake")
+  elseif(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "github")
+    return()
+  else()
+    message(STATUS "ocv_init_download: Unable to recognize git server of OpenCV source code. Using github.com to download 3rdparty components.")
+  endif()
+endfunction()
+
 function(ocv_download)
   cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN})
 
@@ -67,6 +111,8 @@ function(ocv_download)
     set(${DL_STATUS} TRUE PARENT_SCOPE)
   endif()
 
+  ocv_cmake_hook(OPENCV_DOWNLOAD_PRE)
+
   # Check CMake cache for already processed tasks
   string(FIND "${DL_DESTINATION_DIR}" "${CMAKE_BINARY_DIR}" DL_BINARY_PATH_POS)
   if(DL_BINARY_PATH_POS EQUAL 0)
@@ -115,7 +161,7 @@ function(ocv_download)
   if(DL_ID)
     set(__msg_prefix "${DL_ID}: ")
   endif()
-  message(STATUS "${__msg_prefix}Download: ${DL_FILENAME}")
+  message(STATUS "${__msg_prefix}Downloading ${DL_FILENAME} from ${DL_URL}")
 
   # Copy mode: check if copy destination exists and is correct
   if(NOT DL_UNPACK)
@@ -252,3 +298,8 @@ ${OPENCV_DOWNLOAD_LOG}
     set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "")
   endif()
 endfunction()
+
+# ----------------------------------------------------------------------------
+#  Initialize download in case mirror is used
+# ----------------------------------------------------------------------------
+ocv_init_download_mirror()
diff --git a/cmake/mirrors/custom.cmake b/cmake/mirrors/custom.cmake
@@ -0,0 +1,77 @@
+# Gitlab-style mirror
+# CMake scripts look for opencv/opencv_3rdparty,
+#  OAID/Tengine, 01org/tbb(oneAPI/oneTBB), opencv/ade
+#  from OPENCV_DOWNLOAD_MIRROR
+ocv_update(OPENCV_DOWNLOAD_MIRROR_URL "")
+# Tengine
+ocv_update(TENGINE_PKG_MD5_CUSTOM "")
+ocv_update(TENGINE_PKG_MD5_ORIGINAL 23f61ebb1dd419f1207d8876496289c5) # same as tengine_md5sum for TENGINE commit of e89cf8870de2ff0a80cfe626c0b52b2a16fb302e
+
+# TBB
+ocv_update(TBB_RELEASE_CUSTOM "")
+ocv_update(TBB_PKG_NAME_CUSTOM "")
+ocv_update(TBB_PKG_MD5_CUSTOM "")
+ocv_update(TBB_PKG_MD5_ORIGINAL 5af6f6c2a24c2043e62e47205e273b1f) # same as OPENCV_TBB_RELEASE_MD5 for TBB release of v2020.2
+
+# ADE
+ocv_update(ADE_RELEASE_CUSTOM "")
+ocv_update(ADE_PKG_NAME_CUSTOM "")
+ocv_update(ADE_PKG_MD5_CUSTOM "")
+ocv_update(ADE_PKG_MD5_ORIGINAL b624b995ec9c439cbc2e9e6ee940d3a2) # same as ade_md5 for ADE release of v0.1.1f
+
+macro(ocv_download_url_custom_usercontent)
+  string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+  list(GET DL_URL_split 5 __COMMIT_ID)
+  list(GET DL_URL_split 6 __PKG_NAME)
+  set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/opencv/opencv_3rdparty/-/raw/${__COMMIT_ID}/${__PKG_NAME}/")
+endmacro()
+macro(ocv_download_url_custom_archive_commit_id)
+  if("m${${DL_ID}_PKG_MD5_CUSTOM}" STREQUAL "m")
+    message(WARNING "ocv_download: specify ${DL_ID}_PKG_MD5_CUSTOM to download ${DL_ID} from custom source.")
+  elseif(${DL_ID}_PKG_MD5_ORIGINAL STREQUAL "${DL_HASH}")
+    string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+    list(GET DL_URL_split 3 __OWNER)
+    list(GET DL_URL_split 4 __REPO_NAME)
+    set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/${__OWNER}/${__REPO_NAME}/-/archive/")
+    set(DL_HASH "${${DL_ID}_PKG_MD5_CUSTOM}")
+  else()
+    message(WARNING "No information about mirrors for downloading ${DL_FILENAME} from URL='${DL_URL}' and MD5=${DL_HASH}.")
+  endif()
+endmacro()
+macro(ocv_download_url_custom_archive_release)
+    if("m${${DL_ID}_RELEASE_CUSTOM}" STREQUAL "m")
+      message(WARNING "ocv_download: specify ${DL_ID}_RELEASE_CUSTOM to download ${DL_ID} from custom source.")
+      return()
+    endif()
+    if("m${${DL_ID}_PKG_NAME_CUSTOM}" STREQUAL "m")
+      message(WARNING "ocv_download: specify ${DL_ID}_PKG_NAME_CUSTOM to download ${DL_ID} from custom source.")
+      return()
+    endif()
+    if("m${${DL_ID}_PKG_MD5_CUSTOM}" STREQUAL "m")
+      message(WARNING "ocv_download: specify ${DL_ID}_PKG_MD5_CUSTOM to download ${DL_ID} from custom source.")
+      return()
+    endif()
+    string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+    list(GET DL_URL_split 3 __OWNER)
+    list(GET DL_URL_split 4 __REPO_NAME)
+    set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/${__OWNER}/${__REPO_NAME}/-/archive/${${DL_ID}_PKG_RELEASE_CUSTOM}/${__REPO_NAME}-")
+    set(DL_HASH "${${DL_ID}_PKG_MD5_CUSTOM}")
+endmacro()
+
+if("m${OPENCV_DOWNLOAD_MIRROR_URL}" STREQUAL "m")
+  message(WARNING "ocv_download: specify OPENCV_DOWNLOAD_MIRROR_URL to use custom mirror.")
+else()
+  if((DL_ID STREQUAL "FFMPEG") OR (DL_ID STREQUAL "IPPICV"))
+    ocv_download_url_custom_usercontent()
+  elseif(DL_ID STREQUAL "TENGINE")
+    ocv_download_url_custom_archive_commit_id()
+  elseif(DL_ID STREQUAL "TBB")
+    ocv_download_url_custom_archive_release()
+    set(OPENCV_TBB_SUBDIR "${TBB_PKG_NAME_CUSTOM}" PARENT_SCOPE)
+  elseif(DL_ID STREQUAL "ADE")
+    ocv_download_url_custom_archive_release()
+    set(ade_subdir "${ADE_PKG_NAME_CUSTOM}" PARENT_SCOPE)
+  else()
+    message(STATUS "ocv_download: Unknown download ID ${DL_ID} for using mirror ${OPENCV_DOWNLOAD_MIRROR_URL}. Use original source instead.")
+  endif()
+endif()
diff --git a/cmake/mirrors/gitcode.cmake b/cmake/mirrors/gitcode.cmake
@@ -0,0 +1,70 @@
+# Tengine (Download via commit id)
+ocv_update(TENGINE_PKG_MD5_GITCODE 1b5908632b557275cd6e85b0c03f9690)
+ocv_update(TENGINE_PKG_MD5_ORIGINAL 23f61ebb1dd419f1207d8876496289c5) # same as tengine_md5sum for TENGINE commit of e89cf8870de2ff0a80cfe626c0b52b2a16fb302e
+
+# TBB (Download from release page)
+ocv_update(TBB_RELEASE_GITCODE "v2020.2")
+ocv_update(TBB_PKG_NAME_GITCODE "tbb-${TBB_RELEASE_GITCODE}")
+ocv_update(TBB_PKG_MD5_GITCODE 4eeafdf16a90cb66e39a31c8d6c6804e)
+ocv_update(TBB_PKG_MD5_ORIGINAL 5af6f6c2a24c2043e62e47205e273b1f) # same as OPENCV_TBB_RELEASE_MD5 for TBB release of v2020.2
+
+# ADE (Download from release page)
+ocv_update(ADE_RELEASE_GITCODE "v0.1.1f")
+ocv_update(ADE_PKG_NAME_GITCODE "ade-${ADE_RELEASE_GITCODE}")
+ocv_update(ADE_PKG_MD5_GITCODE c12909e0ccfa93138c820ba91ff37b3c)
+ocv_update(ADE_PKG_MD5_ORIGINAL b624b995ec9c439cbc2e9e6ee940d3a2) # same as ade_md5 for ADE release of v0.1.1f
+
+#
+# Replace download links for packages in opencv/opencv_3rdparty:
+# 1. Extract repo owner and repo name from DL_URL.
+# 2. Put repo owner and repo name into the placeholders of new DL_URL.
+#
+macro(ocv_download_url_gitcode_usercontent)
+  string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+  list(GET DL_URL_split 5 __COMMIT_ID)
+  list(GET DL_URL_split 6 __PKG_NAME)
+  set(DL_URL "https://gitcode.net/opencv/opencv_3rdparty/-/raw/${__COMMIT_ID}/${__PKG_NAME}/")
+endmacro()
+#
+# Replace download links and checksums for archives/releases in other repositories:
+# 1. Check if versions matched. If not matched, download from github instead.
+# 2. Extract repo owner and repo name from DL_URL.
+# 3. Put repo owner and repo name into the placeholders of new DL_URL.
+# 4. Replace DL_HASH with the one downloaded from gitcode.net.
+#
+macro(ocv_download_url_gitcode_archive_commit_id)
+  if(DL_HASH STREQUAL "${${DL_ID}_PKG_MD5_ORIGINAL}")
+    string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+    list(GET DL_URL_split 3 __OWNER)
+    list(GET DL_URL_split 4 __REPO_NAME)
+    set(DL_URL "https://gitcode.net/${__OWNER}/${__REPO_NAME}/-/archive/")
+    set(DL_HASH "${${DL_ID}_PKG_MD5_GITCODE}")
+  else()
+    message(WARNING "Package ${DL_ID} from mirror gitcode.net is outdated and will be downloaded from github.com instead.")
+  endif()
+endmacro()
+macro(ocv_download_url_gitcode_archive_release)
+  if(DL_HASH STREQUAL "${${DL_ID}_PKG_MD5_ORIGINAL}")
+    string(REPLACE "/" ";" DL_URL_split ${DL_URL})
+    list(GET DL_URL_split 3 __OWNER)
+    list(GET DL_URL_split 4 __REPO_NAME)
+    set(DL_URL "https://gitcode.net/${__OWNER}/${__REPO_NAME}/-/archive/${${DL_ID}_RELEASE_GITCODE}/${__REPO_NAME}-")
+    set(DL_HASH "${${DL_ID}_PKG_MD5_GITCODE}")
+  else()
+    message(WARNING "Package ${DL_ID} from mirror gitcode.net is outdated and will be downloaded from github.com instead.")
+  endif()
+endmacro()
+
+if((DL_ID STREQUAL "FFMPEG") OR (DL_ID STREQUAL "IPPICV"))
+  ocv_download_url_gitcode_usercontent()
+elseif(DL_ID STREQUAL "TENGINE")
+  ocv_download_url_gitcode_archive_commit_id()
+elseif(DL_ID STREQUAL "TBB")
+  ocv_download_url_gitcode_archive_release()
+  set(OPENCV_TBB_SUBDIR "${TBB_PKG_NAME_GITCODE}" PARENT_SCOPE)
+elseif(DL_ID STREQUAL "ADE")
+  ocv_download_url_gitcode_archive_release()
+  set(ade_subdir "${ADE_PKG_NAME_GITCODE}" PARENT_SCOPE)
+else()
+  message(STATUS "ocv_download: Unknown download ID ${DL_ID} for using mirror gitcode.net. Use original source instead.")
+endif()
