Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules

VadimLevin · web-flow · commit 3c89a28a0612 · 2021-09-18T10:02:55.000+03:00
* feat: OpenCV extension with pure Python modules

* feat: cv2 is now a Python package instead of extension module

Python package cv2 now can handle both Python and C extension modules
properly without additional "subfolders" like "_extra_py_code".

* feat: can call native function from its reimplementation in Python
diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp
@@ -116,6 +116,12 @@ String dumpRange(const Range& argument)
     }
 }
 
+CV_WRAP static inline
+int testOverwriteNativeMethod(int argument)
+{
+    return argument;
+}
+
 CV_WRAP static inline
 String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3)
 {
diff --git a/modules/core/misc/python/package/utils/__init__.py b/modules/core/misc/python/package/utils/__init__.py
@@ -0,0 +1,14 @@
+from collections import namedtuple
+
+import cv2
+
+
+NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult",
+                                       ("py", "native"))
+
+
+def testOverwriteNativeMethod(arg):
+    return NativeMethodPatchedResult(
+        arg + 1,
+        cv2.utils._native.testOverwriteNativeMethod(arg)
+    )
diff --git a/modules/python/common.cmake b/modules/python/common.cmake
@@ -1,6 +1,31 @@
 # This file is included from a subdirectory
 set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}")
 
+function(ocv_add_python_files_from_path search_path)
+  file(GLOB_RECURSE extra_py_files
+       RELATIVE "${search_path}"
+       # Plain Python code
+       "${search_path}/*.py"
+       # Type annotations
+       "${search_path}/*.pyi"
+  )
+  message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}")
+  if(extra_py_files)
+    list(SORT extra_py_files)
+    foreach(filename ${extra_py_files})
+      get_filename_component(module "${filename}" DIRECTORY)
+      if(NOT ${module} IN_LIST extra_modules)
+        list(APPEND extra_modules ${module})
+      endif()
+      configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY)
+      install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python)
+    endforeach()
+    message(STATUS "Found ${extra_modules} Python modules from ${search_path}")
+  else()
+    message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files")
+  endif()
+endfunction()
+
 ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator)
 
 include_directories(SYSTEM
@@ -224,23 +249,15 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER)
     if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m}
         AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package"
     )
-      set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package")
-      file(GLOB_RECURSE extra_py_files
-          RELATIVE "${__base}"
-          "${__base}/**/*.py"
-      )
-      if(extra_py_files)
-        list(SORT extra_py_files)
-        foreach(f ${extra_py_files})
-          get_filename_component(__dir "${f}" DIRECTORY)
-          configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY)
-          install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python)
-        endforeach()
-      else()
-        message(WARNING "Module ${m} has no .py files in misc/python/package")
-      endif()
+      ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package")
     endif()
   endforeach(m)
+
+  if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "")
+    foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH})
+      ocv_add_python_files_from_path(${extra_ocv_py_modules_path})
+    endforeach()
+  endif()
 endif()  # NOT OPENCV_SKIP_PYTHON_LOADER
 
 unset(PYTHON_SRC_DIR)
diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py
@@ -2,6 +2,7 @@
 OpenCV Python binary extension loader
 '''
 import os
+import importlib
 import sys
 
 __all__ = []
@@ -15,16 +16,53 @@
     print('    pip install numpy')
     raise
 
+# TODO
+# is_x64 = sys.maxsize > 2**32
+
 
-py_code_loader = None
-if sys.version_info[:2] >= (3, 0):
+def __load_extra_py_code_for_module(base, name, enable_debug_print=False):
+    module_name = "{}.{}".format(__name__, name)
+    export_module_name = "{}.{}".format(base, name)
+    native_module = sys.modules.pop(module_name, None)
     try:
-        from . import _extra_py_code as py_code_loader
-    except:
-        pass
+        py_module = importlib.import_module(module_name)
+    except ImportError as err:
+        if enable_debug_print:
+            print("Can't load Python code for module:", module_name,
+                  ". Reason:", err)
+        # Extension doesn't contain extra py code
+        return False
+
+    if not hasattr(base, name):
+        setattr(sys.modules[base], name, py_module)
+    sys.modules[export_module_name] = py_module
+    # If it is C extension module it is already loaded by cv2 package
+    if native_module:
+        setattr(py_module, "_native", native_module)
+        for k, v in filter(lambda kv: not hasattr(py_module, kv[0]),
+                           native_module.__dict__.items()):
+            if enable_debug_print: print('    symbol: {} = {}'.format(k, v))
+            setattr(py_module, k, v)
+    return True
+
+
+def __collect_extra_submodules(enable_debug_print=False):
+    def modules_filter(module):
+        return all((
+             # module is not internal
+             not module.startswith("_"),
+             # it is not a file
+             os.path.isdir(os.path.join(_extra_submodules_init_path, module))
+        ))
+    if sys.version_info[0] < 3:
+        if enable_debug_print:
+            print("Extra submodules is loaded only for Python 3")
+        return []
+
+    __INIT_FILE_PATH = os.path.abspath(__file__)
+    _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH)
+    return filter(modules_filter, os.listdir(_extra_submodules_init_path))
 
-# TODO
-# is_x64 = sys.maxsize > 2**32
 
 def bootstrap():
     import sys
@@ -107,23 +145,36 @@ def load_first_config(fnames, required=True):
         # amending of LD_LIBRARY_PATH works for sub-processes only
         os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '')
 
-    if DEBUG: print('OpenCV loader: replacing cv2 module')
-    del sys.modules['cv2']
-    import cv2
+    if DEBUG: print("Relink everything from native cv2 module to cv2 package")
+
+    py_module = sys.modules.pop("cv2")
+
+    native_module = importlib.import_module("cv2")
+
+    sys.modules["cv2"] = py_module
+    setattr(py_module, "_native", native_module)
+
+    for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__",
+                                                           "__name__", "__package__"),
+                                  native_module.__dict__.items()):
+        if item_name not in g_vars:
+            g_vars[item_name] = item
 
     sys.path = save_sys_path  # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502)
 
     try:
-        import sys
         del sys.OpenCV_LOADER
-    except:
-        pass
+    except Exception as e:
+        if DEBUG:
+            print("Exception during delete OpenCV_LOADER:", e)
 
     if DEBUG: print('OpenCV loader: binary extension... OK')
 
-    if py_code_loader:
-        py_code_loader.init('cv2')
+    for submodule in __collect_extra_submodules(DEBUG):
+        if __load_extra_py_code_for_module("cv2", submodule, DEBUG):
+            if DEBUG: print("Extra Python code for", submodule, "is loaded")
 
     if DEBUG: print('OpenCV loader: DONE')
 
+
 bootstrap()
diff --git a/modules/python/package/cv2/_extra_py_code/__init__.py b/modules/python/package/cv2/_extra_py_code/__init__.py
diff --git a/modules/python/package/extra_modules/misc/__init__.py b/modules/python/package/extra_modules/misc/__init__.py
@@ -0,0 +1 @@
+from .version import get_ocv_version
diff --git a/modules/python/package/extra_modules/misc/version.py b/modules/python/package/extra_modules/misc/version.py
@@ -0,0 +1,5 @@
+import cv2
+
+
+def get_ocv_version():
+    return getattr(cv2, "__version__", "unavailable")
diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt
@@ -15,9 +15,23 @@ set(the_description "The python3 bindings")
 set(MODULE_NAME python3)
 set(MODULE_INSTALL_SUBDIR python3)
 
+set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules)
+set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH})
+
+if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "")
+  set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path})
+else()
+  list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path})
+endif()
+
+unset(_ocv_extra_modules_path)
+
 set(PYTHON PYTHON3)
 
 include(../common.cmake)
 
+set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path})
+
+unset(_old_ocv_python_extra_modules_path)
 unset(MODULE_NAME)
 unset(MODULE_INSTALL_SUBDIR)
diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake
@@ -25,7 +25,6 @@ endif()
 set(PYTHON_LOADER_FILES
     "setup.py" "cv2/__init__.py"
     "cv2/load_config_py2.py" "cv2/load_config_py3.py"
-    "cv2/_extra_py_code/__init__.py"
 )
 foreach(fname ${PYTHON_LOADER_FILES})
   get_filename_component(__dir "${fname}" DIRECTORY)
diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py
@@ -4,6 +4,12 @@
 import ctypes
 from functools import partial
 from collections import namedtuple
+import sys
+
+if sys.version_info[0] < 3:
+    from collections import Sequence
+else:
+    from collections.abc import Sequence
 
 import numpy as np
 import cv2 as cv
@@ -585,6 +591,31 @@ def test_vector_fast_return(self):
         self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.")
 
 
+class CanUsePurePythonModuleFunction(NewOpenCVTests):
+    def test_can_get_ocv_version(self):
+        import sys
+        if sys.version_info[0] < 3:
+            raise unittest.SkipTest('Python 2.x is not supported')
+
+        self.assertEqual(cv.misc.get_ocv_version(), cv.__version__,
+                         "Can't get package version using Python misc module")
+
+    def test_native_method_can_be_patched(self):
+        import sys
+
+        if sys.version_info[0] < 3:
+            raise unittest.SkipTest('Python 2.x is not supported')
+
+        res = cv.utils.testOverwriteNativeMethod(10)
+        self.assertTrue(isinstance(res, Sequence),
+                        msg="Overwritten method should return sequence. "
+                            "Got: {} of type {}".format(res, type(res)))
+        self.assertSequenceEqual(res, (11, 10),
+                                 msg="Failed to overwrite native method")
+        res = cv.utils._native.testOverwriteNativeMethod(123)
+        self.assertEqual(res, 123, msg="Failed to call native method implementation")
+
+
 class SamplesFindFile(NewOpenCVTests):
 
     def test_ExistedFile(self):

Original file line number	Diff line number	Diff line change
`@@ -116,6 +116,12 @@ String dumpRange(const Range& argument)`
`116`	`116`	`}`
`117`	`117`	`}`
`118`	`118`
	`119`	`+CV_WRAP static inline`
	`120`	`+int testOverwriteNativeMethod(int argument)`
	`121`	`+{`
	`122`	`+ return argument;`
	`123`	`+}`
	`124`	`+`
`119`	`125`	`CV_WRAP static inline`
`120`	`126`	`String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3)`
`121`	`127`	`{`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,6 @@ endif()`
`25`	`25`	`set(PYTHON_LOADER_FILES`
`26`	`26`	`"setup.py" "cv2/__init__.py"`
`27`	`27`	`"cv2/load_config_py2.py" "cv2/load_config_py3.py"`
`28`		`- "cv2/_extra_py_code/__init__.py"`
`29`	`28`	`)`
`30`	`29`	`foreach(fname ${PYTHON_LOADER_FILES})`
`31`	`30`	`get_filename_component(__dir "${fname}" DIRECTORY)`