Merge pull request #24024 from TolyaTalamanov:at/add-onnx-openvino-execution-provider

TolyaTalamanov · web-flow · commit 5261961a6eb9 · 2023-07-21T12:18:06.000+03:00
G-API: Support OpenVINO Execution Provider for ONNXRT Backend #24024 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp
@@ -33,6 +33,12 @@ class GAPI_EXPORTS_W_SIMPLE PyParams {
     GAPI_WRAP
     PyParams& cfgNormalize(const std::string &layer_name, bool flag);
 
+    GAPI_WRAP
+    PyParams& cfgExecutionProvider(ep::OpenVINO ov_ep);
+
+    GAPI_WRAP
+    PyParams& cfgDisableMemPattern();
+
     GBackend backend() const;
     std::string tag() const;
     cv::util::any params() const;
diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp
@@ -27,6 +27,118 @@ namespace gapi {
  */
 namespace onnx {
 
+/**
+ * @brief This namespace contains Execution Providers structures for G-API ONNX Runtime backend.
+ */
+namespace ep {
+
+/**
+ * @brief This structure provides functions
+ * that fill inference options for ONNX OpenVINO Execution Provider.
+ * Please follow https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options
+ */
+struct GAPI_EXPORTS_W_SIMPLE OpenVINO {
+    // NB: Used from python.
+    /// @private -- Exclude this constructor from OpenCV documentation
+    GAPI_WRAP
+    OpenVINO() = default;
+
+    /** @brief Class constructor.
+
+    Constructs OpenVINO parameters based on device information.
+
+    @param device Target device to use.
+    */
+    GAPI_WRAP
+    OpenVINO(const std::string &device)
+        : device_id(device) {
+    }
+
+    /** @brief Specifies OpenVINO Execution Provider device type.
+
+    This function is used to override the accelerator hardware type
+    and precision at runtime. If this option is not explicitly configured, default
+    hardware and precision specified during onnxruntime build time is used.
+
+    @param type Device type ("CPU_FP32", "GPU_FP16", etc)
+    @return reference to this parameter structure.
+    */
+    GAPI_WRAP
+    OpenVINO& cfgDeviceType(const std::string &type) {
+        device_type = cv::util::make_optional(type);
+        return *this;
+    }
+
+    /** @brief Specifies OpenVINO Execution Provider cache dir.
+
+    This function is used to explicitly specify the path to save and load
+    the blobs enabling model caching feature.
+
+    @param dir Path to the directory what will be used as cache.
+    @return reference to this parameter structure.
+    */
+    GAPI_WRAP
+    OpenVINO& cfgCacheDir(const std::string &dir) {
+        cache_dir = dir;
+        return *this;
+    }
+
+    /** @brief Specifies OpenVINO Execution Provider number of threads.
+
+    This function is used to override the accelerator default value
+    of number of threads with this value at runtime. If this option
+    is not explicitly set, default value of 8 is used during build time.
+
+    @param nthreads Number of threads.
+    @return reference to this parameter structure.
+    */
+    GAPI_WRAP
+    OpenVINO& cfgNumThreads(size_t nthreads) {
+        num_of_threads = cv::util::make_optional(nthreads);
+        return *this;
+    }
+
+    /** @brief Enables OpenVINO Execution Provider opencl throttling.
+
+    This function is used to enable OpenCL queue throttling for GPU devices
+    (reduces CPU utilization when using GPU).
+
+    @return reference to this parameter structure.
+    */
+    GAPI_WRAP
+    OpenVINO& cfgEnableOpenCLThrottling() {
+        enable_opencl_throttling = true;
+        return *this;
+    }
+
+    /** @brief Enables OpenVINO Execution Provider dynamic shapes.
+
+    This function is used to enable OpenCL queue throttling for GPU devices
+    (reduces CPU utilization when using GPU).
+    This function is used to enable work with dynamic shaped models
+    whose shape will be set dynamically based on the infer input
+    image/data shape at run time in CPU.
+
+    @return reference to this parameter structure.
+    */
+    GAPI_WRAP
+    OpenVINO& cfgEnableDynamicShapes() {
+        enable_dynamic_shapes = true;
+        return *this;
+    }
+
+    std::string device_id;
+    std::string cache_dir;
+    cv::optional<std::string> device_type;
+    cv::optional<size_t> num_of_threads;
+    bool enable_opencl_throttling = false;
+    bool enable_dynamic_shapes = false;
+};
+
+using EP = cv::util::variant<cv::util::monostate, OpenVINO>;
+
+} // namespace ep
+
 GAPI_EXPORTS cv::gapi::GBackend backend();
 
 enum class TraitAs: int {
@@ -78,6 +190,9 @@ struct ParamDesc {
     // when the generic infer parameters are unpacked (see GONNXBackendImpl::unpackKernel)
     std::unordered_map<std::string, std::pair<cv::Scalar, cv::Scalar> > generic_mstd;
     std::unordered_map<std::string, bool> generic_norm;
+
+    cv::gapi::onnx::ep::EP execution_provider;
+    bool disable_mem_pattern;
 };
 } // namespace detail
 
@@ -115,6 +230,7 @@ template<typename Net> class Params {
         desc.num_in  = std::tuple_size<typename Net::InArgs>::value;
         desc.num_out = std::tuple_size<typename Net::OutArgs>::value;
         desc.is_generic = false;
+        desc.disable_mem_pattern = false;
     };
 
     /** @brief Specifies sequence of network input layers names for inference.
@@ -279,6 +395,29 @@ template<typename Net> class Params {
         return *this;
     }
 
+    /** @brief Specifies execution provider for runtime.
+
+    The function is used to set ONNX Runtime OpenVINO Execution Provider options.
+
+    @param ovep OpenVINO Execution Provider options.
+    @see cv::gapi::onnx::ep::OpenVINO.
+
+    @return the reference on modified object.
+    */
+    Params<Net>& cfgExecutionProvider(ep::OpenVINO&& ovep) {
+        desc.execution_provider = std::move(ovep);
+        return *this;
+    }
+
+    /** @brief Disables the memory pattern optimization.
+
+    @return the reference on modified object.
+    */
+    Params<Net>& cfgDisableMemPattern() {
+        desc.disable_mem_pattern = true;
+        return *this;
+    }
+
     // BEGIN(G-API's network parametrization API)
     GBackend      backend() const { return cv::gapi::onnx::backend(); }
     std::string   tag()     const { return Net::tag(); }
@@ -306,7 +445,7 @@ class Params<cv::gapi::Generic> {
     @param model_path path to model file (.onnx file).
     */
     Params(const std::string& tag, const std::string& model_path)
-        : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {} }, m_tag(tag) {}
+        : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {}
 
     void cfgMeanStdDev(const std::string &layer,
                        const cv::Scalar &m,
@@ -318,6 +457,14 @@ class Params<cv::gapi::Generic> {
         desc.generic_norm[layer] = flag;
     }
 
+    void cfgExecutionProvider(ep::OpenVINO&& ov_ep) {
+        desc.execution_provider = std::move(ov_ep);
+    }
+
+    void cfgDisableMemPattern() {
+        desc.disable_mem_pattern = true;
+    }
+
     // BEGIN(G-API's network parametrization API)
     GBackend      backend() const { return cv::gapi::onnx::backend(); }
     std::string   tag()     const { return m_tag; }
diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp
@@ -29,6 +29,7 @@ using map_string_and_string         = std::map<std::string, std::string>;
 using map_string_and_vector_size_t  = std::map<std::string, std::vector<size_t>>;
 using map_string_and_vector_float   = std::map<std::string, std::vector<float>>;
 using map_int_and_double            = std::map<int, double>;
+using ep_OpenVINO                   = cv::gapi::onnx::ep::OpenVINO;
 
 // NB: Python wrapper generate T_U for T<U>
 // This behavior is only observed for inputs
diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp
@@ -21,6 +21,18 @@ cv::gapi::onnx::PyParams& cv::gapi::onnx::PyParams::cfgNormalize(const std::stri
     return *this;
 }
 
+cv::gapi::onnx::PyParams&
+cv::gapi::onnx::PyParams::cfgExecutionProvider(cv::gapi::onnx::ep::OpenVINO ov_ep) {
+    m_priv->cfgExecutionProvider(std::move(ov_ep));
+    return *this;
+}
+
+cv::gapi::onnx::PyParams&
+cv::gapi::onnx::PyParams::cfgDisableMemPattern() {
+    m_priv->cfgDisableMemPattern();
+    return *this;
+}
+
 cv::gapi::GBackend cv::gapi::onnx::PyParams::backend() const {
     return m_priv->backend();
 }
diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp
@@ -143,6 +143,41 @@ class ONNXCompiled {
     void run();
 };
 
+static void appendExecutionProvider(Ort::SessionOptions          *session_options,
+                                    const cv::gapi::onnx::ep::EP &execution_provider) {
+    namespace ep = cv::gapi::onnx::ep;
+    switch (execution_provider.index()) {
+        case ep::EP::index_of<ep::OpenVINO>(): {
+             GAPI_LOG_INFO(NULL, "OpenVINO Execution Provider is selected.");
+             const auto &ovep = cv::util::get<ep::OpenVINO>(execution_provider);
+             OrtOpenVINOProviderOptions options;
+             options.device_id = ovep.device_id.c_str();
+             options.cache_dir = ovep.cache_dir.c_str();
+             options.enable_opencl_throttling = ovep.enable_opencl_throttling;
+             options.enable_dynamic_shapes = ovep.enable_dynamic_shapes;
+             // NB: If are not specified, will be taken from onnxruntime build.
+             if (ovep.device_type) {
+                options.device_type = ovep.device_type->c_str();
+             }
+             if (ovep.num_of_threads) {
+                options.num_of_threads = *ovep.num_of_threads;
+             }
+             try {
+                session_options->AppendExecutionProvider_OpenVINO(options);
+             } catch (const std::exception &e) {
+                 std::stringstream ss;
+                 ss << "ONNX Backend: Failed to enable OpenVINO Execution Provider: "
+                    << e.what() << "\nMake sure that onnxruntime has"
+                                   " been compiled with OpenVINO support.";
+                 cv::util::throw_error(std::runtime_error(ss.str()));
+             }
+             break;
+        }
+        default:
+            break;
+    }
+}
+
 } // namespace onnx
 } // namespace gimpl
 } // namespace cv
@@ -592,9 +627,13 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
         cv::util::throw_error(std::logic_error("Please specify output layer names for "
                                                + params.model_path));
     }
-
     // Create and initialize the ONNX session
     Ort::SessionOptions session_options;
+    cv::gimpl::onnx::appendExecutionProvider(&session_options, pp.execution_provider);
+
+    if (pp.disable_mem_pattern) {
+        session_options.DisableMemPattern();
+    }
     this_env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "");
 #ifndef _WIN32
     this_session = Ort::Session(this_env, params.model_path.data(), session_options);
