Merge pull request #18240 from mpashchenkov:mp/ocv-gapi-input-cnn-reshape

Maxim Pashchenkov · web-flow · commit 12fa8d8444ec · 2021-03-10T16:06:46.000Z
[G-API]: Adding reshape for CNN input.

* Added CNN input IE reshape

* rbs

* Added unordered_set instead vector

* Alignment
diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp
@@ -8,6 +8,7 @@
 #define OPENCV_GAPI_INFER_IE_HPP
 
 #include <unordered_map>
+#include <unordered_set>
 #include <string>
 #include <array>
 #include <tuple> // tuple, tuple_size
@@ -68,6 +69,9 @@ namespace detail {
         bool is_generic;
         IEConfig config;
 
+        std::map<std::string, std::vector<std::size_t>> reshape_table;
+        std::unordered_set<std::string> layer_names_to_reshape;
+
         // NB: Number of asyncrhonious infer requests
         size_t nireq;
     };
@@ -95,6 +99,8 @@ template<typename Net> class Params {
               , detail::ParamDesc::Kind::Load
               , false
               , {}
+              , {}
+              , {}
               , 1u} {
     };
 
@@ -106,6 +112,8 @@ template<typename Net> class Params {
               , detail::ParamDesc::Kind::Import
               , false
               , {}
+              , {}
+              , {}
               , 1u} {
     };
 
@@ -148,6 +156,36 @@ template<typename Net> class Params {
         return *this;
     }
 
+    Params<Net>& cfgInputReshape(std::map<std::string, std::vector<std::size_t>>&& reshape_table) {
+        desc.reshape_table = std::move(reshape_table);
+        return *this;
+    }
+
+    Params<Net>& cfgInputReshape(const std::map<std::string, std::vector<std::size_t>>& reshape_table) {
+        desc.reshape_table = reshape_table;
+        return *this;
+    }
+
+    Params<Net>& cfgInputReshape(std::string&& layer_name, std::vector<size_t>&& layer_dims) {
+        desc.reshape_table.emplace(layer_name, layer_dims);
+        return *this;
+    }
+
+    Params<Net>& cfgInputReshape(const std::string& layer_name, const std::vector<size_t>& layer_dims) {
+        desc.reshape_table.emplace(layer_name, layer_dims);
+        return *this;
+    }
+
+    Params<Net>& cfgInputReshape(std::unordered_set<std::string>&& layer_names) {
+        desc.layer_names_to_reshape = std::move(layer_names);
+        return *this;
+    }
+
+    Params<Net>& cfgInputReshape(const std::unordered_set<std::string>& layer_names) {
+        desc.layer_names_to_reshape = layer_names;
+        return *this;
+    }
+
     // BEGIN(G-API's network parametrization API)
     GBackend      backend()    const { return cv::gapi::ie::backend();  }
     std::string   tag()        const { return Net::tag(); }
@@ -165,13 +203,13 @@ class Params<cv::gapi::Generic> {
            const std::string &model,
            const std::string &weights,
            const std::string &device)
-        : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, 1u}, m_tag(tag) {
+        : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) {
     };
 
     Params(const std::string &tag,
            const std::string &model,
            const std::string &device)
-        : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, 1u}, m_tag(tag) {
+        : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) {
     };
 
     Params& pluginConfig(IEConfig&& cfg) {
diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp
@@ -45,6 +45,7 @@
 #include "backends/ie/giebackend/giewrapper.hpp"
 
 #include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
+#include "logger.hpp"
 
 #if INF_ENGINE_RELEASE < 2021010000
 #include "ie_compound_blob.h"
@@ -224,6 +225,9 @@ struct IEUnit {
             // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap
             inputs  = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo());
             outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo());
+            if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) {
+                GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network");
+            }
         } else {
             cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind"));
         }
@@ -249,6 +253,11 @@ struct IEUnit {
         if (params.num_out == 1u && params.output_names.empty()) {
             params.output_names = { outputs.begin()->first };
         }
+        if (!params.reshape_table.empty()) {
+            GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <=
+                         params.num_in &&
+                        "Number of layers to reshape must be less than or equal to number of inputs");
+        }
     }
 
     // This method is [supposed to be] called at Island compilation stage
@@ -669,6 +678,46 @@ void cv::gimpl::ie::GIEExecutable::run(cv::gimpl::GIslandExecutable::IInput  &in
 namespace cv {
 namespace gimpl {
 namespace ie {
+static void configureInputReshapeByImage(const IE::InputInfo::Ptr& ii,
+                                         const cv::GMetaArg mm,
+                                         IE::ICNNNetwork::InputShapes& input_reshape_table) {
+    const auto& layer_name = ii->name();
+    // Finding name in reshape table
+    const auto name_pos_in_table = input_reshape_table.find(layer_name);
+    // If contains then reshape for this layer already configured by shapes
+    // otherwise create a new element of reshape table with name and dimension
+    // which based on input image size.
+    if (name_pos_in_table != input_reshape_table.end()) {
+        GAPI_Assert(false &&
+                    "Names of layers for reshape with specified dimensions shouldn't intersect with names for reshape by image");
+    }
+    cv::Size image_sz;
+    switch (mm.index()) {
+        case cv::GMetaArg::index_of<cv::GMatDesc>():
+            {
+                const auto &meta = util::get<cv::GMatDesc>(mm);
+                image_sz = meta.size;
+                break;
+            }
+        case cv::GMetaArg::index_of<cv::GFrameDesc>():
+            {
+                const auto &meta = util::get<cv::GFrameDesc>(mm);
+                image_sz = meta.size;
+                break;
+            }
+        default:
+            util::throw_error(std::runtime_error("Unsupported input meta for IE backend"));
+    }
+    auto input_dims = ii->getTensorDesc().getDims();
+    const auto size = input_dims.size();
+    if (size <= 1) {
+        GAPI_Assert(false && "Unsupported number of dimensions for reshape by image");
+    }
+    input_dims.at(size - 2) = static_cast<size_t>(image_sz.height);
+    input_dims.at(size - 1) = static_cast<size_t>(image_sz.width);
+    // Adding new element to reshape table
+    input_reshape_table.emplace(layer_name, input_dims);
+}
 
 static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) {
     switch (mm.index()) {
@@ -732,22 +781,34 @@ struct Infer: public cv::detail::KernelTag {
 
         GConstGIEModel gm(gr);
         const auto &uu = gm.metadata(nh).get<IEUnit>();
+        IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
 
         // Initialize input information
         // Note our input layers list order matches the API order and so
         // meta order.
         GAPI_Assert(uu.params.input_names.size() == in_metas.size()
                     && "Known input layers count doesn't match input meta count");
-
         for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names),
                                         ade::util::toRange(in_metas))) {
-            auto       &&ii = uu.inputs.at(std::get<0>(it));
-            const auto & mm =              std::get<1>(it);
+            const auto &input_name = std::get<0>(it);
+            auto       &&ii = uu.inputs.at(input_name);
+            const auto & mm = std::get<1>(it);
 
             configureInputInfo(ii, mm);
+            if (uu.params.layer_names_to_reshape.find(input_name) !=
+                uu.params.layer_names_to_reshape.end()) {
+                configureInputReshapeByImage(ii, mm, input_reshape_table);
+            }
             ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
         }
 
+        // FIXME: This isn't the best place to call reshape function.
+        // Сorrect solution would be to do this in compile() method of network,
+        // but now input meta isn't passed to compile() method.
+        if (!input_reshape_table.empty()) {
+            const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+        }
+
         // FIXME: It would be nice here to have an exact number of network's
         // input/output parameters. Probably GCall should store it here for us.
         // It doesn't, as far as I know..
@@ -802,18 +863,31 @@ struct InferROI: public cv::detail::KernelTag {
 
         GConstGIEModel gm(gr);
         const auto &uu = gm.metadata(nh).get<IEUnit>();
+        IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
 
         // Initialize input information
         // FIXME: So far it is pretty limited
         GAPI_Assert(1u == uu.params.input_names.size());
         GAPI_Assert(2u == in_metas.size());
 
         // 0th is ROI, 1st is input image
-        auto &&ii = uu.inputs.at(uu.params.input_names.at(0));
+        const auto &input_name = uu.params.input_names.at(0);
+        auto &&ii = uu.inputs.at(input_name);
         auto &&mm = in_metas.at(1u);
         configureInputInfo(ii, mm);
+        if (uu.params.layer_names_to_reshape.find(input_name) !=
+            uu.params.layer_names_to_reshape.end()) {
+            configureInputReshapeByImage(ii, mm, input_reshape_table);
+        }
         ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
 
+        // FIXME: This isn't the best place to call reshape function.
+        // Сorrect solution would be to do this in compile() method of network,
+        // but now input meta isn't passed to compile() method.
+        if (!input_reshape_table.empty()) {
+            const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+        }
+
         // FIXME: It would be nice here to have an exact number of network's
         // input/output parameters. Probably GCall should store it here for us.
         // It doesn't, as far as I know..
@@ -870,6 +944,7 @@ struct InferList: public cv::detail::KernelTag {
 
         GConstGIEModel gm(gr);
         const auto &uu = gm.metadata(nh).get<IEUnit>();
+        IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
 
         // Initialize input information
         // Note our input layers list order matches the API order and so
@@ -882,9 +957,20 @@ struct InferList: public cv::detail::KernelTag {
             auto       &&ii = uu.inputs.at(input_name);
             const auto & mm = in_metas[idx++];
             configureInputInfo(ii, mm);
+            if (uu.params.layer_names_to_reshape.find(input_name) !=
+                uu.params.layer_names_to_reshape.end()) {
+                configureInputReshapeByImage(ii, mm, input_reshape_table);
+            }
             ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
         }
 
+        // FIXME: This isn't the best place to call reshape function.
+        // Сorrect solution would be to do this in compile() method of network,
+        // but now input meta isn't passed to compile() method.
+        if (!input_reshape_table.empty()) {
+            const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+        }
+
         // roi-list version is much easier at the moment.
         // All our outputs are vectors which don't have
         // metadata at the moment - so just create a vector of
@@ -973,6 +1059,7 @@ struct InferList2: public cv::detail::KernelTag {
 
         GConstGIEModel gm(gr);
         const auto &uu = gm.metadata(nh).get<IEUnit>();
+        IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
 
         // Initialize input information
         // Note our input layers list order matches the API order and so
@@ -1023,7 +1110,18 @@ struct InferList2: public cv::detail::KernelTag {
             if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) {
                 // This is a cv::Rect -- configure the IE preprocessing
                 configureInputInfo(ii, mm_0);
+                if (uu.params.layer_names_to_reshape.find(input_name) !=
+                    uu.params.layer_names_to_reshape.end()) {
+                    configureInputReshapeByImage(ii, mm_0, input_reshape_table);
+                }
                 ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+
+                // FIXME: This isn't the best place to call reshape function.
+                // Сorrect solution would be to do this in compile() method of network,
+                // but now input meta isn't passed to compile() method.
+                if (!input_reshape_table.empty()) {
+                    const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+                }
             } else {
                 // This is a cv::GMat (equals to: cv::Mat)
                 // Just validate that it is really the type
diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp
