Merge pull request #20925 from TolyaTalamanov:at/enable-2d-inputs-giebackend

TolyaTalamanov · web-flow · commit c7df82460c85 · 2021-11-16T12:52:42.000Z
[G-API] Enable 2D inputs for GIEBackend

* Enable 2D inputs

* Fix typo
diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt
@@ -208,6 +208,10 @@ if(OPENCV_GAPI_INF_ENGINE)
   ocv_target_link_libraries(${the_module} PRIVATE ${INF_ENGINE_TARGET})
 endif()
 
+if (HAVE_NGRAPH)
+    ocv_target_link_libraries(${the_module} PRIVATE ngraph::ngraph)
+endif()
+
 if(HAVE_TBB)
   ocv_target_link_libraries(${the_module} PRIVATE tbb)
 endif()
@@ -223,6 +227,9 @@ set(__test_extra_deps "")
 if(OPENCV_GAPI_INF_ENGINE)
   list(APPEND __test_extra_deps ${INF_ENGINE_TARGET})
 endif()
+if(HAVE_NGRAPH)
+  list(APPEND __test_extra_deps ngraph::ngraph)
+endif()
 ocv_add_accuracy_tests(${__test_extra_deps})
 
 # FIXME: test binary is linked with ADE directly since ADE symbols
@@ -232,6 +239,9 @@ ocv_add_accuracy_tests(${__test_extra_deps})
 if(TARGET opencv_test_gapi)
   target_include_directories(opencv_test_gapi PRIVATE "${CMAKE_CURRENT_LIST_DIR}/src")
   target_link_libraries(opencv_test_gapi PRIVATE ade)
+  if (HAVE_NGRAPH)
+      ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_NGRAPH)
+  endif()
 endif()
 
 if(HAVE_TBB AND TARGET opencv_test_gapi)
diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp
@@ -80,7 +80,10 @@ struct ParamDesc {
     // NB: An optional config to setup RemoteContext for IE
     cv::util::any context_config;
 
-    size_t batch_size;
+    // NB: batch_size can't be equal to 1 by default, because some of models
+    // have 2D (Layout::NC) input and if the first dimension not equal to 1
+    // net.setBatchSize(1) will overwrite it.
+    cv::optional<size_t> batch_size;
 };
 } // namespace detail
 
@@ -123,7 +126,7 @@ template<typename Net> class Params {
               , {}
               , 1u
               , {}
-              , 1u} {
+              , {}} {
     };
 
     /** @overload
@@ -145,7 +148,7 @@ template<typename Net> class Params {
               , {}
               , 1u
               , {}
-              , 1u} {
+              , {}} {
     };
 
     /** @brief Specifies sequence of network input layers names for inference.
@@ -329,7 +332,7 @@ template<typename Net> class Params {
     @return reference to this parameter structure.
     */
     Params<Net>& cfgBatchSize(const size_t size) {
-        desc.batch_size = size;
+        desc.batch_size = cv::util::make_optional(size);
         return *this;
     }
 
@@ -367,7 +370,7 @@ class Params<cv::gapi::Generic> {
            const std::string &device)
         : desc{ model, weights, device, {}, {}, {}, 0u, 0u,
                 detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u,
-                {}, 1u},
+                {}, {}},
           m_tag(tag) {
     };
 
@@ -385,7 +388,7 @@ class Params<cv::gapi::Generic> {
            const std::string &device)
         : desc{ model, {}, device, {}, {}, {}, 0u, 0u,
                 detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u,
-                {}, 1u},
+                {}, {}},
           m_tag(tag) {
     };
 
@@ -454,7 +457,7 @@ class Params<cv::gapi::Generic> {
 
     /** @see ie::Params::cfgBatchSize */
     Params& cfgBatchSize(const size_t size) {
-        desc.batch_size = size;
+        desc.batch_size = cv::util::make_optional(size);
         return *this;
     }
 
diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp
@@ -127,7 +127,6 @@ inline int toCV(IE::Precision prec) {
 
 inline IE::TensorDesc toIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) {
     const auto &sz = mat.size;
-
     // NB: For some reason RGB image is 2D image
     // (since channel component is not counted here).
     // Note: regular 2D vectors also fall into this category
@@ -148,7 +147,6 @@ inline IE::TensorDesc toIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) {
         return IE::TensorDesc(toIE(mat.depth()),
                               IE::SizeVector{1, channels, height, width}, bdesc);
     }
-
     return IE::TensorDesc(toIE(mat.depth()), toIE(sz), toIELayout(sz.dims()));
 }
 
@@ -241,7 +239,10 @@ struct IEUnit {
 
         if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) {
             net = cv::gimpl::ie::wrap::readNetwork(params);
-            net.setBatchSize(params.batch_size);
+            // NB: Set batch size only if user asked. (don't set by default)
+            if (params.batch_size.has_value())  {
+                net.setBatchSize(params.batch_size.value());
+            }
         } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) {
             this_plugin = cv::gimpl::ie::wrap::getPlugin(params);
             this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx);
@@ -520,7 +521,9 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) {
                                    blob_params->second);
 }
 
-inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) {
+inline IE::Blob::Ptr extractBlob(IECallContext& ctx,
+                                 std::size_t i,
+                                 cv::gapi::ie::TraitAs hint) {
     if (ctx.uu.rctx != nullptr) {
         return extractRemoteBlob(ctx, i);
     }
@@ -532,7 +535,7 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) {
             return wrapIE(*(ctx.views.back()), frame.desc());
         }
         case cv::GShape::GMAT: {
-            return wrapIE(ctx.inMat(i), cv::gapi::ie::TraitAs::IMAGE);
+            return wrapIE(ctx.inMat(i), hint);
         }
         default:
             GAPI_Assert("Unsupported input shape for IE backend");
@@ -545,6 +548,9 @@ static void setBlob(InferenceEngine::InferRequest& req,
                     const std::string&             layer_name,
                     const IE::Blob::Ptr&           blob,
                     const IECallContext&           ctx) {
+    // TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead,
+    // and redirect our data producers to this memory
+    // (A memory dialog comes to the picture again)
     using namespace cv::gapi::ie::detail;
     if (ctx.uu.params.kind == ParamDesc::Kind::Load) {
         req.SetBlob(layer_name, blob);
@@ -950,7 +956,14 @@ struct Infer: public cv::detail::KernelTag {
                         uu.params.layer_names_to_reshape.end()) {
                         configureInputReshapeByImage(ii, mm, input_reshape_table);
                     }
-                    ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+
+                    // NB: Configure resize only for NCHW/NHWC layout,
+                    // since it isn't supposed to work with others.
+                    auto layout = ii->getTensorDesc().getLayout();
+                    if (layout == IE::Layout::NCHW ||
+                        layout == IE::Layout::NHWC) {
+                        ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+                    }
             }
 
             // FIXME: This isn't the best place to call reshape function.
@@ -1000,11 +1013,16 @@ struct Infer: public cv::detail::KernelTag {
                         // non-generic version for now:
                         // - assumes all inputs/outputs are always Mats
                         for (auto i : ade::util::iota(ctx->uu.params.num_in)) {
-                            // TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead,
-                            // and redirect our data producers to this memory
-                            // (A memory dialog comes to the picture again)
-                            IE::Blob::Ptr this_blob = extractBlob(*ctx, i);
-                            setBlob(req, ctx->uu.params.input_names[i], this_blob, *ctx);
+                            const auto& layer_name = ctx->uu.params.input_names[i];
+                            auto layout =
+                                ctx->uu.this_network.GetInputsInfo().
+                                    at(layer_name)->getTensorDesc().getLayout();
+                            auto hint =
+                                (layout == IE::Layout::NCHW || layout == IE::Layout::NHWC)
+                                ? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR;
+
+                            IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint);
+                            setBlob(req, layer_name, this_blob, *ctx);
                         }
                         // FIXME: Should it be done by kernel ?
                         // What about to do that in RequestPool ?
@@ -1092,7 +1110,10 @@ struct InferROI: public cv::detail::KernelTag {
                         GAPI_Assert(ctx->uu.params.num_in == 1);
                         auto&& this_roi = ctx->inArg<cv::detail::OpaqueRef>(0).rref<cv::Rect>();
 
-                        IE::Blob::Ptr this_blob = extractBlob(*ctx, 1);
+                        // NB: This blob will be used to make roi from its, so
+                        // it should be treated as image
+                        IE::Blob::Ptr this_blob =
+                            extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE);
                         setBlob(req,
                                 *(ctx->uu.params.input_names.begin()),
                                 IE::make_shared_blob(this_blob, toIE(this_roi)),
@@ -1187,7 +1208,9 @@ struct InferList: public cv::detail::KernelTag {
             return;
         }
 
-        IE::Blob::Ptr this_blob = extractBlob(*ctx, 1);
+        // NB: This blob will be used to make roi from its, so
+        // it should be treated as image
+        IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE);
 
         std::vector<std::vector<int>> cached_dims(ctx->uu.params.num_out);
         for (auto i : ade::util::iota(ctx->uu.params.num_out)) {
@@ -1331,7 +1354,9 @@ struct InferList2: public cv::detail::KernelTag {
                     cv::gimpl::ie::RequestPool    &reqPool) {
         GAPI_Assert(ctx->inArgs().size() > 1u
                 && "This operation must have at least two arguments");
-        IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0);
+        // NB: This blob will be used to make roi from its, so
+        // it should be treated as image
+        IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE);
         const auto list_size = ctx->inArg<cv::detail::VectorRef>(1u).size();
         if (list_size == 0u) {
             for (auto i : ade::util::iota(ctx->uu.params.num_out)) {
diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp
@@ -22,6 +22,20 @@
 #include "backends/ie/util.hpp"
 #include "backends/ie/giebackend/giewrapper.hpp"
 
+#ifdef HAVE_NGRAPH
+#if defined(__clang__)  // clang or MSVC clang
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 4100)
+#elif defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+#include <ngraph/ngraph.hpp>
+#endif
+
 namespace opencv_test
 {
 namespace {
@@ -2858,6 +2872,66 @@ TEST(TestAgeGender, ThrowBlobAndInputPrecisionMismatch)
                      cv::compile_args(cv::gapi::networks(pp))));
 }
 
+#ifdef HAVE_NGRAPH
+
+TEST(Infer, ModelWith2DInputs)
+{
+    const std::string model_name   = "ModelWith2DInputs";
+    const std::string model_path   = model_name + ".xml";
+    const std::string weights_path = model_name + ".bin";
+    const std::string device_id    = "CPU";
+    const int W                    = 10;
+    const int H                    = 5;
+
+    // NB: Define model with 2D inputs.
+    auto in1 = std::make_shared<ngraph::op::Parameter>(
+        ngraph::element::Type_t::u8,
+        ngraph::Shape(std::vector<size_t>{{H, W}})
+    );
+    auto in2 = std::make_shared<ngraph::op::Parameter>(
+        ngraph::element::Type_t::u8,
+        ngraph::Shape(std::vector<size_t>{{H, W}})
+    );
+    auto result = std::make_shared<ngraph::op::v1::Add>(in1, in2);
+    auto func   = std::make_shared<ngraph::Function>(
+        ngraph::OutputVector{result},
+        ngraph::ParameterVector{in1, in2}
+    );
+
+    cv::Mat in_mat1(std::vector<int>{H, W}, CV_8U),
+            in_mat2(std::vector<int>{H, W}, CV_8U),
+            gapi_mat, ref_mat;
+
+    cv::randu(in_mat1, 0, 100);
+    cv::randu(in_mat2, 0, 100);
+    cv::add(in_mat1, in_mat2, ref_mat, cv::noArray(), CV_32F);
+
+    // Compile xml file
+    IE::CNNNetwork(func).serialize(model_path);
+
+    // Configure & run G-API
+    cv::GMat g_in1, g_in2;
+    cv::GInferInputs inputs;
+    inputs[in1->get_name()] = g_in1;
+    inputs[in2->get_name()] = g_in2;
+    auto outputs = cv::gapi::infer<cv::gapi::Generic>(model_name, inputs);
+    auto out = outputs.at(result->get_name());
+
+    cv::GComputation comp(cv::GIn(g_in1, g_in2), cv::GOut(out));
+
+    auto pp = cv::gapi::ie::Params<cv::gapi::Generic>(model_name,
+                                                      model_path,
+                                                      weights_path,
+                                                      device_id);
+
+    comp.apply(cv::gin(in_mat1, in_mat2), cv::gout(gapi_mat),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+    normAssert(ref_mat, gapi_mat, "Test model output");
+}
+
+#endif // HAVE_NGRAPH
+
 } // namespace opencv_test
 
 #endif //  HAVE_INF_ENGINE
