Merge pull request #25055 from dmatveev:dm/value_initialized_gmat

dmatveev · web-flow · commit f174363f600a · 2024-03-06T17:15:49.000+03:00
G-API: A quick value-initialization support GMat #25055 This PR enables `GMat` objects to be value-initialized in the same way as it was done for `GScalar`s (and, possibly, other types). - Added some helper methods in backends to distinguish if a certain G-type value initialization is supported or not; - Added tests, including negative. Where it is needed: - Further extension of the OVCV backend (#24379 - will be refreshed soon); - Further experiments with DNN module; - Further experiments with "G-API behind UMat" sort of aggregation. In the current form, PR can be reviewed & merged (@TolyaTalamanov please have a look) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
diff --git a/modules/gapi/include/opencv2/gapi/gmat.hpp b/modules/gapi/include/opencv2/gapi/gmat.hpp
@@ -77,6 +77,18 @@ class GAPI_EXPORTS_W_SIMPLE GMat
      */
     GAPI_WRAP GMat();                       // Empty constructor
 
+    /**
+     * @brief Constructs a value-initialized GMat
+     *
+     * GMat may be associated with a buffer at graph construction time.
+     * It is useful when some operation has a Mat input which doesn't
+     * change during the program execution, and is set only once.
+     * In this case, there's no need to declare such GMat as graph input.
+     *
+     * @param m a cv::Mat buffer to associate with this GMat object.
+     */
+    GAPI_WRAP explicit GMat(cv::Mat m);     // Value-initialization constructor
+
     /// @private
     GMat(const GNode &n, std::size_t out);  // Operation result constructor
     /// @private
diff --git a/modules/gapi/include/opencv2/gapi/gscalar.hpp b/modules/gapi/include/opencv2/gapi/gscalar.hpp
@@ -54,12 +54,11 @@ class GAPI_EXPORTS_W_SIMPLE GScalar
     /**
      * @brief Constructs a value-initialized GScalar
      *
-     * In contrast with GMat (which can be either an explicit graph input
-     * or a result of some operation), GScalars may have their values
-     * be associated at graph construction time. It is useful when
-     * some operation has a GScalar input which doesn't change during
-     * the program execution, and is set only once. In this case,
-     * there is no need to declare such GScalar as a graph input.
+     * GScalars may have their values be associated at graph
+     * construction time. It is useful when some operation has a
+     * GScalar input which doesn't change during the program
+     * execution, and is set only once. In this case, there is no need
+     * to declare such GScalar as a graph input.
      *
      * @note The value of GScalar may be overwritten by assigning some
      * other GScalar to the object using `operator=` -- on the
diff --git a/modules/gapi/src/api/gbackend.cpp b/modules/gapi/src/api/gbackend.cpp
@@ -80,6 +80,9 @@ bool cv::gapi::GBackend::Priv::allowsMerge(const cv::gimpl::GIslandModel::Graph
     return true;
 }
 
+bool cv::gapi::GBackend::Priv::supportsConst(cv::GShape) const {
+    return false;
+}
 
 // GBackend public implementation //////////////////////////////////////////////
 cv::gapi::GBackend::GBackend()
diff --git a/modules/gapi/src/api/gbackend_priv.hpp b/modules/gapi/src/api/gbackend_priv.hpp
@@ -84,6 +84,14 @@ class GAPI_EXPORTS cv::gapi::GBackend::Priv
                              const ade::NodeHandle &slot_nh,
                              const ade::NodeHandle &b_nh) const;
 
+    // Ask backend if it supports CONST_VAL data of the given shape or not.
+    // If the backend does support this data type, a Data node with such
+    // value can be fused into the backend's Island body.
+    // If the backend doesn't support this data type, a Data node won't
+    // be fused into the Islands's body -- will be marked as an in-graph
+    // input connection for this Island.
+    virtual bool supportsConst(cv::GShape shape) const;
+
     virtual ~Priv() = default;
 };
 
diff --git a/modules/gapi/src/api/gcomputation.cpp b/modules/gapi/src/api/gcomputation.cpp
@@ -191,8 +191,8 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)
     if (m_priv->m_lastMetas != in_metas)
     {
         if (m_priv->m_lastCompiled &&
-                m_priv->m_lastCompiled.canReshape() &&
-                formats_are_same(m_priv->m_lastMetas, in_metas))
+            m_priv->m_lastCompiled.canReshape() &&
+            formats_are_same(m_priv->m_lastMetas, in_metas))
         {
             m_priv->m_lastCompiled.reshape(in_metas, args);
         }
@@ -203,6 +203,11 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)
         }
         m_priv->m_lastMetas = in_metas;
     }
+    else if (in_metas.size() == 0) {
+        // Happens when the graph is head-less (e.g. starts with const-vals only)
+        // always compile ad-hoc
+        m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args));
+    }
 }
 
 void cv::GComputation::apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args)
diff --git a/modules/gapi/src/api/gmat.cpp b/modules/gapi/src/api/gmat.cpp
@@ -26,6 +26,10 @@ cv::GMat::GMat(const GNode &n, std::size_t out)
 {
 }
 
+cv::GMat::GMat(cv::Mat m)
+    : m_priv(new GOrigin(GShape::GMAT, cv::gimpl::ConstVal(m))) {
+}
+
 cv::GOrigin& cv::GMat::priv()
 {
     return *m_priv;
diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp
@@ -80,6 +80,7 @@ cv::GRunArg cv::value_of(const cv::GOrigin &origin)
     {
     case GShape::GSCALAR: return GRunArg(util::get<cv::Scalar>(origin.value));
     case GShape::GARRAY:  return GRunArg(util::get<cv::detail::VectorRef>(origin.value));
+    case GShape::GMAT:    return GRunArg(util::get<cv::Mat>(origin.value));
     default: util::throw_error(std::logic_error("Unsupported shape for constant"));
     }
 }
diff --git a/modules/gapi/src/backends/cpu/gcpubackend.cpp b/modules/gapi/src/backends/cpu/gcpubackend.cpp
@@ -65,6 +65,17 @@ namespace
         {
             return EPtr{new cv::gimpl::GCPUExecutable(graph, compileArgs, nodes)};
         }
+
+        virtual bool supportsConst(cv::GShape shape) const override
+        {
+            // Supports all types of const values
+            return shape == cv::GShape::GOPAQUE
+                || shape == cv::GShape::GSCALAR
+                || shape == cv::GShape::GARRAY;
+            // yes, value-initialized GMats are not supported currently
+            // as in-island data -- compiler will lift these values to the
+            // GIslandModel's SLOT level (will be handled uniformly)
+        }
    };
 }
 
diff --git a/modules/gapi/src/backends/ocl/goclbackend.cpp b/modules/gapi/src/backends/ocl/goclbackend.cpp
@@ -62,6 +62,17 @@ namespace
         {
             return EPtr{new cv::gimpl::GOCLExecutable(graph, nodes)};
         }
+
+        virtual bool supportsConst(cv::GShape shape) const override
+        {
+            // Supports all types of const values
+            return shape == cv::GShape::GOPAQUE
+                || shape == cv::GShape::GSCALAR
+                || shape == cv::GShape::GARRAY;
+            // yes, value-initialized GMats are not supported currently
+            // as in-island data -- compiler will lift these values to the
+            // GIslandModel's SLOT level (will be handled uniformly)
+        }
    };
 }
 
diff --git a/modules/gapi/src/compiler/gobjref.hpp b/modules/gapi/src/compiler/gobjref.hpp
@@ -24,6 +24,7 @@ namespace gimpl
     < util::monostate
     , cv::Scalar
     , cv::detail::VectorRef
+    , cv::Mat
     >;
 
     struct RcDesc
diff --git a/modules/gapi/src/compiler/passes/exec.cpp b/modules/gapi/src/compiler/passes/exec.cpp
@@ -9,7 +9,7 @@
 
 #include <string>
 #include <list> // list
-#include <iomanip>  // setw, etc
+#include <iomanip> // setw, etc
 #include <fstream> // ofstream
 #include <memory>
 #include <functional>
@@ -85,7 +85,7 @@ namespace
 
         const auto& backend = *src_g.metadata().get<ActiveBackends>().backends.cbegin();
         const auto& proto = src_g.metadata().get<Protocol>();
-        GIsland::node_set all, in_ops, out_ops;
+        GIsland::node_set all, in_ops, out_ops, in_cvals;
 
         all.insert(src_g.nodes().begin(), src_g.nodes().end());
 
@@ -99,7 +99,22 @@ namespace
             all.erase(nh);
             out_ops.insert(nh->inNodes().begin(), nh->inNodes().end());
         }
-
+        for (const auto& nh : src_g.nodes())
+        {
+            if (src_g.metadata(nh).get<NodeType>().t == NodeType::DATA)
+            {
+                const auto &d = src_g.metadata(nh).get<Data>();
+                if (d.storage == Data::Storage::CONST_VAL
+                    && !backend.priv().supportsConst(d.shape)) {
+                    // don't put this node into the island's graph - so the island
+                    // executable don't need to handle value-initialized G-type manually.
+                    // Still mark its readers as inputs
+                    all.erase(nh);
+                    in_cvals.insert(nh);
+                    in_ops.insert(nh->outNodes().begin(), nh->outNodes().end());
+                }
+            }
+        }
         auto isl = std::make_shared<GIsland>(backend,
                                              std::move(all),
                                              std::move(in_ops),
@@ -108,7 +123,8 @@ namespace
 
         auto ih = GIslandModel::mkIslandNode(g, std::move(isl));
 
-        for (const auto& nh : proto.in_nhs)
+        for (const auto& nh : ade::util::chain(ade::util::toRange(proto.in_nhs),
+                                               ade::util::toRange(in_cvals)))
         {
             auto slot = GIslandModel::mkSlotNode(g, nh);
             g.link(slot, ih);
diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp
@@ -208,6 +208,12 @@ void cv::gimpl::GExecutor::initResource(const ade::NodeHandle & nh, const ade::N
     switch (d.shape)
     {
     case GShape::GMAT:
+        if (d.storage == Data::Storage::CONST_VAL)
+        {
+            auto rc = RcDesc{d.rc, d.shape, d.ctor};
+            magazine::bindInArgExec(m_res, rc, m_gm.metadata(orig_nh).get<ConstValue>().arg);
+        }
+        else
         {
             // Let island allocate it's outputs if it can,
             // allocate cv::Mat and wrap it with RMat otherwise
diff --git a/modules/gapi/src/executor/gthreadedexecutor.cpp b/modules/gapi/src/executor/gthreadedexecutor.cpp
@@ -175,7 +175,11 @@ void cv::gimpl::GThreadedExecutor::initResource(const ade::NodeHandle &nh, const
     // to as it is bound externally (e.g. already in the m_state.mag)
 
     switch (d.shape) {
-    case GShape::GMAT: {
+    case GShape::GMAT:
+        if (d.storage == Data::Storage::CONST_VAL) {
+            auto rc = RcDesc{d.rc, d.shape, d.ctor};
+            magazine::bindInArgExec(m_state.mag, rc, m_gm.metadata(orig_nh).get<ConstValue>().arg);
+        } else {
             // Let island allocate it's outputs if it can,
             // allocate cv::Mat and wrap it with RMat otherwise
             GAPI_Assert(!nh->inNodes().empty());
diff --git a/modules/gapi/test/gapi_mat_tests.cpp b/modules/gapi/test/gapi_mat_tests.cpp
@@ -0,0 +1,114 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at https://opencv.org/license.html.
+//
+// Copyright (C) 2024 Intel Corporation
+
+
+#include "test_precomp.hpp"
+
+#include <opencv2/gapi/cpu/core.hpp>
+#include <opencv2/gapi/ocl/core.hpp>
+#include <opencv2/gapi/fluid/core.hpp>
+
+namespace opencv_test
+{
+namespace
+{
+enum class KernelPackage: int
+{
+    OCV,
+    OCL,
+    FLUID,
+};
+std::ostream& operator<< (std::ostream &os, const KernelPackage &e)
+{
+    switch (e)
+    {
+#define _C(X) case KernelPackage::X: os << #X; break
+        _C(OCV);
+        _C(OCL);
+        _C(FLUID);
+#undef _C
+    default: GAPI_Error("Unknown package");
+    }
+    return os;
+}
+} // namespace
+
+struct GMatWithValue : public TestWithParam <KernelPackage> {
+    cv::GKernelPackage getKernelPackage() {
+        switch (GetParam()) {
+        case KernelPackage::OCV: return cv::gapi::core::cpu::kernels();
+        case KernelPackage::OCL: return cv::gapi::core::ocl::kernels();
+        case KernelPackage::FLUID: return cv::gapi::core::fluid::kernels();
+        default: GAPI_Error("Unknown package");
+        }
+    }
+};
+
+TEST_P(GMatWithValue, SingleIsland)
+{
+    cv::Size sz(2, 2);
+    cv::Mat in_mat = cv::Mat::eye(sz, CV_8U);
+
+    cv::GComputationT<cv::GMat(cv::GMat)> addEye([&](cv::GMat in) {
+        return in + cv::GMat(cv::Mat::eye(sz, CV_8U));
+    });
+
+    cv::Mat out_mat;
+    addEye.apply(in_mat, out_mat, cv::compile_args(cv::gapi::use_only{getKernelPackage()}));
+
+    cv::Mat out_mat_ref = in_mat*2;
+    EXPECT_EQ(0, cvtest::norm(out_mat, out_mat_ref, NORM_INF));
+}
+
+TEST_P(GMatWithValue, GraphWithNoInput)
+{
+    cv::Mat cval = cv::Mat::eye(cv::Size(2, 2), CV_8U);
+    cv::GMat gval = cv::GMat(cval);
+    cv::GMat out = cv::gapi::bitwise_not(gval);
+
+    cv::Mat out_mat;
+    cv::GComputation f(cv::GIn(), cv::GOut(out));
+
+    // Compiling this isn't supported for now
+    EXPECT_ANY_THROW(f.compile(cv::descr_of(cval),
+                               cv::compile_args(cv::gapi::use_only{getKernelPackage()})));
+}
+
+INSTANTIATE_TEST_CASE_P(GAPI_GMat, GMatWithValue,
+                        Values(KernelPackage::OCV,
+                               KernelPackage::OCL,
+                               KernelPackage::FLUID));
+
+TEST(GAPI_MatWithValue, MultipleIslands)
+{
+    // This test employs a non-trivial island fusion process
+    // as there's multiple backends in the graph
+
+    cv::Size sz(2, 2);
+    cv::Mat cval2 = cv::Mat::eye(sz, CV_8U) * 2;
+    cv::Mat cval1 = cv::Mat::eye(sz, CV_8U);
+
+    cv::GMat in;
+    cv::GMat tmp = in  + cv::GMat(cval2); // Will be a Fluid operation
+    cv::GMat out = tmp - cv::GMat(cval1); // Will be an OCV operation
+
+    cv::GKernelPackage fluid_kernels = cv::gapi::core::fluid::kernels();
+    cv::GKernelPackage opencv_kernels = cv::gapi::core::cpu::kernels();
+    fluid_kernels.remove<cv::gapi::core::GSub>();
+    opencv_kernels.remove<cv::gapi::core::GAdd>();
+    auto kernels = cv::gapi::combine(fluid_kernels, opencv_kernels);
+
+    cv::Mat in_mat = cv::Mat::zeros(sz, CV_8U);
+    cv::Mat out_mat;
+    auto cc = cv::GComputation(in, out)
+        .compile(cv::descr_of(in_mat),
+                 cv::compile_args(cv::gapi::use_only{kernels}));
+    cc(cv::gin(in_mat), cv::gout(out_mat));
+
+    EXPECT_EQ(0, cvtest::norm(out_mat, cv::Mat::eye(sz, CV_8U), NORM_INF));
+}
+
+} // namespace opencv_test

Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,9 @@ bool cv::gapi::GBackend::Priv::allowsMerge(const cv::gimpl::GIslandModel::Graph`
`80`	`80`	`return true;`
`81`	`81`	`}`
`82`	`82`
	`83`	`+bool cv::gapi::GBackend::Priv::supportsConst(cv::GShape) const {`
	`84`	`+ return false;`
	`85`	`+}`
`83`	`86`
`84`	`87`	`// GBackend public implementation //////////////////////////////////////////////`
`85`	`88`	`cv::gapi::GBackend::GBackend()`
Original file line number	Diff line number	Diff line change
`@@ -191,8 +191,8 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)`
`191`	`191`	`if (m_priv->m_lastMetas != in_metas)`
`192`	`192`	`{`
`193`	`193`	`if (m_priv->m_lastCompiled &&`
`194`		`- m_priv->m_lastCompiled.canReshape() &&`
`195`		`- formats_are_same(m_priv->m_lastMetas, in_metas))`
	`194`	`+ m_priv->m_lastCompiled.canReshape() &&`
	`195`	`+ formats_are_same(m_priv->m_lastMetas, in_metas))`
`196`	`196`	`{`
`197`	`197`	`m_priv->m_lastCompiled.reshape(in_metas, args);`
`198`	`198`	`}`
`@@ -203,6 +203,11 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)`
`203`	`203`	`}`
`204`	`204`	`m_priv->m_lastMetas = in_metas;`
`205`	`205`	`}`
	`206`	`+ else if (in_metas.size() == 0) {`
	`207`	`+ // Happens when the graph is head-less (e.g. starts with const-vals only)`
	`208`	`+ // always compile ad-hoc`
	`209`	`+ m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args));`
	`210`	`+ }`
`206`	`211`	`}`
`207`	`212`
`208`	`213`	`void cv::GComputation::apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args)`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,10 @@ cv::GMat::GMat(const GNode &n, std::size_t out)`
`26`	`26`	`{`
`27`	`27`	`}`
`28`	`28`
	`29`	`+cv::GMat::GMat(cv::Mat m)`
	`30`	`+ : m_priv(new GOrigin(GShape::GMAT, cv::gimpl::ConstVal(m))) {`
	`31`	`+}`
	`32`	`+`
`29`	`33`	`cv::GOrigin& cv::GMat::priv()`
`30`	`34`	`{`
`31`	`35`	`return *m_priv;`
Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,7 @@ cv::GRunArg cv::value_of(const cv::GOrigin &origin)`
`80`	`80`	`{`
`81`	`81`	`case GShape::GSCALAR: return GRunArg(util::get<cv::Scalar>(origin.value));`
`82`	`82`	`case GShape::GARRAY: return GRunArg(util::get<cv::detail::VectorRef>(origin.value));`
	`83`	`+ case GShape::GMAT: return GRunArg(util::get<cv::Mat>(origin.value));`
`83`	`84`	`default: util::throw_error(std::logic_error("Unsupported shape for constant"));`
`84`	`85`	`}`
`85`	`86`	`}`
Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,17 @@ namespace`
`65`	`65`	`{`
`66`	`66`	`return EPtr{new cv::gimpl::GCPUExecutable(graph, compileArgs, nodes)};`
`67`	`67`	`}`
	`68`	`+`
	`69`	`+ virtual bool supportsConst(cv::GShape shape) const override`
	`70`	`+ {`
	`71`	`+ // Supports all types of const values`
	`72`	`+ return shape == cv::GShape::GOPAQUE`
	`73`	`+ \|\| shape == cv::GShape::GSCALAR`
	`74`	`+ \|\| shape == cv::GShape::GARRAY;`
	`75`	`+ // yes, value-initialized GMats are not supported currently`
	`76`	`+ // as in-island data -- compiler will lift these values to the`
	`77`	`+ // GIslandModel's SLOT level (will be handled uniformly)`
	`78`	`+ }`
`68`	`79`	`};`
`69`	`80`	`}`
`70`	`81`
Original file line number	Diff line number	Diff line change
`@@ -62,6 +62,17 @@ namespace`
`62`	`62`	`{`
`63`	`63`	`return EPtr{new cv::gimpl::GOCLExecutable(graph, nodes)};`
`64`	`64`	`}`
	`65`	`+`
	`66`	`+ virtual bool supportsConst(cv::GShape shape) const override`
	`67`	`+ {`
	`68`	`+ // Supports all types of const values`
	`69`	`+ return shape == cv::GShape::GOPAQUE`
	`70`	`+ \|\| shape == cv::GShape::GSCALAR`
	`71`	`+ \|\| shape == cv::GShape::GARRAY;`
	`72`	`+ // yes, value-initialized GMats are not supported currently`
	`73`	`+ // as in-island data -- compiler will lift these values to the`
	`74`	`+ // GIslandModel's SLOT level (will be handled uniformly)`
	`75`	`+ }`
`65`	`76`	`};`
`66`	`77`	`}`
`67`	`78`