Merge pull request #20747 from rgarnov:rg/desync_rmats_and_media

alalek · alalek · commit 1261f250c6cb · 2021-09-24T13:27:15.000Z
diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp
@@ -2,7 +2,7 @@
 // It is subject to the license terms in the LICENSE file found in the top-level directory
 // of this distribution and at https://opencv.org/license.html.
 //
-// Copyright (C) 2018 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 
 
 #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP
@@ -65,6 +65,7 @@ using OptionalOpaqueRef = OptRef<cv::detail::OpaqueRef>;
 using GOptRunArgP = util::variant<
     optional<cv::Mat>*,
     optional<cv::RMat>*,
+    optional<cv::MediaFrame>*,
     optional<cv::Scalar>*,
     cv::detail::OptionalVectorRef,
     cv::detail::OptionalOpaqueRef
@@ -74,6 +75,7 @@ using GOptRunArgsP = std::vector<GOptRunArgP>;
 using GOptRunArg = util::variant<
     optional<cv::Mat>,
     optional<cv::RMat>,
+    optional<cv::MediaFrame>,
     optional<cv::Scalar>,
     optional<cv::detail::VectorRef>,
     optional<cv::detail::OpaqueRef>
@@ -95,6 +97,14 @@ template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Mat> &m) {
     return GOptRunArgP{&m};
 }
 
+template<> inline GOptRunArgP wrap_opt_arg(optional<cv::RMat> &m) {
+    return GOptRunArgP{&m};
+}
+
+template<> inline GOptRunArgP wrap_opt_arg(optional<cv::MediaFrame> &f) {
+    return GOptRunArgP{&f};
+}
+
 template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Scalar> &s) {
     return GOptRunArgP{&s};
 }
diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp
@@ -2,7 +2,7 @@
 // It is subject to the license terms in the LICENSE file found in the top-level directory
 // of this distribution and at https://opencv.org/license.html.
 //
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 
 
 #ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP
@@ -73,9 +73,10 @@ G desync(const G &g) {
  * which produces an array of cv::util::optional<> objects.
  *
  * @note This feature is highly experimental now and is currently
- * limited to a single GMat argument only.
+ * limited to a single GMat/GFrame argument only.
  */
 GAPI_EXPORTS GMat desync(const GMat &g);
+GAPI_EXPORTS GFrame desync(const GFrame &f);
 
 } // namespace streaming
 } // namespace gapi
diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp
@@ -2,7 +2,7 @@
 // It is subject to the license terms in the LICENSE file found in the top-level directory
 // of this distribution and at https://opencv.org/license.html.
 //
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 
 #include "precomp.hpp"
 
@@ -75,6 +75,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) {
     // object will feed both branches of the streaming executable.
 }
 
+// All notes from the above desync(GMat) are also applicable here
+cv::GFrame cv::gapi::streaming::desync(const cv::GFrame &f) {
+    return cv::gapi::copy(detail::desync(f));
+}
+
 cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) {
     return cv::gapi::streaming::GBGR::on(in);
 }
diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp
@@ -186,8 +186,9 @@ void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs)
         // FIXME: this conversion should be unified
         switch (out_obj.index())
         {
-            HANDLE_CASE(cv::Scalar); break;
-            HANDLE_CASE(cv::RMat);   break;
+            HANDLE_CASE(cv::Scalar);     break;
+            HANDLE_CASE(cv::RMat);       break;
+            HANDLE_CASE(cv::MediaFrame); break;
 
         case T::index_of<O<cv::Mat>*>(): {
             // Mat: special handling.
diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp
@@ -2,7 +2,7 @@
 // It is subject to the license terms in the LICENSE file found in the top-level directory
 // of this distribution and at https://opencv.org/license.html.
 //
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 
 
 #include "../test_precomp.hpp"
@@ -2284,4 +2284,70 @@ TEST(OneVPL_Source, Init)
     EXPECT_TRUE(stream_data_provider->empty());
 }
 #endif
+
+TEST(GAPI_Streaming, TestDesyncRMat) {
+    cv::GMat in;
+    auto blurred = cv::gapi::blur(in, cv::Size{3,3});
+    auto desynced = cv::gapi::streaming::desync(blurred);
+    auto out = in - blurred;
+    auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)).compileStreaming();
+
+    cv::Size sz(32,32);
+    cv::Mat in_mat(sz, CV_8UC3);
+    cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255));
+    pipe.setSource(cv::gin(in_mat));
+    pipe.start();
+
+    cv::optional<cv::RMat> out_desync;
+    cv::optional<cv::RMat> out_rmat;
+    while (true) {
+        // Initially it throwed "bad variant access" since there was
+        // no RMat handling in wrap_opt_arg
+        EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_rmat)));
+        if (out_rmat) break;
+    }
+}
+
+G_API_OP(GTestBlur, <GFrame(GFrame)>, "test.blur") {
+    static GFrameDesc outMeta(GFrameDesc d) { return d; }
+};
+GAPI_OCV_KERNEL(GOcvTestBlur, GTestBlur) {
+    static void run(const cv::MediaFrame& in, cv::MediaFrame& out) {
+        auto d = in.desc();
+        GAPI_Assert(d.fmt == cv::MediaFormat::BGR);
+        auto view = in.access(cv::MediaFrame::Access::R);
+        cv::Mat mat(d.size, CV_8UC3, view.ptr[0]);
+        cv::Mat blurred;
+        cv::blur(mat, blurred, cv::Size{3,3});
+        out = cv::MediaFrame::Create<TestMediaBGR>(blurred);
+    }
+};
+
+TEST(GAPI_Streaming, TestDesyncMediaFrame) {
+    initTestDataPath();
+    cv::GFrame in;
+    auto blurred = GTestBlur::on(in);
+    auto desynced = cv::gapi::streaming::desync(blurred);
+    auto out = GTestBlur::on(blurred);
+    auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out))
+        .compileStreaming(cv::compile_args(cv::gapi::kernels<GOcvTestBlur>()));
+
+    std::string filepath = findDataFile("cv/video/768x576.avi");
+    try {
+        pipe.setSource<BGRSource>(filepath);
+    } catch(...) {
+        throw SkipTestException("Video file can not be opened");
+    }
+    pipe.start();
+
+    cv::optional<cv::MediaFrame> out_desync;
+    cv::optional<cv::MediaFrame> out_frame;
+    while (true) {
+        // Initially it throwed "bad variant access" since there was
+        // no MediaFrame handling in wrap_opt_arg
+        EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_frame)));
+        if (out_frame) break;
+    }
+}
+
 } // namespace opencv_test
