Merge pull request #18493 from TolyaTalamanov:at/wrap-streaming

TolyaTalamanov · web-flow · commit 0d3e05f9b3a3 · 2020-10-14T22:21:09.000Z
[G-API Wrap streaming

* Wrap streaming

* Fix build

* Add comments

* Remove comment

* Fix comments to review

* Add test for python pull overload
diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp
@@ -436,7 +436,7 @@ class GAPI_EXPORTS_W GComputation
      *
      * @sa @ref gapi_compile_args
      */
-    GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {});
+    GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {});
 
     /**
      * @brief Compile the computation for streaming mode.
@@ -457,7 +457,7 @@ class GAPI_EXPORTS_W GComputation
      *
      * @sa @ref gapi_compile_args
      */
-    GStreamingCompiled compileStreaming(GCompileArgs &&args = {});
+    GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {});
 
     // 2. Direct metadata version
     /**
diff --git a/modules/gapi/include/opencv2/gapi/gproto.hpp b/modules/gapi/include/opencv2/gapi/gproto.hpp
@@ -135,7 +135,7 @@ GRunArg value_of(const GOrigin &origin);
 // Transform run-time computation arguments into a collection of metadata
 // extracted from that arguments
 GMetaArg  GAPI_EXPORTS descr_of(const GRunArg  &arg );
-GMetaArgs GAPI_EXPORTS descr_of(const GRunArgs &args);
+GMetaArgs GAPI_EXPORTS_W descr_of(const GRunArgs &args);
 
 // Transform run-time operation result argument into metadata extracted from that argument
 // Used to compare the metadata, which generated at compile time with the metadata result operation in run time
diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp
@@ -49,11 +49,11 @@ namespace cv {
  *
  * @sa GCompiled
  */
-class GAPI_EXPORTS GStreamingCompiled
+class GAPI_EXPORTS_W_SIMPLE GStreamingCompiled
 {
 public:
     class GAPI_EXPORTS Priv;
-    GStreamingCompiled();
+    GAPI_WRAP GStreamingCompiled();
 
     // FIXME: More overloads?
     /**
@@ -96,7 +96,7 @@ class GAPI_EXPORTS GStreamingCompiled
      * @param ins vector of inputs to process.
      * @sa gin
      */
-    void setSource(GRunArgs &&ins);
+    GAPI_WRAP void setSource(GRunArgs &&ins);
 
     /**
      * @brief Specify an input video stream for a single-input
@@ -109,7 +109,7 @@ class GAPI_EXPORTS GStreamingCompiled
      * @param s a shared pointer to IStreamSource representing the
      * input video stream.
      */
-    void setSource(const gapi::wip::IStreamSource::Ptr& s);
+    GAPI_WRAP void setSource(const gapi::wip::IStreamSource::Ptr& s);
 
     /**
      * @brief Start the pipeline execution.
@@ -126,7 +126,7 @@ class GAPI_EXPORTS GStreamingCompiled
      * start()/stop()/setSource() may be called on the same object in
      * multiple threads in your application.
      */
-    void start();
+    GAPI_WRAP void start();
 
     /**
      * @brief Get the next processed frame from the pipeline.
@@ -150,6 +150,9 @@ class GAPI_EXPORTS GStreamingCompiled
      */
     bool pull(cv::GRunArgsP &&outs);
 
+    // NB: Used from python
+    GAPI_WRAP std::tuple<bool, cv::GRunArgs> pull();
+
     /**
      * @brief Try to get the next processed frame from the pipeline.
      *
@@ -172,7 +175,7 @@ class GAPI_EXPORTS GStreamingCompiled
      *
      * Throws if the pipeline is not running.
      */
-    void stop();
+    GAPI_WRAP void stop();
 
     /**
      * @brief Test if the pipeline is running.
@@ -184,7 +187,7 @@ class GAPI_EXPORTS GStreamingCompiled
      *
      * @return true if the current stream is not over yet.
      */
-    bool running() const;
+    GAPI_WRAP bool running() const;
 
     /// @private
     Priv& priv();
diff --git a/modules/gapi/include/opencv2/gapi/imgproc.hpp b/modules/gapi/include/opencv2/gapi/imgproc.hpp
@@ -497,7 +497,7 @@ The median filter uses cv::BORDER_REPLICATE internally to cope with border pixel
 @param ksize aperture linear size; it must be odd and greater than 1, for example: 3, 5, 7 ...
 @sa  boxFilter, gaussianBlur
  */
-GAPI_EXPORTS GMat medianBlur(const GMat& src, int ksize);
+GAPI_EXPORTS_W GMat medianBlur(const GMat& src, int ksize);
 
 /** @brief Erodes an image by using a specific structuring element.
 
diff --git a/modules/gapi/include/opencv2/gapi/streaming/cap.hpp b/modules/gapi/include/opencv2/gapi/streaming/cap.hpp
@@ -103,6 +103,12 @@ class GCaptureSource: public IStreamSource
     }
 };
 
+// NB: Overload for using from python
+GAPI_EXPORTS_W cv::Ptr<IStreamSource> inline make_capture_src(const std::string& path)
+{
+    return make_src<GCaptureSource>(path);
+}
+
 } // namespace wip
 } // namespace gapi
 } // namespace cv
diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp
@@ -3,7 +3,14 @@
 
 #ifdef HAVE_OPENCV_GAPI
 
+// NB: Python wrapper replaces :: with _ for classes
 using gapi_GKernelPackage = cv::gapi::GKernelPackage;
+using gapi_wip_IStreamSource_Ptr = cv::Ptr<cv::gapi::wip::IStreamSource>;
+
+// FIXME: Python wrapper generate code without namespace std,
+// so it cause error: "string wasn't declared"
+// WA: Create using
+using std::string;
 
 template<>
 bool pyopencv_to(PyObject* obj, std::vector<GCompileArg>& value, const ArgInfo& info)
@@ -78,6 +85,18 @@ PyObject* pyopencv_from(const GRunArgs& value)
     return list;
 }
 
+template<>
+bool pyopencv_to(PyObject* obj, GMetaArgs& value, const ArgInfo& info)
+{
+    return pyopencv_to_generic_vec(obj, value, info);
+}
+
+template<>
+PyObject* pyopencv_from(const GMetaArgs& value)
+{
+    return pyopencv_from_generic_vec(value);
+}
+
 template <typename T>
 static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw)
 {
@@ -151,6 +170,19 @@ static PyObject* pyopencv_cv_gin(PyObject* , PyObject* py_args, PyObject* kw)
                 return NULL;
             }
         }
+        else if (PyObject_TypeCheck(item,
+                    reinterpret_cast<PyTypeObject*>(pyopencv_gapi_wip_IStreamSource_TypePtr)))
+        {
+            cv::gapi::wip::IStreamSource::Ptr source =
+                reinterpret_cast<pyopencv_gapi_wip_IStreamSource_t*>(item)->v;
+            args.emplace_back(source);
+        }
+        else
+        {
+            PyErr_SetString(PyExc_TypeError, "cv.gin can works only with cv::Mat,"
+                                             "cv::Scalar, cv::gapi::wip::IStreamSource::Ptr");
+            return NULL;
+        }
     }
 
     return pyopencv_from_generic_vec(args);
diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp
@@ -7,11 +7,22 @@ namespace cv
 
    GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg);
 
+   // NB: This classes doesn't exist in *.so
+   // HACK: Mark them as a class to force python wrapper generate code for this entities
    class GAPI_EXPORTS_W_SIMPLE GProtoArg { };
    class GAPI_EXPORTS_W_SIMPLE GProtoInputArgs { };
    class GAPI_EXPORTS_W_SIMPLE GProtoOutputArgs { };
    class GAPI_EXPORTS_W_SIMPLE GRunArg {  };
+   class GAPI_EXPORTS_W_SIMPLE GMetaArg {  };
 
    using GProtoInputArgs  = GIOProtoArgs<In_Tag>;
    using GProtoOutputArgs = GIOProtoArgs<Out_Tag>;
+
+   namespace gapi
+   {
+       namespace wip
+       {
+           class GAPI_EXPORTS_W IStreamSource { };
+       }
+   }
 } // namespace cv
diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+import numpy as np
+import cv2 as cv
+import os
+
+from tests_common import NewOpenCVTests
+
+class test_gapi_streaming(NewOpenCVTests):
+
+    def test_image_input(self):
+        sz = (1280, 720)
+        in_mat = np.random.randint(0, 100, sz).astype(np.uint8)
+
+        # OpenCV
+        expected = cv.medianBlur(in_mat, 3)
+
+        # G-API
+        g_in = cv.GMat()
+        g_out = cv.gapi.medianBlur(g_in, 3)
+        c = cv.GComputation(g_in, g_out)
+        ccomp = c.compileStreaming(cv.descr_of(cv.gin(in_mat)))
+        ccomp.setSource(cv.gin(in_mat))
+        ccomp.start()
+
+        _, actual = ccomp.pull()
+
+        # Assert
+        self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF))
+
+
+    def test_video_input(self):
+        ksize = 3
+        path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']])
+
+        # OpenCV
+        cap = cv.VideoCapture(path)
+
+        # G-API
+        g_in = cv.GMat()
+        g_out = cv.gapi.medianBlur(g_in, ksize)
+        c = cv.GComputation(g_in, g_out)
+
+        ccomp = c.compileStreaming()
+        source = cv.gapi.wip.make_capture_src(path)
+        ccomp.setSource(source)
+        ccomp.start()
+
+        # Assert
+        while cap.isOpened():
+            has_expected, expected = cap.read()
+            has_actual,   actual   = ccomp.pull()
+
+            self.assertEqual(has_expected, has_actual)
+
+            if not has_actual:
+                break
+
+            self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF))
+
+
+    def test_video_split3(self):
+        path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']])
+
+        # OpenCV
+        cap = cv.VideoCapture(path)
+
+        # G-API
+        g_in = cv.GMat()
+        b, g, r = cv.gapi.split3(g_in)
+        c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r))
+
+        ccomp = c.compileStreaming()
+        source = cv.gapi.wip.make_capture_src(path)
+        ccomp.setSource(source)
+        ccomp.start()
+
+        # Assert
+        while cap.isOpened():
+            has_expected, frame = cap.read()
+            has_actual,   actual   = ccomp.pull()
+
+            self.assertEqual(has_expected, has_actual)
+
+            if not has_actual:
+                break
+
+            expected = cv.split(frame)
+            for e, a in zip(expected, actual):
+                self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF))
+
+
+    def test_video_add(self):
+        sz = (576, 768, 3)
+        in_mat = np.random.randint(0, 100, sz).astype(np.uint8)
+
+        path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']])
+
+        # OpenCV
+        cap = cv.VideoCapture(path)
+
+        # G-API
+        g_in1 = cv.GMat()
+        g_in2 = cv.GMat()
+        out = cv.gapi.add(g_in1, g_in2)
+        c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out))
+
+        ccomp = c.compileStreaming()
+        source = cv.gapi.wip.make_capture_src(path)
+        ccomp.setSource(cv.gin(source, in_mat))
+        ccomp.start()
+
+        # Assert
+        while cap.isOpened():
+            has_expected, frame  = cap.read()
+            has_actual,   actual = ccomp.pull()
+
+            self.assertEqual(has_expected, has_actual)
+
+            if not has_actual:
+                break
+
+            expected = cv.add(frame, in_mat)
+            self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF))
+
+
+
+if __name__ == '__main__':
+    NewOpenCVTests.bootstrap()
diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp
@@ -448,6 +448,14 @@ cv::GStreamingCompiled cv::gimpl::GCompiler::produceStreamingCompiled(GPtr &&pg)
         outMetas = GModel::ConstGraph(*pg).metadata().get<OutputMeta>().outMeta;
     }
 
+    auto out_desc = GModel::ConstGraph(*pg).metadata().get<cv::gimpl::Protocol>().outputs;
+    GShapes out_shapes;
+    for (auto&& desc : out_desc)
+    {
+        out_shapes.push_back(desc.shape);
+    }
+    compiled.priv().setOutShapes(std::move(out_shapes));
+
     std::unique_ptr<GStreamingExecutor> pE(new GStreamingExecutor(std::move(pg),
                                                                   m_args));
     if (!m_metas.empty() && !outMetas.empty())
diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp
@@ -111,6 +111,39 @@ bool cv::GStreamingCompiled::pull(cv::GRunArgsP &&outs)
     return m_priv->pull(std::move(outs));
 }
 
+std::tuple<bool, cv::GRunArgs> cv::GStreamingCompiled::pull()
+{
+    GRunArgs run_args;
+    GRunArgsP outs;
+    const auto& out_shapes = m_priv->outShapes();
+    run_args.reserve(out_shapes.size());
+    outs.reserve(out_shapes.size());
+
+    for (auto&& shape : out_shapes)
+    {
+        switch (shape)
+        {
+            case cv::GShape::GMAT:
+            {
+                run_args.emplace_back(cv::Mat{});
+                outs.emplace_back(&cv::util::get<cv::Mat>(run_args.back()));
+                break;
+            }
+            case cv::GShape::GSCALAR:
+            {
+                run_args.emplace_back(cv::Scalar{});
+                outs.emplace_back(&cv::util::get<cv::Scalar>(run_args.back()));
+                break;
+            }
+            default:
+                util::throw_error(std::logic_error("Only cv::GMat and cv::GScalar are supported for python output"));
+        }
+    }
+
+    bool is_over = m_priv->pull(std::move(outs));
+    return std::make_tuple(is_over, run_args);
+}
+
 bool cv::GStreamingCompiled::try_pull(cv::GRunArgsP &&outs)
 {
     return m_priv->try_pull(std::move(outs));
diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp
@@ -27,6 +27,7 @@ class GAPI_EXPORTS GStreamingCompiled::Priv
     GMetaArgs  m_metas;    // passed by user
     GMetaArgs  m_outMetas; // inferred by compiler
     std::unique_ptr<cv::gimpl::GStreamingExecutor> m_exec;
+    GShapes m_out_shapes;
 
 public:
     void setup(const GMetaArgs &metaArgs,
@@ -45,6 +46,11 @@ class GAPI_EXPORTS GStreamingCompiled::Priv
     void stop();
 
     bool running() const;
+
+    // NB: std::tuple<bool, cv::GRunArgs> pull() creates GRunArgs for outputs,
+    // so need to know out shapes to create corresponding GRunArg
+    void setOutShapes(GShapes shapes) { m_out_shapes = std::move(shapes); }
+    const GShapes& outShapes() const { return m_out_shapes; }
 };
 
 } // namespace cv
diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp
