Merge pull request #20832 from TolyaTalamanov:at/python-gstreamer-source

TolyaTalamanov · OrestChura · web-flow · commit 2b79a6ff8f8c · 2022-01-26T14:01:13.000Z
G-API: Wrap GStreamerSource

* Wrap GStreamerSource into python

* Fixed test skipping when can't make Gst-src

* Wrapped GStreamerPipeline class, added dummy test for it

* Fix no_gst testing

* Changed wrap for GStreamerPipeline::getStreamingSource() : now python-specific in-class method GStreamerPipeline::get_streaming_source()

* Added accuracy tests vs OCV:VideoCapture(Gstreamer)

* Add skipping when can't use VideoCapture(GSTREAMER);
Add better handling of GStreamer backend unavailable;
Changed video to avoid terminations

* Applying comments

* back to a separate get_streaming_source function, with comment

Co-authored-by: OrestChura &lt;orest.chura@intel.com&gt;
diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt
@@ -55,6 +55,7 @@ file(GLOB gapi_ext_hdrs
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/gstreamer/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/onevpl/*.hpp"
+    "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/util/*.hpp"
     )
 
diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp
@@ -19,12 +19,12 @@ namespace gapi {
 namespace wip {
 namespace gst {
 
-class GAPI_EXPORTS GStreamerPipeline
+class GAPI_EXPORTS_W GStreamerPipeline
 {
 public:
     class Priv;
 
-    explicit GStreamerPipeline(const std::string& pipeline);
+    GAPI_WRAP explicit GStreamerPipeline(const std::string& pipeline);
     IStreamSource::Ptr getStreamingSource(const std::string& appsinkName,
                                           const GStreamerSource::OutputType outputType =
                                               GStreamerSource::OutputType::MAT);
@@ -40,6 +40,18 @@ class GAPI_EXPORTS GStreamerPipeline
 
 using GStreamerPipeline = gst::GStreamerPipeline;
 
+// NB: Function for using from python
+// FIXME: a separate function is created due to absence of wrappers for `shared_ptr<> `
+// Ideally would be to wrap the `GStreamerPipeline::getStreamingSource()` method as is
+GAPI_EXPORTS_W cv::Ptr<IStreamSource>
+inline get_streaming_source(cv::Ptr<GStreamerPipeline>& pipeline,
+                            const std::string& appsinkName,
+                            const GStreamerSource::OutputType outputType
+                                = GStreamerSource::OutputType::MAT)
+{
+    return pipeline->getStreamingSource(appsinkName, outputType);
+}
+
 } // namespace wip
 } // namespace gapi
 } // namespace cv
diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp
@@ -82,6 +82,14 @@ class GAPI_EXPORTS GStreamerSource : public IStreamSource
 
 using GStreamerSource = gst::GStreamerSource;
 
+// NB: Overload for using from python
+GAPI_EXPORTS_W cv::Ptr<IStreamSource>
+inline make_gst_src(const std::string& pipeline,
+                    const GStreamerSource::OutputType outputType =
+                    GStreamerSource::OutputType::MAT)
+{
+    return make_src<GStreamerSource>(pipeline, outputType);
+}
 } // namespace wip
 } // namespace gapi
 } // namespace cv
diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py
@@ -297,3 +297,5 @@ def kernel_with_params(cls):
 cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly
 
 cv.gapi.streaming.queue_capacity = cv.gapi_streaming_queue_capacity
+
+cv.gapi.wip.GStreamerPipeline = cv.gapi_wip_gst_GStreamerPipeline
diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp
@@ -19,6 +19,7 @@ using detail_ExtractArgsCallback    = cv::detail::ExtractArgsCallback;
 using detail_ExtractMetaCallback    = cv::detail::ExtractMetaCallback;
 using vector_GNetParam              = std::vector<cv::gapi::GNetParam>;
 using gapi_streaming_queue_capacity = cv::gapi::streaming::queue_capacity;
+using GStreamerSource_OutputType    = cv::gapi::wip::GStreamerSource::OutputType;
 
 // NB: Python wrapper generate T_U for T<U>
 // This behavior is only observed for inputs
@@ -230,7 +231,7 @@ PyObject* pyopencv_from(const cv::GArg& value)
     {
         HANDLE_CASE(BOOL,      bool);
         HANDLE_CASE(INT,       int);
-        HANDLE_CASE(INT64,   int64_t);
+        HANDLE_CASE(INT64,     int64_t);
         HANDLE_CASE(DOUBLE,    double);
         HANDLE_CASE(FLOAT,     float);
         HANDLE_CASE(STRING,    std::string);
diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py
@@ -34,6 +34,16 @@ def run(img):
             return img
 
 
+    def convertNV12p2BGR(in_nv12):
+        shape = in_nv12.shape
+        y_height = shape[0] // 3 * 2
+        uv_shape = (shape[0] // 3, shape[1])
+        new_uv_shape = (uv_shape[0], uv_shape[1] // 2, 2)
+        return cv.cvtColorTwoPlane(in_nv12[:y_height,  :],
+                                   in_nv12[ y_height:, :].reshape(new_uv_shape),
+                                   cv.COLOR_YUV2BGR_NV12)
+
+
     class test_gapi_streaming(NewOpenCVTests):
 
         def test_image_input(self):
@@ -229,7 +239,6 @@ def test_video_good_features_to_track(self):
 
 
         def test_gapi_streaming_meta(self):
-            ksize = 3
             path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']])
 
             # G-API
@@ -350,6 +359,189 @@ def test_compile_streaming_meta_and_args(self):
                     cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1)))
 
 
+        def get_gst_source(self, gstpipeline):
+            # NB: Skip test in case gstreamer isn't available.
+            try:
+                return cv.gapi.wip.make_gst_src(gstpipeline)
+            except cv.error as e:
+                if str(e).find('Built without GStreamer support!') == -1:
+                    raise e
+                else:
+                    raise unittest.SkipTest(str(e))
+
+
+        def test_gst_source(self):
+            if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER):
+                raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER")
+
+            gstpipeline = """videotestsrc is-live=true pattern=colors num-buffers=10 !
+                             videorate ! videoscale ! video/x-raw,width=1920,height=1080,
+                             framerate=30/1 ! appsink"""
+
+            g_in = cv.GMat()
+            g_out = cv.gapi.copy(g_in)
+            c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out))
+
+            ccomp = c.compileStreaming()
+
+            source = self.get_gst_source(gstpipeline)
+
+            ccomp.setSource(cv.gin(source))
+            ccomp.start()
+
+            has_frame, output = ccomp.pull()
+            while has_frame:
+                self.assertTrue(output.size != 0)
+                has_frame, output = ccomp.pull()
+
+
+        def open_VideoCapture_gstreamer(self, gstpipeline):
+            try:
+                cap = cv.VideoCapture(gstpipeline, cv.CAP_GSTREAMER)
+            except Exception as e:
+                raise unittest.SkipTest("Backend GSTREAMER can't open the video; " +
+                                        "cause: " + str(e))
+            if not cap.isOpened():
+                raise unittest.SkipTest("Backend GSTREAMER can't open the video")
+            return cap
+
+
+        def test_gst_source_accuracy(self):
+            if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER):
+                raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER")
+
+            path = self.find_file('highgui/video/big_buck_bunny.avi',
+                                  [os.environ['OPENCV_TEST_DATA_PATH']])
+            gstpipeline = """filesrc location=""" + path + """ ! decodebin ! videoconvert !
+                             videoscale ! video/x-raw,format=NV12 ! appsink"""
+
+            # G-API pipeline
+            g_in = cv.GMat()
+            g_out = cv.gapi.copy(g_in)
+            c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out))
+
+            ccomp = c.compileStreaming()
+
+            # G-API Gst-source
+            source = self.get_gst_source(gstpipeline)
+            ccomp.setSource(cv.gin(source))
+            ccomp.start()
+
+            # OpenCV Gst-source
+            cap = self.open_VideoCapture_gstreamer(gstpipeline)
+
+            # Assert
+            max_num_frames = 10
+            for _ in range(max_num_frames):
+                has_expected, expected = cap.read()
+                has_actual,   actual   = ccomp.pull()
+
+                self.assertEqual(has_expected, has_actual)
+
+                if not has_expected:
+                    break
+
+                self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected), actual, cv.NORM_INF))
+
+
+        def get_gst_pipeline(self, gstpipeline):
+            # NB: Skip test in case gstreamer isn't available.
+            try:
+                return cv.gapi.wip.GStreamerPipeline(gstpipeline)
+            except cv.error as e:
+                if str(e).find('Built without GStreamer support!') == -1:
+                    raise e
+                else:
+                    raise unittest.SkipTest(str(e))
+            except SystemError as e:
+                raise unittest.SkipTest(str(e) + ", casued by " + str(e.__cause__))
+
+
+        def test_gst_multiple_sources(self):
+            if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER):
+                raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER")
+
+            gstpipeline = """videotestsrc is-live=true pattern=colors num-buffers=10 !
+                             videorate ! videoscale !
+                             video/x-raw,width=1920,height=1080,framerate=30/1 !
+                             appsink name=sink1
+                             videotestsrc is-live=true pattern=colors num-buffers=10 !
+                             videorate ! videoscale !
+                             video/x-raw,width=1920,height=1080,framerate=30/1 !
+                             appsink name=sink2"""
+
+            g_in1 = cv.GMat()
+            g_in2 = cv.GMat()
+            g_out = cv.gapi.add(g_in1, g_in2)
+            c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out))
+
+            ccomp = c.compileStreaming()
+
+            pp = self.get_gst_pipeline(gstpipeline)
+            src1 = cv.gapi.wip.get_streaming_source(pp, "sink1")
+            src2 = cv.gapi.wip.get_streaming_source(pp, "sink2")
+
+            ccomp.setSource(cv.gin(src1, src2))
+            ccomp.start()
+
+            has_frame, out = ccomp.pull()
+            while has_frame:
+                self.assertTrue(out.size != 0)
+                has_frame, out = ccomp.pull()
+
+
+        def test_gst_multiple_sources_accuracy(self):
+            if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER):
+                raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER")
+
+            path = self.find_file('highgui/video/big_buck_bunny.avi',
+                                  [os.environ['OPENCV_TEST_DATA_PATH']])
+            gstpipeline1 = """filesrc location=""" + path + """ ! decodebin ! videoconvert !
+                              videoscale ! video/x-raw,format=NV12 ! appsink"""
+            gstpipeline2 = """filesrc location=""" + path + """ ! decodebin !
+                              videoflip method=clockwise ! videoconvert ! videoscale !
+                              video/x-raw,format=NV12 ! appsink"""
+            gstpipeline_gapi = gstpipeline1 + ' name=sink1 ' + gstpipeline2 + ' name=sink2'
+
+            # G-API pipeline
+            g_in1 = cv.GMat()
+            g_in2 = cv.GMat()
+            g_out1 = cv.gapi.copy(g_in1)
+            g_out2 = cv.gapi.copy(g_in2)
+            c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out1, g_out2))
+
+            ccomp = c.compileStreaming()
+
+            # G-API Gst-source
+            pp = self.get_gst_pipeline(gstpipeline_gapi)
+
+            src1 = cv.gapi.wip.get_streaming_source(pp, "sink1")
+            src2 = cv.gapi.wip.get_streaming_source(pp, "sink2")
+            ccomp.setSource(cv.gin(src1, src2))
+            ccomp.start()
+
+            # OpenCV Gst-source
+            cap1 = self.open_VideoCapture_gstreamer(gstpipeline1)
+            cap2 = self.open_VideoCapture_gstreamer(gstpipeline2)
+
+            # Assert
+            max_num_frames = 10
+            for _ in range(max_num_frames):
+                has_expected1, expected1 = cap1.read()
+                has_expected2, expected2 = cap2.read()
+                has_actual, (actual1, actual2) = ccomp.pull()
+
+                self.assertEqual(has_expected1, has_expected2)
+                has_expected = has_expected1 and has_expected2
+                self.assertEqual(has_expected, has_actual)
+
+                if not has_expected:
+                    break
+
+                self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected1), actual1, cv.NORM_INF))
+                self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected2), actual2, cv.NORM_INF))
+
+
 
 except unittest.SkipTest as e:
 

Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,7 @@ file(GLOB gapi_ext_hdrs`
`55`	`55`	`"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/*.hpp"`
`56`	`56`	`"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/gstreamer/*.hpp"`
`57`	`57`	`"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/onevpl/*.hpp"`
	`58`	`+ "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp"`
`58`	`59`	`"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/util/*.hpp"`
`59`	`60`	`)`
`60`	`61`