Merge pull request #21658 from sivanov-work:vpp_core_add_roi

sivanov-work · web-flow · commit 44c2c77548fc · 2022-03-05T13:50:40.000+03:00
G-API: Add ROI processing in VPP preproc

* Add ROI in VPP prepro

* Apply comments
diff --git a/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp b/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp
@@ -205,14 +205,15 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)
     }
     VPPPreprocEngine preproc_engine(std::move(policy));
     cv::gapi::wip::Data out;
+    cv::util::optional<cv::Rect> empty_roi;
     TEST_CYCLE()
     {
         source_ptr->pull(out);
         cv::MediaFrame frame = cv::util::get<cv::MediaFrame>(out);
         cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
         pp_session sess = preproc_engine.initialize_preproc(param.value(),
                                                             required_frame_param);
-        (void)preproc_engine.run_sync(sess, frame);
+        (void)preproc_engine.run_sync(sess, frame, empty_roi);
     }
 
     SANITY_CHECK_NOTHING();
@@ -269,14 +270,15 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)
     }
     VPPPreprocEngine preproc_engine(std::move(policy));
     cv::gapi::wip::Data out;
+    cv::util::optional<cv::Rect> empty_roi;
     TEST_CYCLE()
     {
         source_ptr->pull(out);
         cv::MediaFrame frame = cv::util::get<cv::MediaFrame>(out);
         cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
         pp_session sess = preproc_engine.initialize_preproc(param.value(),
                                                             required_frame_param);
-        (void)preproc_engine.run_sync(sess, frame);
+        (void)preproc_engine.run_sync(sess, frame, empty_roi);
     }
 
     SANITY_CHECK_NOTHING();
diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
@@ -34,6 +34,21 @@ bool FrameInfoComparator::equal_to(const mfxFrameInfo& lhs, const mfxFrameInfo&
     return lhs == rhs;
 }
 
+void apply_roi(mfxFrameSurface1* surface_handle,
+               const cv::util::optional<cv::Rect> &opt_roi) {
+    if (opt_roi.has_value()) {
+        const cv::Rect &roi = opt_roi.value();
+        surface_handle->Info.CropX = static_cast<mfxU16>(roi.x);
+        surface_handle->Info.CropY = static_cast<mfxU16>(roi.y);
+        surface_handle->Info.CropW = static_cast<mfxU16>(roi.width);
+        surface_handle->Info.CropH = static_cast<mfxU16>(roi.height);
+        GAPI_LOG_DEBUG(nullptr, "applied ROI {" << surface_handle->Info.CropX <<
+                                ", " << surface_handle->Info.CropY << "}, "
+                                "{ " << surface_handle->Info.CropX + surface_handle->Info.CropW <<
+                                ", " << surface_handle->Info.CropY + surface_handle->Info.CropH << "}");
+    }
+}
+
 VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& accel) :
     ProcessingEngineBase(std::move(accel)) {
     GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine");
@@ -57,31 +72,25 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
                         my_sess.sync_in_queue.pop();
                         auto *vpp_suface = my_sess.processing_surface_ptr.lock()->get_handle();
 
-                       /* TODO: consider CROP/ROI here
-                        static int x_offset = 0;
-                        static int y_offset = 0;
-                        dec_surface->Info.CropX = x_offset;
-                        dec_surface->Info.CropY = y_offset;
-                        dec_surface->Info.CropW = 100 + x_offset++;
-                        dec_surface->Info.CropH = 100 + y_offset++;
-                        */
-                        session_type::outgoing_task vpp_pending_op {pending_op.sync_handle, nullptr};
+                        apply_roi(pending_op.decoded_surface_ptr, pending_op.roi);
+
+                        mfxSyncPoint vpp_sync_handle{};
                         my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session,
                                                                            pending_op.decoded_surface_ptr,
                                                                            vpp_suface,
-                                                                           nullptr, &vpp_pending_op.sync_handle);
-                        vpp_pending_op.vpp_surface_ptr = vpp_suface;
-
+                                                                           nullptr, &vpp_sync_handle);
+                        session_type::outgoing_task vpp_pending_op {vpp_sync_handle,
+                                                                    vpp_suface,
+                                                                    std::move(pending_op) };
                         GAPI_LOG_DEBUG(nullptr, "Got VPP async operation" <<
                                                 ", sync id:  " <<
                                                 vpp_pending_op.sync_handle <<
                                                 ", dec surface:  " <<
-                                                pending_op.decoded_surface_ptr <<
+                                                vpp_pending_op.original_surface_ptr <<
                                                 ", trans surface: " <<
                                                 vpp_pending_op.vpp_surface_ptr <<
                                                 ", status: " <<
                                                 mfxstatus_to_string(my_sess.last_status));
-
                         // NB: process status
                         if (my_sess.last_status == MFX_ERR_MORE_SURFACE ||
                             my_sess.last_status == MFX_ERR_NONE) {
@@ -131,6 +140,7 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
 
                     // put frames in ready queue on success
                     if (MFX_ERR_NONE == sess.last_status) {
+                        pending_op.release_frame();
                         on_frame_ready(my_sess, pending_op.vpp_surface_ptr);
                     }
                 }
@@ -327,8 +337,8 @@ VPPPreprocEngine::initialize_session(mfxSession,
     return {};
 }
 
-cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame) {
-
+cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame,
+                                          const cv::util::optional<cv::Rect> &roi) {
     std::shared_ptr<EngineSession> pp_sess_impl = sess.get<EngineSession>();
     if (!pp_sess_impl) {
         // bypass case
@@ -347,8 +357,10 @@ cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::Medi
 
     // schedule decoded surface into preproc queue
     session_type::incoming_task in_preproc_request {nullptr,
-                                                  vpl_adapter->get_surface()->get_handle(),
-                                                  in_frame};
+                                                    vpl_adapter->get_surface()->get_handle(),
+                                                    vpl_adapter->get_surface()->get_info(),
+                                                    in_frame,
+                                                    roi};
     s->sync_in_queue.emplace(in_preproc_request);
 
     // invoke pipeline to transform decoded surface into preprocessed surface
diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp
@@ -47,7 +47,8 @@ class GAPI_EXPORTS VPPPreprocEngine final : public ProcessingEngineBase,
                                   const GFrameDesc& required_frame_descr) override;
 
     cv::MediaFrame run_sync(const pp_session &session_handle,
-                            const cv::MediaFrame& in_frame) override;
+                            const cv::MediaFrame& in_frame,
+                            const cv::util::optional<cv::Rect> &opt_roi) override;
 
 private:
     std::map<mfxFrameInfo, session_ptr_type, FrameInfoComparator> preproc_session_map;
diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp
@@ -60,6 +60,24 @@ void VPPPreprocSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key)
 const mfxFrameInfo& VPPPreprocSession::get_video_param() const {
     return mfx_vpp_out_param.vpp.Out;
 }
+
+VPPPreprocSession::outgoing_task::outgoing_task(mfxSyncPoint acquired_sync_handle,
+                                                mfxFrameSurface1* acquired_surface_ptr,
+                                                VPPPreprocSession::incoming_task &&in) :
+    sync_handle(acquired_sync_handle),
+    vpp_surface_ptr(acquired_surface_ptr),
+    original_surface_ptr(in.decoded_surface_ptr),
+    original_frame_info(std::move(in.decoded_frame_info)),
+    original_frame(in.decoded_frame_copy) {
+}
+
+void VPPPreprocSession::outgoing_task::release_frame() {
+    // restore initial surface params
+    memcpy(&(original_surface_ptr->Info),
+           &original_frame_info, sizeof(Surface::info_t));
+    // release references on frame adapter
+    original_frame = cv::MediaFrame();
+}
 } // namespace onevpl
 } // namespace wip
 } // namespace gapi
diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp
@@ -41,12 +41,24 @@ class VPPPreprocSession : public EngineSession {
     struct incoming_task {
         mfxSyncPoint sync_handle;
         mfxFrameSurface1* decoded_surface_ptr;
+        Surface::info_t decoded_frame_info;
         cv::MediaFrame decoded_frame_copy;
+        cv::util::optional<cv::Rect> roi;
     };
 
     struct outgoing_task {
+        outgoing_task() = default;
+        outgoing_task(mfxSyncPoint acquired_sync_handle,
+                      mfxFrameSurface1* acquired_surface_ptr,
+                      incoming_task &&in);
         mfxSyncPoint sync_handle;
         mfxFrameSurface1* vpp_surface_ptr;
+
+        mfxFrameSurface1* original_surface_ptr;
+        void release_frame();
+    private:
+        Surface::info_t original_frame_info;
+        cv::MediaFrame original_frame;
     };
 
     std::queue<incoming_task> sync_in_queue;
diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp
@@ -27,7 +27,8 @@ struct IPreprocEngine {
         initialize_preproc(const pp_params& initial_frame_param,
                            const GFrameDesc& required_frame_descr) = 0;
     virtual cv::MediaFrame
-        run_sync(const pp_session &sess, const cv::MediaFrame& in_frame) = 0;
+        run_sync(const pp_session &sess, const cv::MediaFrame& in_frame,
+                 const cv::util::optional<cv::Rect> &opt_roi = {}) = 0;
 };
 } // namespace wip
 } // namespace gapi
diff --git a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp

Original file line number	Diff line number	Diff line change
`@@ -205,14 +205,15 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)`
`205`	`205`	`}`
`206`	`206`	`VPPPreprocEngine preproc_engine(std::move(policy));`
`207`	`207`	`cv::gapi::wip::Data out;`
	`208`	`+ cv::util::optional<cv::Rect> empty_roi;`
`208`	`209`	`TEST_CYCLE()`
`209`	`210`	`{`
`210`	`211`	`source_ptr->pull(out);`
`211`	`212`	`cv::MediaFrame frame = cv::util::get<cv::MediaFrame>(out);`
`212`	`213`	`cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);`
`213`	`214`	`pp_session sess = preproc_engine.initialize_preproc(param.value(),`
`214`	`215`	`required_frame_param);`
`215`		`- (void)preproc_engine.run_sync(sess, frame);`
	`216`	`+ (void)preproc_engine.run_sync(sess, frame, empty_roi);`
`216`	`217`	`}`
`217`	`218`
`218`	`219`	`SANITY_CHECK_NOTHING();`
`@@ -269,14 +270,15 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)`
`269`	`270`	`}`
`270`	`271`	`VPPPreprocEngine preproc_engine(std::move(policy));`
`271`	`272`	`cv::gapi::wip::Data out;`
	`273`	`+ cv::util::optional<cv::Rect> empty_roi;`
`272`	`274`	`TEST_CYCLE()`
`273`	`275`	`{`
`274`	`276`	`source_ptr->pull(out);`
`275`	`277`	`cv::MediaFrame frame = cv::util::get<cv::MediaFrame>(out);`
`276`	`278`	`cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);`
`277`	`279`	`pp_session sess = preproc_engine.initialize_preproc(param.value(),`
`278`	`280`	`required_frame_param);`
`279`		`- (void)preproc_engine.run_sync(sess, frame);`
	`281`	`+ (void)preproc_engine.run_sync(sess, frame, empty_roi);`
`280`	`282`	`}`
`281`	`283`
`282`	`284`	`SANITY_CHECK_NOTHING();`