Merge pull request #19325 from AsyaPronina:asyadev/extract_y

alalek · web-flow · commit 7949983c419d · 2021-03-24T16:37:05.000+03:00
[G-API] Added Y, UV accessors for cv::MediaFrame
diff --git a/modules/gapi/include/opencv2/gapi/streaming/format.hpp b/modules/gapi/include/opencv2/gapi/streaming/format.hpp
@@ -20,6 +20,19 @@ G_API_OP(GBGR, <GMat(GFrame)>, "org.opencv.streaming.BGR")
     static GMatDesc outMeta(const GFrameDesc& in) { return GMatDesc{CV_8U, 3, in.size}; }
 };
 
+G_API_OP(GY, <GMat(GFrame)>, "org.opencv.streaming.Y") {
+    static GMatDesc outMeta(const GFrameDesc& frameDesc) {
+        return GMatDesc { CV_8U, 1, frameDesc.size , false };
+    }
+};
+
+G_API_OP(GUV, <GMat(GFrame)>, "org.opencv.streaming.UV") {
+    static GMatDesc outMeta(const GFrameDesc& frameDesc) {
+        return GMatDesc { CV_8U, 2, cv::Size(frameDesc.size.width / 2, frameDesc.size.height / 2),
+                          false };
+    }
+};
+
 /** @brief Gets bgr plane from input frame
 
 @note Function textual ID is "org.opencv.streaming.BGR"
@@ -29,6 +42,25 @@ G_API_OP(GBGR, <GMat(GFrame)>, "org.opencv.streaming.BGR")
 */
 GAPI_EXPORTS cv::GMat BGR(const cv::GFrame& in);
 
+/** @brief Extracts Y plane from media frame.
+
+Output image is 8-bit 1-channel image of @ref CV_8UC1.
+
+@note Function textual ID is "org.opencv.streaming.Y"
+
+@param frame input media frame.
+*/
+GAPI_EXPORTS GMat Y(const cv::GFrame& frame);
+
+/** @brief Extracts UV plane from media frame.
+
+Output image is 8-bit 2-channel image of @ref CV_8UC2.
+
+@note Function textual ID is "org.opencv.streaming.UV"
+
+@param frame input media frame.
+*/
+GAPI_EXPORTS GMat UV(const cv::GFrame& frame);
 } // namespace streaming
 
 //! @addtogroup gapi_transform
diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp
@@ -78,3 +78,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) {
 cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) {
     return cv::gapi::streaming::GBGR::on(in);
 }
+
+cv::GMat cv::gapi::streaming::Y(const cv::GFrame& in){
+    return cv::gapi::streaming::GY::on(in);
+}
+
+cv::GMat cv::gapi::streaming::UV(const cv::GFrame& in){
+    return cv::gapi::streaming::GUV::on(in);
+}
diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp
@@ -58,32 +58,51 @@ namespace gimpl {
     struct Data;
     struct RcDesc;
 
-    struct GAPI_EXPORTS RMatMediaAdapterBGR final: public cv::RMat::Adapter
+    struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::Adapter
     {
-        explicit RMatMediaAdapterBGR(const cv::MediaFrame& frame) : m_frame(frame) { };
+        using MapDescF = std::function<cv::GMatDesc(const GFrameDesc&)>;
+        using MapDataF = std::function<cv::Mat(const GFrameDesc&, const cv::MediaFrame::View&)>;
+
+        RMatMediaFrameAdapter(const cv::MediaFrame& frame,
+                              const MapDescF& frameDescToMatDesc,
+                              const MapDataF& frameViewToMat) :
+            m_frame(frame),
+            m_frameDesc(frame.desc()),
+            m_frameDescToMatDesc(frameDescToMatDesc),
+            m_frameViewToMat(frameViewToMat)
+        { }
 
         virtual cv::RMat::View access(cv::RMat::Access a) override
         {
-            auto view = m_frame.access(a == cv::RMat::Access::W ? cv::MediaFrame::Access::W
-                                                                : cv::MediaFrame::Access::R);
-            auto ptr = reinterpret_cast<uchar*>(view.ptr[0]);
-            auto stride = view.stride[0];
-
-            std::shared_ptr<cv::MediaFrame::View> view_ptr =
-                std::make_shared<cv::MediaFrame::View>(std::move(view));
-            auto callback = [view_ptr]() mutable { view_ptr.reset(); };
-
-            return cv::RMat::View(desc(), ptr, stride, callback);
+            auto rmatToFrameAccess = [](cv::RMat::Access rmatAccess) {
+                switch(rmatAccess) {
+                    case cv::RMat::Access::R:
+                        return cv::MediaFrame::Access::R;
+                    case cv::RMat::Access::W:
+                        return cv::MediaFrame::Access::W;
+                    default:
+                        cv::util::throw_error(std::logic_error("cv::RMat::Access::R or "
+                            "cv::RMat::Access::W can only be mapped to cv::MediaFrame::Access!"));
+                }
+            };
+
+            auto fv = m_frame.access(rmatToFrameAccess(a));
+
+            auto fvHolder = std::make_shared<cv::MediaFrame::View>(std::move(fv));
+            auto callback = [fvHolder]() mutable { fvHolder.reset(); };
+
+            return asView(m_frameViewToMat(m_frame.desc(), *fvHolder), callback);
         }
 
         virtual cv::GMatDesc desc() const override
         {
-            const auto& desc = m_frame.desc();
-            GAPI_Assert(desc.fmt == cv::MediaFormat::BGR);
-            return cv::GMatDesc{CV_8U, 3, desc.size};
+            return m_frameDescToMatDesc(m_frameDesc);
         }
 
         cv::MediaFrame m_frame;
+        cv::GFrameDesc m_frameDesc;
+        MapDescF m_frameDescToMatDesc;
+        MapDataF m_frameViewToMat;
     };
 
 
diff --git a/modules/gapi/src/backends/streaming/gstreamingbackend.cpp b/modules/gapi/src/backends/streaming/gstreamingbackend.cpp
@@ -4,13 +4,16 @@
 //
 // Copyright (C) 2020 Intel Corporation
 
+#include <mutex>
+
 #if !defined(GAPI_STANDALONE)
 #include <opencv2/imgproc.hpp>
 #endif // !defined(GAPI_STANDALONE)
 
 #include <opencv2/gapi/util/throw.hpp> // throw_error
 #include <opencv2/gapi/streaming/format.hpp> // kernels
 
+#include "logger.hpp"
 #include "api/gbackend_priv.hpp"
 #include "backends/common/gbackend.hpp"
 
@@ -197,16 +200,47 @@ cv::gapi::GKernelPackage cv::gimpl::streaming::kernels()
 
 #if !defined(GAPI_STANDALONE)
 
+class GAccessorActorBase : public cv::gapi::streaming::IActor {
+public:
+    explicit GAccessorActorBase(const cv::GCompileArgs&) {}
+    virtual void run(cv::gimpl::GIslandExecutable::IInput  &in,
+                     cv::gimpl::GIslandExecutable::IOutput &out) override {
+        const auto in_msg = in.get();
+        if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
+        {
+            out.post(cv::gimpl::EndOfStream{});
+            return;
+        }
+
+        const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
+        GAPI_Assert(in_args.size() == 1u);
+        auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
+
+        cv::GRunArgP out_arg = out.get(0);
+        auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
+
+        extractRMat(frame, rmat);
+
+        out.meta(out_arg, in_args[0].meta);
+        out.post(std::move(out_arg));
+    }
+
+    virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) = 0;
+
+protected:
+    std::once_flag m_warnFlag;
+};
+
 struct GOCVBGR: public cv::detail::KernelTag
 {
     using API = cv::gapi::streaming::GBGR;
     static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
 
-    class Actor final: public cv::gapi::streaming::IActor {
-        public:
-            explicit Actor(const cv::GCompileArgs&) {}
-            virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
-                             cv::gimpl::GIslandExecutable::IOutput&out) override;
+    class Actor final: public GAccessorActorBase
+    {
+    public:
+        using GAccessorActorBase::GAccessorActorBase;
+        virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
     };
 
     static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
@@ -216,49 +250,173 @@ struct GOCVBGR: public cv::detail::KernelTag
     static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
 };
 
-void GOCVBGR::Actor::run(cv::gimpl::GIslandExecutable::IInput  &in,
-                     cv::gimpl::GIslandExecutable::IOutput &out)
+void GOCVBGR::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
 {
-    const auto in_msg = in.get();
-    if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
+    const auto& desc = frame.desc();
+    switch (desc.fmt)
     {
-        out.post(cv::gimpl::EndOfStream{});
-        return;
+        case cv::MediaFormat::BGR:
+        {
+            rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
+            [](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 3, d.size); },
+            [](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
+                return cv::Mat(d.size, CV_8UC3, v.ptr[0], v.stride[0]);
+            });
+            break;
+        }
+        case cv::MediaFormat::NV12:
+        {
+            std::call_once(m_warnFlag,
+                [](){
+                    GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from NV12 to BGR will happen.\n"
+                        "Conversion may cost a lot for images with high resolution.\n"
+                        "To retrieve cv::Mat-s from NV12 cv::MediaFrame for free, you may use "
+                        "cv::gapi::streaming::Y and cv::gapi::streaming::UV accessors.\n");
+                });
+
+            cv::Mat bgr;
+            auto view = frame.access(cv::MediaFrame::Access::R);
+            cv::Mat y_plane (desc.size,     CV_8UC1, view.ptr[0], view.stride[0]);
+            cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
+            cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
+            rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
+            break;
+        }
+        default:
+            cv::util::throw_error(
+                    std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR"));
     }
+}
 
-    const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
-    GAPI_Assert(in_args.size() == 1u);
+struct GOCVY: public cv::detail::KernelTag
+{
+    using API = cv::gapi::streaming::GY;
+    static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
+
+    class Actor final: public GAccessorActorBase
+    {
+    public:
+        using GAccessorActorBase::GAccessorActorBase;
+        virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
+    };
 
-    cv::GRunArgP out_arg = out.get(0);
-    auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
+    static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
+    {
+        return cv::gapi::streaming::IActor::Ptr(new Actor(args));
+    }
+    static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
+};
+
+void GOCVY::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
+{
     const auto& desc = frame.desc();
+    switch (desc.fmt)
+    {
+        case cv::MediaFormat::BGR:
+        {
+            std::call_once(m_warnFlag,
+                [](){
+                    GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 Y plane will "
+                        "happen.\n"
+                        "Conversion may cost a lot for images with high resolution.\n"
+                        "To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
+                        "cv::gapi::streaming::BGR accessor.\n");
+                });
+
+            auto view = frame.access(cv::MediaFrame::Access::R);
+            cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
+            cv::Mat yuv;
+            cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
+            rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(yuv.rowRange(0, desc.size.height));
+            break;
+        }
+        case cv::MediaFormat::NV12:
+        {
+            rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
+            [](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 1, d.size); },
+            [](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
+                return cv::Mat(d.size, CV_8UC1, v.ptr[0], v.stride[0]);
+            });
+            break;
+        }
+        default:
+            cv::util::throw_error(
+                    std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::Y"));
+    }
+}
 
-    auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
+struct GOCVUV: public cv::detail::KernelTag
+{
+    using API = cv::gapi::streaming::GUV;
+    static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
+
+    class Actor final: public GAccessorActorBase
+    {
+    public:
+        using GAccessorActorBase::GAccessorActorBase;
+        virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
+    };
+
+    static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
+    {
+        return cv::gapi::streaming::IActor::Ptr(new Actor(args));
+    }
+    static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
+};
+
+void GOCVUV::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
+{
+    const auto& desc = frame.desc();
     switch (desc.fmt)
     {
         case cv::MediaFormat::BGR:
-            rmat = cv::make_rmat<cv::gimpl::RMatMediaAdapterBGR>(frame);
+        {
+            std::call_once(m_warnFlag,
+                [](){
+                    GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 UV plane will "
+                        "happen.\n"
+                        "Conversion may cost a lot for images with high resolution.\n"
+                        "To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
+                        "cv::gapi::streaming::BGR accessor.\n");
+                });
+
+            auto view = frame.access(cv::MediaFrame::Access::R);
+
+            cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
+            cv::Mat yuv;
+            cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
+
+            cv::Mat uv;
+            std::vector<int> dims = { desc.size.height / 2,
+                                        desc.size.width / 2  };
+            auto start = desc.size.height;
+            auto range_h = desc.size.height / 4;
+            std::vector<cv::Mat> uv_planes = {
+                yuv.rowRange(start, start + range_h).reshape(0, dims),
+                yuv.rowRange(start + range_h, start + range_h * 2).reshape(0, dims)
+            };
+            cv::merge(uv_planes, uv);
+            rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(uv);
             break;
+        }
         case cv::MediaFormat::NV12:
-            {
-                cv::Mat bgr;
-                auto view = frame.access(cv::MediaFrame::Access::R);
-                cv::Mat y_plane (desc.size,     CV_8UC1, view.ptr[0], view.stride[0]);
-                cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
-                cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
-                rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
-                break;
-            }
+        {
+            rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
+            [](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 2, d.size / 2); },
+            [](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
+                return cv::Mat(d.size / 2, CV_8UC2, v.ptr[1], v.stride[1]);
+            });
+            break;
+        }
         default:
             cv::util::throw_error(
-                    std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR"));
+                    std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::UV"));
     }
-    out.post(std::move(out_arg));
 }
 
 cv::gapi::GKernelPackage cv::gapi::streaming::kernels()
 {
-    return cv::gapi::kernels<GOCVBGR>();
+    return cv::gapi::kernels<GOCVBGR, GOCVY, GOCVUV>();
 }
 
 #else
diff --git a/modules/gapi/test/common/gapi_tests_common.hpp b/modules/gapi/test/common/gapi_tests_common.hpp
diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp
