Merge pull request #26656 from warped-rudi:mediandk

warped-rudi · web-flow · commit d39aae6bdfc1 · 2024-12-30T11:09:11.000+03:00
AndroidMediaNdkCapture pixel format enhancement #26656 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
diff --git a/modules/java/generator/android-24/java/org/opencv/android/NativeCameraView.java b/modules/java/generator/android-24/java/org/opencv/android/NativeCameraView.java
@@ -2,6 +2,7 @@
 
 import org.opencv.core.Mat;
 import org.opencv.core.Size;
+import org.opencv.core.MatOfInt;
 
 import org.opencv.imgproc.Imgproc;
 
@@ -123,8 +124,11 @@ private boolean initializeCamera(int width, int height) {
                 return false;
             }
 
+            MatOfInt params = new MatOfInt(Videoio.CAP_PROP_FRAME_WIDTH, width,
+                                           Videoio.CAP_PROP_FRAME_HEIGHT, height);
+
             Log.d(TAG, "Try to open camera with index " + localCameraIndex);
-            mCamera = new VideoCapture(localCameraIndex, Videoio.CAP_ANDROID);
+            mCamera = new VideoCapture(localCameraIndex, Videoio.CAP_ANDROID, params);
 
             if (mCamera == null)
                 return false;
@@ -139,9 +143,6 @@ private boolean initializeCamera(int width, int height) {
 
             mFrame = new RotatedCameraFrame(new NativeCameraFrame(mCamera), frameRotation);
 
-            mCamera.set(Videoio.CAP_PROP_FRAME_WIDTH, width);
-            mCamera.set(Videoio.CAP_PROP_FRAME_HEIGHT, height);
-
             if (frameRotation % 180 == 0) {
                 mFrameWidth = (int) mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
                 mFrameHeight = (int) mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
@@ -181,39 +182,35 @@ private static class NativeCameraFrame implements CvCameraViewFrame {
 
         @Override
         public Mat rgba() {
-            mCapture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('R','G','B','3'));
-            mCapture.retrieve(mBgr);
-            Log.d(TAG, "Retrived frame with size " + mBgr.cols() + "x" + mBgr.rows() + " and channels: " + mBgr.channels());
-            Imgproc.cvtColor(mBgr, mRgba, Imgproc.COLOR_RGB2RGBA);
+            mCapture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('R','G','B','4'));
+            mCapture.retrieve(mRgba);
+            Log.d(TAG, "Retrieved frame with size " + mRgba.cols() + "x" + mRgba.rows() + " and channels: " + mRgba.channels());
             return mRgba;
         }
 
         @Override
         public Mat gray() {
             mCapture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('G','R','E','Y'));
             mCapture.retrieve(mGray);
-            Log.d(TAG, "Retrived frame with size " + mGray.cols() + "x" + mGray.rows() + " and channels: " + mGray.channels());
+            Log.d(TAG, "Retrieved frame with size " + mGray.cols() + "x" + mGray.rows() + " and channels: " + mGray.channels());
             return mGray;
         }
 
         public NativeCameraFrame(VideoCapture capture) {
             mCapture = capture;
             mGray = new Mat();
             mRgba = new Mat();
-            mBgr = new Mat();
         }
 
         @Override
         public void release() {
             if (mGray != null) mGray.release();
             if (mRgba != null) mRgba.release();
-            if (mBgr != null) mBgr.release();
         }
 
         private VideoCapture mCapture;
         private Mat mRgba;
         private Mat mGray;
-        private Mat mBgr;
     };
 
     private class CameraWorker implements Runnable {
diff --git a/modules/videoio/src/cap_android_mediandk.cpp b/modules/videoio/src/cap_android_mediandk.cpp
@@ -21,10 +21,20 @@
 
 #define INPUT_TIMEOUT_MS 2000
 
+#define COLOR_FormatUnknown -1
 #define COLOR_FormatYUV420Planar 19
 #define COLOR_FormatYUV420SemiPlanar 21
 #define COLOR_FormatSurface 0x7f000789 //See https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities for codes
 
+#define FOURCC_BGR CV_FOURCC_MACRO('B','G','R','3')
+#define FOURCC_RGB CV_FOURCC_MACRO('R','G','B','3')
+#define FOURCC_BGRA CV_FOURCC_MACRO('B','G','R','4')
+#define FOURCC_RGBA CV_FOURCC_MACRO('R','G','B','4')
+#define FOURCC_GRAY CV_FOURCC_MACRO('G','R','E','Y')
+#define FOURCC_NV12 CV_FOURCC_MACRO('N','V','1','2')
+#define FOURCC_YV12 CV_FOURCC_MACRO('Y','V','1','2')
+#define FOURCC_UNKNOWN  0xFFFFFFFF
+
 using namespace cv;
 
 #define TAG "NativeCodec"
@@ -51,9 +61,9 @@ class AndroidMediaNdkCapture : public IVideoCapture
 public:
     AndroidMediaNdkCapture():
         sawInputEOS(false), sawOutputEOS(false),
-        frameStride(0), frameWidth(0), frameHeight(0), colorFormat(0),
-        videoWidth(0), videoHeight(0),
-        videoFrameCount(0),
+        frameStride(0), frameWidth(0), frameHeight(0),
+        colorFormat(COLOR_FormatUnknown), fourCC(FOURCC_BGR),
+        videoWidth(0), videoHeight(0), videoFrameCount(0),
         videoRotation(0), videoRotationCode(-1),
         videoOrientationAuto(false) {}
 
@@ -65,6 +75,7 @@ class AndroidMediaNdkCapture : public IVideoCapture
     int32_t frameWidth;
     int32_t frameHeight;
     int32_t colorFormat;
+    uint32_t fourCC;
     int32_t videoWidth;
     int32_t videoHeight;
     float videoFrameRate;
@@ -73,7 +84,6 @@ class AndroidMediaNdkCapture : public IVideoCapture
     int32_t videoRotationCode;
     bool videoOrientationAuto;
     std::vector<uint8_t> buffer;
-    Mat frame;
 
     ~AndroidMediaNdkCapture() { cleanUp(); }
 
@@ -157,23 +167,51 @@ class AndroidMediaNdkCapture : public IVideoCapture
             return false;
         }
 
-        Mat yuv(frameHeight + frameHeight/2, frameStride, CV_8UC1, buffer.data());
+        ColorConversionCodes ccCode;
+        const Mat yuv(frameHeight + frameHeight/2,
+                      frameWidth, CV_8UC1, buffer.data(), frameStride);
 
         if (colorFormat == COLOR_FormatYUV420Planar) {
-            cv::cvtColor(yuv, frame, cv::COLOR_YUV2BGR_YV12);
+            switch(fourCC)
+            {
+                case     FOURCC_BGR: ccCode = COLOR_YUV2BGR_YV12;   break;
+                case     FOURCC_RGB: ccCode = COLOR_YUV2RGB_YV12;   break;
+                case    FOURCC_BGRA: ccCode = COLOR_YUV2BGRA_YV12;  break;
+                case    FOURCC_RGBA: ccCode = COLOR_YUV2RGBA_YV12;  break;
+                case    FOURCC_GRAY: ccCode = COLOR_YUV2GRAY_YV12;  break;
+                case    FOURCC_YV12: break;
+                case FOURCC_UNKNOWN: fourCC = FOURCC_YV12;          break;
+                            default: LOGE("Unexpected FOURCC value: %d", fourCC);
+                                     return false;
+            }
         } else if (colorFormat == COLOR_FormatYUV420SemiPlanar) {
-            cv::cvtColor(yuv, frame, cv::COLOR_YUV2BGR_NV21);
+            // Attention: COLOR_FormatYUV420SemiPlanar seems to correspond to NV12.
+            //            This is different from the Camera2 interface, where NV21
+            //            is used in this situation.
+            switch(fourCC)
+            {
+                case     FOURCC_BGR: ccCode = COLOR_YUV2BGR_NV12;   break;
+                case     FOURCC_RGB: ccCode = COLOR_YUV2RGB_NV12;   break;
+                case    FOURCC_BGRA: ccCode = COLOR_YUV2BGRA_NV12;  break;
+                case    FOURCC_RGBA: ccCode = COLOR_YUV2RGBA_NV12;  break;
+                case    FOURCC_GRAY: ccCode = COLOR_YUV2GRAY_NV12;  break;
+                case    FOURCC_NV12: break;
+                case FOURCC_UNKNOWN: fourCC = FOURCC_NV12;          break;
+                            default: LOGE("Unexpected FOURCC value: %d", fourCC);
+                                     return false;
+            }
         } else {
             LOGE("Unsupported video format: %d", colorFormat);
             return false;
         }
 
-        Mat croppedFrame = frame(Rect(0, 0, videoWidth, videoHeight));
-        out.assign(croppedFrame);
+        if (fourCC == FOURCC_YV12 || fourCC == FOURCC_NV12)
+            yuv.copyTo(out);
+        else
+            cvtColor(yuv, out, ccCode);
 
-        if (videoOrientationAuto && -1 != videoRotationCode) {
-            cv::rotate(out, out, videoRotationCode);
-        }
+        if (videoOrientationAuto && -1 != videoRotationCode)
+            rotate(out, out, videoRotationCode);
 
         return true;
     }
@@ -194,8 +232,11 @@ class AndroidMediaNdkCapture : public IVideoCapture
             case CAP_PROP_FRAME_COUNT: return videoFrameCount;
             case CAP_PROP_ORIENTATION_META: return videoRotation;
             case CAP_PROP_ORIENTATION_AUTO: return videoOrientationAuto ? 1 : 0;
+            case CAP_PROP_FOURCC: return fourCC;
         }
-        return 0;
+
+        // unknown parameter or value not available
+        return -1;
     }
 
     bool setProperty(int property_id, double value) CV_OVERRIDE
@@ -206,6 +247,31 @@ class AndroidMediaNdkCapture : public IVideoCapture
                 videoOrientationAuto = value != 0 ? true : false;
                 return true;
             }
+            case CAP_PROP_FOURCC: {
+                uint32_t newFourCC = cvRound(value);
+                switch (newFourCC)
+                {
+                    case FOURCC_BGR:
+                    case FOURCC_RGB:
+                    case FOURCC_BGRA:
+                    case FOURCC_RGBA:
+                    case FOURCC_GRAY:
+                        fourCC = newFourCC;
+                        return true;
+                    case FOURCC_YV12:
+                        if (colorFormat != COLOR_FormatYUV420SemiPlanar) {
+                            fourCC = (colorFormat == COLOR_FormatUnknown) ? FOURCC_UNKNOWN : FOURCC_YV12;
+                            return true;
+                        }
+                        break;
+                    case FOURCC_NV12:
+                        if (colorFormat != COLOR_FormatYUV420Planar) {
+                            fourCC = (colorFormat == COLOR_FormatUnknown) ? FOURCC_UNKNOWN : FOURCC_NV12;
+                            return true;
+                        }
+                        break;
+                }
+            }
         }
 
         return false;
diff --git a/samples/android/video-recorder/src/org/opencv/samples/recorder/RecorderActivity.java b/samples/android/video-recorder/src/org/opencv/samples/recorder/RecorderActivity.java
@@ -57,7 +57,6 @@ public class RecorderActivity extends CameraActivity implements CvCameraViewList
     private VideoWriter mVideoWriter = null;
     private VideoCapture mVideoCapture = null;
     private Mat mVideoFrame;
-    private Mat mRenderFrame;
 
     public RecorderActivity() {
         Log.i(TAG, "Instantiated new " + this.getClass());
@@ -122,7 +121,6 @@ public void onPause()
         mTriggerButton.setText("Start Camera");
 
         mVideoFrame.release();
-        mRenderFrame.release();
     }
 
     @Override
@@ -132,7 +130,6 @@ public void onResume()
         super.onResume();
 
         mVideoFrame = new Mat();
-        mRenderFrame = new Mat();
 
         changeStatus();
     }
@@ -294,12 +291,16 @@ public boolean startPlayback() {
             mVideoCapture = new VideoCapture(mVideoFilename, Videoio.CAP_OPENCV_MJPEG);
         }
 
-        if (!mVideoCapture.isOpened()) {
+        if (mVideoCapture == null || !mVideoCapture.isOpened()) {
             Log.e(TAG, "Can't open video");
             Toast.makeText(this, "Can't open file " + mVideoFilename, Toast.LENGTH_SHORT).show();
             return false;
         }
 
+        if (!mUseBuiltInMJPG){
+            mVideoCapture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('R','G','B','4'));
+        }
+
         Toast.makeText(this, "Starting playback from file " + mVideoFilename, Toast.LENGTH_SHORT).show();
 
         mPlayerThread = new Runnable() {
@@ -315,11 +316,14 @@ public void run() {
                     }
                     return;
                 }
-                // VideoCapture with CAP_ANDROID generates RGB frames instead of BGR
-                // https://github.com/opencv/opencv/issues/24687
-                Imgproc.cvtColor(mVideoFrame, mRenderFrame, mUseBuiltInMJPG ? Imgproc.COLOR_BGR2RGBA: Imgproc.COLOR_RGB2RGBA);
-                Bitmap bmp = Bitmap.createBitmap(mRenderFrame.cols(), mRenderFrame.rows(), Bitmap.Config.ARGB_8888);
-                Utils.matToBitmap(mRenderFrame, bmp);
+
+                // MJPEG codec will output BGR only. So we need to convert to RGBA.
+                if (mUseBuiltInMJPG) {
+                    Imgproc.cvtColor(mVideoFrame, mVideoFrame, Imgproc.COLOR_BGR2RGBA);
+                }
+
+                Bitmap bmp = Bitmap.createBitmap(mVideoFrame.cols(), mVideoFrame.rows(), Bitmap.Config.ARGB_8888);
+                Utils.matToBitmap(mVideoFrame, bmp);
                 mImageView.setImageBitmap(bmp);
                 Handler h = new Handler();
                 h.postDelayed(this, 33);
