Merge pull request #24869 from alexlyulkov:al/android-camera-view-rotate

alexlyulkov · web-flow · commit bfad61f43344 · 2024-01-17T21:35:35.000+03:00
Added screen rotation support to JavaCamera2View amd NativeCameraView. Fixed JavaCamera2View initialization. #24869 Added automatic image rotation to JavaCamera2View and NativeCameraView so the video preview was matched with screen orientation. Fixed double preview initialization bug in JavaCamera2View. Added proper cameraID parsing to NativeCameraView similar to JavaCameraView ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
diff --git a/modules/java/generator/android-21/java/org/opencv/android/JavaCamera2View.java b/modules/java/generator/android-21/java/org/opencv/android/JavaCamera2View.java
@@ -46,6 +46,7 @@ public class JavaCamera2View extends CameraBridgeViewBase {
     protected ImageReader mImageReader;
     protected int mPreviewFormat = ImageFormat.YUV_420_888;
     protected int mRequestTemplate = CameraDevice.TEMPLATE_PREVIEW;
+    private int mFrameRotation;
 
     protected CameraDevice mCameraDevice;
     protected CameraCaptureSession mCaptureSession;
@@ -86,8 +87,8 @@ private void stopBackgroundThread() {
         }
     }
 
-    protected boolean initializeCamera() {
-        Log.i(LOGTAG, "initializeCamera");
+    protected boolean selectCamera() {
+        Log.i(LOGTAG, "selectCamera");
         CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
         try {
             String camList[] = manager.getCameraIdList();
@@ -110,26 +111,22 @@ protected boolean initializeCamera() {
                     }
                 }
             }
-            if (mCameraID != null) {
-                Log.i(LOGTAG, "Opening camera: " + mCameraID);
-                manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
-            } else { // make JavaCamera2View behaves in the same way as JavaCameraView
-                Log.i(LOGTAG, "Trying to open camera with the value (" + mCameraIndex + ")");
+            if (mCameraID == null) { // make JavaCamera2View behaves in the same way as JavaCameraView
+                Log.i(LOGTAG, "Selecting camera by index (" + mCameraIndex + ")");
                 if (mCameraIndex < camList.length) {
                     mCameraID = camList[mCameraIndex];
-                    manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
                 } else {
                     // CAMERA_DISCONNECTED is used when the camera id is no longer valid
                     throw new CameraAccessException(CameraAccessException.CAMERA_DISCONNECTED);
                 }
             }
             return true;
         } catch (CameraAccessException e) {
-            Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
+            Log.e(LOGTAG, "selectCamera - Camera Access Exception", e);
         } catch (IllegalArgumentException e) {
-            Log.e(LOGTAG, "OpenCamera - Illegal Argument Exception", e);
+            Log.e(LOGTAG, "selectCamera - Illegal Argument Exception", e);
         } catch (SecurityException e) {
-            Log.e(LOGTAG, "OpenCamera - Security Exception", e);
+            Log.e(LOGTAG, "selectCamera - Security Exception", e);
         }
         return false;
     }
@@ -204,6 +201,7 @@ private void createCameraPreviewSession() {
             mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
                 @Override
                 public void onImageAvailable(ImageReader reader) {
+
                     Image image = reader.acquireLatestImage();
                     if (image == null)
                         return;
@@ -213,8 +211,9 @@ public void onImageAvailable(ImageReader reader) {
                     assert (planes.length == 3);
                     assert (image.getFormat() == mPreviewFormat);
 
-                    JavaCamera2Frame tempFrame = new JavaCamera2Frame(image);
+                    RotatedCameraFrame tempFrame = new RotatedCameraFrame(new JavaCamera2Frame(image), mFrameRotation);
                     deliverAndDrawFrame(tempFrame);
+                    tempFrame.mFrame.release();
                     tempFrame.release();
                     image.close();
                 }
@@ -303,11 +302,22 @@ boolean calcPreviewSize(final int width, final int height) {
     protected boolean connectCamera(int width, int height) {
         Log.i(LOGTAG, "setCameraPreviewSize(" + width + "x" + height + ")");
         startBackgroundThread();
-        initializeCamera();
+        selectCamera();
         try {
+            CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
+            CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraID);
+            mFrameRotation = getFrameRotation(
+                    characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT,
+                    characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION));
+
             boolean needReconfig = calcPreviewSize(width, height);
-            mFrameWidth = mPreviewSize.getWidth();
-            mFrameHeight = mPreviewSize.getHeight();
+            if (mFrameRotation % 180 == 0) {
+                mFrameWidth = mPreviewSize.getWidth();
+                mFrameHeight = mPreviewSize.getHeight();
+            } else {
+                mFrameWidth = mPreviewSize.getHeight();
+                mFrameHeight = mPreviewSize.getWidth();
+            }
 
             if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                 mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
@@ -322,12 +332,16 @@ protected boolean connectCamera(int width, int height) {
                     mCaptureSession.close();
                     mCaptureSession = null;
                 }
-                createCameraPreviewSession();
             }
 
             if (mFpsMeter != null) {
                 mFpsMeter.setResolution(mFrameWidth, mFrameHeight);
             }
+
+            Log.i(LOGTAG, "Opening camera: " + mCameraID);
+            manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
+        } catch (CameraAccessException e) {
+            Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
         } catch (RuntimeException e) {
             throw new RuntimeException("Interrupted while setCameraPreviewSize.", e);
         }
@@ -442,6 +456,7 @@ public JavaCamera2Frame(Image image) {
             mGray = new Mat();
         }
 
+        @Override
         public void release() {
             mRgba.release();
             mGray.release();
diff --git a/modules/java/generator/android-24/java/org/opencv/android/NativeCameraView.java b/modules/java/generator/android-24/java/org/opencv/android/NativeCameraView.java
@@ -10,6 +10,7 @@
 import org.opencv.videoio.VideoWriter;
 
 import android.content.Context;
+import android.hardware.Camera;
 import android.util.AttributeSet;
 import android.util.Log;
 import android.view.ViewGroup.LayoutParams;
@@ -25,7 +26,7 @@ public class NativeCameraView extends CameraBridgeViewBase {
     private Thread mThread;
 
     protected VideoCapture mCamera;
-    protected NativeCameraFrame mFrame;
+    protected RotatedCameraFrame mFrame;
 
     public NativeCameraView(Context context, int cameraId) {
         super(context, cameraId);
@@ -89,28 +90,65 @@ public int getHeight(Object obj) {
 
     private boolean initializeCamera(int width, int height) {
         synchronized (this) {
-
-            if (mCameraIndex == -1) {
+            Camera.CameraInfo cameraInfo = new Camera.CameraInfo();
+            int localCameraIndex = mCameraIndex;
+            if (mCameraIndex == CAMERA_ID_ANY) {
                 Log.d(TAG, "Try to open default camera");
-                mCamera = new VideoCapture(0, Videoio.CAP_ANDROID);
-            } else {
-                Log.d(TAG, "Try to open camera with index " + mCameraIndex);
-                mCamera = new VideoCapture(mCameraIndex, Videoio.CAP_ANDROID);
+                localCameraIndex = 0;
+            } else if (mCameraIndex == CAMERA_ID_BACK) {
+                Log.i(TAG, "Trying to open back camera");
+                for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
+                    Camera.getCameraInfo( camIdx, cameraInfo );
+                    if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
+                        localCameraIndex = camIdx;
+                        break;
+                    }
+                }
+            } else if (mCameraIndex == CAMERA_ID_FRONT) {
+                Log.i(TAG, "Trying to open front camera");
+                for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
+                    Camera.getCameraInfo( camIdx, cameraInfo );
+                    if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) {
+                        localCameraIndex = camIdx;
+                        break;
+                    }
+                }
             }
 
-            if (mCamera == null)
+            if (localCameraIndex == CAMERA_ID_BACK) {
+                Log.e(TAG, "Back camera not found!");
+                return false;
+            } else if (localCameraIndex == CAMERA_ID_FRONT) {
+                Log.e(TAG, "Front camera not found!");
                 return false;
+            }
+
+            Log.d(TAG, "Try to open camera with index " + localCameraIndex);
+            mCamera = new VideoCapture(localCameraIndex, Videoio.CAP_ANDROID);
 
+            if (mCamera == null)
+                return false;
             if (mCamera.isOpened() == false)
                 return false;
 
-            mFrame = new NativeCameraFrame(mCamera);
+            if (mCameraIndex != CAMERA_ID_BACK && mCameraIndex != CAMERA_ID_FRONT)
+                Camera.getCameraInfo(localCameraIndex, cameraInfo);
+            int frameRotation = getFrameRotation(
+                    cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT,
+                    cameraInfo.orientation);
+
+            mFrame = new RotatedCameraFrame(new NativeCameraFrame(mCamera), frameRotation);
 
             mCamera.set(Videoio.CAP_PROP_FRAME_WIDTH, width);
             mCamera.set(Videoio.CAP_PROP_FRAME_HEIGHT, height);
 
-            mFrameWidth = (int)mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
-            mFrameHeight = (int)mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+            if (frameRotation % 180 == 0) {
+                mFrameWidth = (int) mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
+                mFrameHeight = (int) mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+            } else {
+                mFrameWidth = (int) mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+                mFrameHeight = (int) mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
+            }
 
             if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                 mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
@@ -131,7 +169,10 @@ private boolean initializeCamera(int width, int height) {
 
     private void releaseCamera() {
         synchronized (this) {
-            if (mFrame != null) mFrame.release();
+            if (mFrame != null) {
+                mFrame.mFrame.release();
+                mFrame.release();
+            }
             if (mCamera != null) mCamera.release();
         }
     }
@@ -162,6 +203,7 @@ public NativeCameraFrame(VideoCapture capture) {
             mBgr = new Mat();
         }
 
+        @Override
         public void release() {
             if (mGray != null) mGray.release();
             if (mRgba != null) mRgba.release();
diff --git a/modules/java/generator/android/java/org/opencv/android/CameraBridgeViewBase.java b/modules/java/generator/android/java/org/opencv/android/CameraBridgeViewBase.java
@@ -4,6 +4,7 @@
 
 import org.opencv.BuildConfig;
 import org.opencv.R;
+import org.opencv.core.Core;
 import org.opencv.core.Mat;
 import org.opencv.core.Size;
 
@@ -17,8 +18,10 @@
 import android.graphics.Rect;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
+import android.view.WindowManager;
 
 /**
  * This is a basic class, implementing the interaction with Camera and OpenCV library.
@@ -189,8 +192,93 @@ public interface CvCameraViewFrame {
          * This method returns single channel gray scale Mat with frame
          */
         public Mat gray();
+
+        public void release();
+    };
+
+    public class RotatedCameraFrame implements CvCameraViewFrame {
+        @Override
+        public Mat gray() {
+            if (mRotation != 0) {
+                Core.rotate(mFrame.gray(), mGrayRotated, getCvRotationCode(mRotation));
+                return mGrayRotated;
+            } else {
+                return mFrame.gray();
+            }
+        }
+
+        @Override
+        public Mat rgba() {
+            if (mRotation != 0) {
+                Core.rotate(mFrame.rgba(), mRgbaRotated, getCvRotationCode(mRotation));
+                return mRgbaRotated;
+            } else {
+                return mFrame.rgba();
+            }
+        }
+
+        private int getCvRotationCode(int degrees) {
+            if  (degrees == 90) {
+                return Core.ROTATE_90_CLOCKWISE;
+            } else if (degrees == 180) {
+                return Core.ROTATE_180;
+            } else {
+                return Core.ROTATE_90_COUNTERCLOCKWISE;
+            }
+        }
+
+        public RotatedCameraFrame(CvCameraViewFrame frame, int rotation) {
+            super();
+            mFrame = frame;
+            mRgbaRotated = new Mat();
+            mGrayRotated = new Mat();
+            mRotation = rotation;
+        }
+
+        @Override
+        public void release() {
+            mRgbaRotated.release();
+            mGrayRotated.release();
+        }
+
+        public CvCameraViewFrame mFrame;
+        private Mat mRgbaRotated;
+        private Mat mGrayRotated;
+        private int mRotation;
     };
 
+    /**
+     * Calculates how to rotate camera frame to match current screen orientation
+     */
+    protected int getFrameRotation(boolean cameraFacingFront, int cameraSensorOrientation) {
+        WindowManager windowManager = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
+        int screenOrientation = windowManager.getDefaultDisplay().getRotation();
+        int screenRotation = 0;
+        switch (screenOrientation) {
+            case Surface.ROTATION_0:
+                screenRotation = 0;
+                break;
+            case Surface.ROTATION_90:
+                screenRotation = 90;
+                break;
+            case Surface.ROTATION_180:
+                screenRotation = 180;
+                break;
+            case Surface.ROTATION_270:
+                screenRotation = 270;
+                break;
+        }
+
+        int frameRotation;
+        if (cameraFacingFront) {
+            frameRotation = (cameraSensorOrientation + screenRotation) % 360;
+        } else {
+            frameRotation = (cameraSensorOrientation - screenRotation + 360) % 360;
+        }
+
+        return frameRotation;
+    }
+
     public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) {
         Log.d(TAG, "call surfaceChanged event");
         synchronized(mSyncObject) {
diff --git a/modules/java/generator/android/java/org/opencv/android/JavaCameraView.java b/modules/java/generator/android/java/org/opencv/android/JavaCameraView.java
