Updates to cameras and rasterizer to infer camera type correctly

nikhilaravi · facebook-github-bot · commit 838b73d3b6ea · 2021-02-08T14:32:39.000-08:00
Summary: Small update to the cameras and rasterizer to correctly infer the type of camera (perspective vs orthographic).

Reviewed By: jcjohnson

Differential Revision: D26267225

fbshipit-source-id: a58ed3bc2ab25553d2a4307c734204c1d41b5176
diff --git a/pytorch3d/renderer/cameras.py b/pytorch3d/renderer/cameras.py
@@ -42,15 +42,16 @@ class CamerasBase(TensorProperties):
 
     It defines methods that are common to all camera models:
         - `get_camera_center` that returns the optical center of the camera in
-    world coordinates
+            world coordinates
         - `get_world_to_view_transform` which returns a 3D transform from
-    world coordinates to the camera view coordinates (R, T)
+            world coordinates to the camera view coordinates (R, T)
         - `get_full_projection_transform` which composes the projection
-    transform (P) with the world-to-view transform (R, T)
+            transform (P) with the world-to-view transform (R, T)
         - `transform_points` which takes a set of input points in world coordinates and
-    projects to NDC coordinates ranging from [-1, -1, znear] to [+1, +1, zfar].
+            projects to NDC coordinates ranging from [-1, -1, znear] to [+1, +1, zfar].
         - `transform_points_screen` which takes a set of input points in world coordinates and
-    projects them to the screen coordinates ranging from [0, 0, znear] to [W-1, H-1, zfar]
+            projects them to the screen coordinates ranging from
+            [0, 0, znear] to [W-1, H-1, zfar]
 
     For each new camera, one should implement the `get_projection_transform`
     routine that returns the mapping from camera view coordinates to NDC coordinates.
@@ -268,6 +269,12 @@ def clone(self):
         other = cam_type(device=self.device)
         return super().clone(other)
 
+    def is_perspective(self):
+        raise NotImplementedError()
+
+    def get_znear(self):
+        return self.znear if hasattr(self, "znear") else None
+
 
 ############################################################
 #             Field of View Camera Classes                 #
@@ -534,6 +541,9 @@ def unproject_points(
         unprojection_transform = to_ndc_transform.inverse()
         return unprojection_transform.transform_points(xy_sdepth)
 
+    def is_perspective(self):
+        return True
+
 
 def OpenGLOrthographicCameras(
     znear=1.0,
@@ -752,6 +762,9 @@ def unproject_points(
         unprojection_transform = to_ndc_transform.inverse()
         return unprojection_transform.transform_points(xy_sdepth)
 
+    def is_perspective(self):
+        return False
+
 
 ############################################################
 #             MultiView Camera Classes                     #
@@ -927,6 +940,9 @@ def unproject_points(
         )
         return unprojection_transform.transform_points(xy_inv_depth)
 
+    def is_perspective(self):
+        return True
+
 
 def SfMOrthographicCameras(
     focal_length=1.0, principal_point=((0.0, 0.0),), R=_R, T=_T, device="cpu"
@@ -1086,6 +1102,9 @@ def unproject_points(
         unprojection_transform = to_ndc_transform.inverse()
         return unprojection_transform.transform_points(xy_depth)
 
+    def is_perspective(self):
+        return False
+
 
 ################################################
 #       Helper functions for cameras           #
diff --git a/pytorch3d/renderer/mesh/rasterizer.py b/pytorch3d/renderer/mesh/rasterizer.py
@@ -139,16 +139,21 @@ def forward(self, meshes_world, **kwargs) -> Fragments:
         if clip_barycentric_coords is None:
             clip_barycentric_coords = raster_settings.blur_radius > 0.0
 
-        # TODO(jcjohns): Should we try to set perspective_correct automatically
-        # based on the type of the camera?
+        # If not specified, infer perspective_correct from the camera
+        cameras = kwargs.get("cameras", self.cameras)
+        if raster_settings.perspective_correct is not None:
+            perspective_correct = raster_settings.perspective_correct
+        else:
+            perspective_correct = cameras.is_perspective()
+
         pix_to_face, zbuf, bary_coords, dists = rasterize_meshes(
             meshes_screen,
             image_size=raster_settings.image_size,
             blur_radius=raster_settings.blur_radius,
             faces_per_pixel=raster_settings.faces_per_pixel,
             bin_size=raster_settings.bin_size,
             max_faces_per_bin=raster_settings.max_faces_per_bin,
-            perspective_correct=raster_settings.perspective_correct,
+            perspective_correct=perspective_correct,
             clip_barycentric_coords=clip_barycentric_coords,
             cull_backfaces=raster_settings.cull_backfaces,
         )
diff --git a/tests/test_cameras.py b/tests/test_cameras.py
@@ -794,6 +794,11 @@ def test_transform_points(self):
         new_points = cam.transform_points(points)
         self.assertClose(new_points, projected_points)
 
+    def test_perspective_type(self):
+        cam = FoVPerspectiveCameras(znear=1.0, zfar=10.0, fov=60.0)
+        self.assertTrue(cam.is_perspective())
+        self.assertEquals(cam.get_znear(), 1.0)
+
 
 ############################################################
 #                FoVOrthographic Camera                    #
@@ -885,6 +890,11 @@ def test_orthographic_mixed_inputs_grad(self):
         )
         self.assertClose(scale_grad, grad_scale)
 
+    def test_perspective_type(self):
+        cam = FoVOrthographicCameras(znear=1.0, zfar=10.0)
+        self.assertFalse(cam.is_perspective())
+        self.assertEquals(cam.get_znear(), 1.0)
+
 
 ############################################################
 #                Orthographic Camera                       #
@@ -937,6 +947,11 @@ def test_orthographic_kwargs(self):
         v1 = P.transform_points(vertices)
         self.assertClose(v1, projected_verts)
 
+    def test_perspective_type(self):
+        cam = OrthographicCameras(focal_length=5.0, principal_point=((2.5, 2.5),))
+        self.assertFalse(cam.is_perspective())
+        self.assertEquals(cam.get_znear(), None)
+
 
 ############################################################
 #                Perspective Camera                        #
@@ -983,3 +998,8 @@ def test_perspective_kwargs(self):
         v1 = P.transform_points(vertices)
         v2 = sfm_perspective_project_naive(vertices, fx=2.0, fy=2.0, p0x=2.5, p0y=3.5)
         self.assertClose(v1, v2, atol=1e-6)
+
+    def test_perspective_type(self):
+        cam = PerspectiveCameras(focal_length=5.0, principal_point=((2.5, 2.5),))
+        self.assertTrue(cam.is_perspective())
+        self.assertEquals(cam.get_znear(), None)
diff --git a/tests/test_render_implicit.py b/tests/test_render_implicit.py
@@ -242,7 +242,10 @@ def test_compare_with_meshes_renderer(
             rasterizer=MeshRasterizer(
                 cameras=cameras_randomized,
                 raster_settings=RasterizationSettings(
-                    image_size=image_size, blur_radius=1e-3, faces_per_pixel=10
+                    image_size=image_size,
+                    blur_radius=1e-3,
+                    faces_per_pixel=10,
+                    perspective_correct=False,
                 ),
             ),
             shader=SoftPhongShader(
diff --git a/tests/test_render_meshes.py b/tests/test_render_meshes.py
@@ -500,6 +500,7 @@ def test_texture_map(self):
             blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
             faces_per_pixel=100,
             clip_barycentric_coords=True,
+            perspective_correct=False,
         )
 
         # Load reference image
@@ -844,7 +845,10 @@ def test_join_atlas(self):
         cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
 
         raster_settings = RasterizationSettings(
-            image_size=512, blur_radius=0.0, faces_per_pixel=1
+            image_size=512,
+            blur_radius=0.0,
+            faces_per_pixel=1,
+            perspective_correct=False,
         )
 
         lights = PointLights(
@@ -919,7 +923,10 @@ def test_joined_spheres(self):
         R, T = look_at_view_transform(2.7, 0.0, 0.0)
         cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
         raster_settings = RasterizationSettings(
-            image_size=512, blur_radius=0.0, faces_per_pixel=1
+            image_size=512,
+            blur_radius=0.0,
+            faces_per_pixel=1,
+            perspective_correct=False,
         )
 
         # Init shader settings
