From 2727741b98004e8233804b2353cde6564f7e4f25 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:43:42 -0800 Subject: [PATCH 1/8] Create Camera class with basic attributes --- sleap_io/model/camera.py | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 sleap_io/model/camera.py diff --git a/sleap_io/model/camera.py b/sleap_io/model/camera.py new file mode 100644 index 00000000..2edb1d74 --- /dev/null +++ b/sleap_io/model/camera.py @@ -0,0 +1,77 @@ +"""Data structure for a single camera view in a multi-camera setup.""" + +from __future__ import annotations + +import attrs +import numpy as np +from attrs import define, field + + +@define +class Camera: + """A camera used to record in a multi-view `RecordingSession`. + + Attributes: + matrix: Intrinsic camera matrix of size 3 x 3. + dist: Radial-tangential distortion coefficients [k_1, k_2, p_1, p_2, k_3]. + size: Image size. + rvec: Rotation vector in unnormalized axis-angle representation of size 3. + tvec: Translation vector of size 3. + name: Camera name. + """ + + matrix: np.ndarray = field( + default=np.eye(3), + converter=np.array, + ) + dist: np.ndarray = field( + default=np.zeros(5), converter=lambda x: np.array(x).ravel() + ) + size: tuple[int, int] = field( + default=None, converter=attrs.converters.optional(tuple) + ) + rvec: np.ndarray = field( + default=np.zeros(3), converter=lambda x: np.array(x).ravel() + ) + tvec: np.ndarray = field( + default=np.zeros(3), converter=lambda x: np.array(x).ravel() + ) + name: str = field(default=None, converter=attrs.converters.optional(str)) + + @matrix.validator + @dist.validator + @size.validator + @rvec.validator + @tvec.validator + def _validate_shape(self, attribute: attrs.Attribute, value): + """Validate shape of attribute based on metadata. + + Args: + attribute: Attribute to validate. + value: Value of attribute to validate. + + Raises: + ValueError: If attribute shape is not as expected. + """ + + # Define metadata for each attribute + attr_metadata = { + "matrix": {"shape": (3, 3), "type": np.ndarray}, + "dist": {"shape": (5,), "type": np.ndarray}, + "size": {"shape": (2,), "type": tuple}, + "rvec": {"shape": (3,), "type": np.ndarray}, + "tvec": {"shape": (3,), "type": np.ndarray}, + } + optional_attrs = ["size"] + + # Skip validation if optional attribute is None + if attribute.name in optional_attrs and value is None: + return + + # Validate shape of attribute + expected_shape = attr_metadata[attribute.name]["shape"] + expected_type = attr_metadata[attribute.name]["type"] + if np.shape(value) != expected_shape: + raise ValueError( + f"{attribute.name} must be a {expected_type} of size {expected_shape}" + ) From 30aecee55f791d5e2947cb4fa1bea7624df5cc61 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:44:30 -0800 Subject: [PATCH 2/8] Test Camera's core attributes --- tests/model/test_camera.py | 122 +++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/model/test_camera.py diff --git a/tests/model/test_camera.py b/tests/model/test_camera.py new file mode 100644 index 00000000..34183866 --- /dev/null +++ b/tests/model/test_camera.py @@ -0,0 +1,122 @@ +"""Tests for methods in the sleap_io.model.instance file.""" + +import numpy as np +import pytest + +from sleap_io.model.camera import Camera + + +def test_camera_name(): + """Test camera name converter always converts to string.""" + + # During initialization + camera = Camera(name=12) + assert camera.name == "12" + + # After initialization + camera = Camera() + assert camera.name is None + camera.name = 12 + assert camera.name == "12" + camera.name = "12" + assert camera.name == "12" + + +def test_camera_matrix(): + """Test camera matrix converter and validator.""" + + matrix_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + matrix_array = np.array(matrix_list) + + # During initialization + camera = Camera(matrix=matrix_list) + np.testing.assert_array_equal(camera.matrix, matrix_array) + with pytest.raises(ValueError): + camera = Camera(matrix=[[1, 2], [3, 4]]) + + # Test matrix converter + camera = Camera() + camera.matrix = matrix_list + np.testing.assert_array_equal(camera.matrix, matrix_array) + matrix_array = np.array(matrix_list) + camera.matrix = matrix_array + np.testing.assert_array_equal(camera.matrix, matrix_array) + with pytest.raises(ValueError): + camera.matrix = [[1, 2], [3, 4]] + + +def test_camera_distortions(): + """Test camera distortion converter and validator.""" + + distortions_unraveled = [[1], [2], [3], [4], [5]] + distortions_raveled = np.array(distortions_unraveled).ravel() + + # During initialization + camera = Camera(dist=distortions_unraveled) + np.testing.assert_array_equal(camera.dist, distortions_raveled) + with pytest.raises(ValueError): + camera = Camera(dist=distortions_raveled[:3]) + + # Test distortion converter + camera = Camera() + camera.dist = distortions_unraveled + np.testing.assert_array_equal(camera.dist, distortions_raveled) + with pytest.raises(ValueError): + camera.dist = distortions_raveled[:3] + + +def test_camera_size(): + """Test camera size converter and validator.""" + + size = (100, 200) + + # During initialization + camera = Camera(size=size) + assert camera.size == size + with pytest.raises(ValueError): + camera = Camera(size=(100, 200, 300)) + + # Test size converter + camera = Camera() + camera.size = size + assert camera.size == size + with pytest.raises(ValueError): + camera.size = (100, 200, 300) + + +def test_camera_rvec(): + """Test camera rotation vector converter and validator.""" + + rvec = [1, 2, 3] + + # During initialization + camera = Camera(rvec=rvec) + np.testing.assert_array_equal(camera.rvec, rvec) + with pytest.raises(ValueError): + camera = Camera(rvec=[1, 2]) + + # Test rvec validator + camera = Camera() + camera.rvec = rvec + np.testing.assert_array_equal(camera.rvec, rvec) + with pytest.raises(ValueError): + camera.rvec = [1, 2] + + +def test_camera_tvec(): + """Test camera translation vector converter and validator.""" + + tvec = [1, 2, 3] + + # During initialization + camera = Camera(tvec=tvec) + np.testing.assert_array_equal(camera.tvec, tvec) + with pytest.raises(ValueError): + camera = Camera(tvec=[1, 2]) + + # Test tvec validator + camera = Camera() + camera.tvec = tvec + np.testing.assert_array_equal(camera.tvec, tvec) + with pytest.raises(ValueError): + camera.tvec = [1, 2] From c582bfd540283bc5474020d0157e1bceef19345b Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:52:35 -0800 Subject: [PATCH 3/8] Add extrinsic_matrix, undistort, and project --- sleap_io/model/camera.py | 185 +++++++++++++++++++++++++++++++++++---- 1 file changed, 169 insertions(+), 16 deletions(-) diff --git a/sleap_io/model/camera.py b/sleap_io/model/camera.py index 2edb1d74..8c902238 100644 --- a/sleap_io/model/camera.py +++ b/sleap_io/model/camera.py @@ -3,6 +3,7 @@ from __future__ import annotations import attrs +import cv2 import numpy as np from attrs import define, field @@ -12,37 +13,42 @@ class Camera: """A camera used to record in a multi-view `RecordingSession`. Attributes: - matrix: Intrinsic camera matrix of size 3 x 3. - dist: Radial-tangential distortion coefficients [k_1, k_2, p_1, p_2, k_3]. - size: Image size. - rvec: Rotation vector in unnormalized axis-angle representation of size 3. - tvec: Translation vector of size 3. + matrix: Intrinsic camera matrix of size (3, 3) and type float64. + dist: Radial-tangential distortion coefficients [k_1, k_2, p_1, p_2, k_3] of + size (5,) and type float64. + size: Image size of camera in pixels of size (2,) and type int. + rvec: Rotation vector in unnormalized axis-angle representation of size (3,) and + type float64. + tvec: Translation vector of size (3,) and type float64. + extrinsic_matrix: Extrinsic matrix of camera of size (4, 4) and type float64. name: Camera name. """ matrix: np.ndarray = field( default=np.eye(3), - converter=np.array, + converter=lambda x: np.array(x, dtype="float64"), ) dist: np.ndarray = field( - default=np.zeros(5), converter=lambda x: np.array(x).ravel() + default=np.zeros(5), converter=lambda x: np.array(x, dtype="float64").ravel() ) size: tuple[int, int] = field( default=None, converter=attrs.converters.optional(tuple) ) - rvec: np.ndarray = field( - default=np.zeros(3), converter=lambda x: np.array(x).ravel() + _rvec: np.ndarray = field( + default=np.zeros(3), converter=lambda x: np.array(x, dtype="float64").ravel() ) - tvec: np.ndarray = field( - default=np.zeros(3), converter=lambda x: np.array(x).ravel() + _tvec: np.ndarray = field( + default=np.zeros(3), converter=lambda x: np.array(x, dtype="float64").ravel() ) name: str = field(default=None, converter=attrs.converters.optional(str)) + _extrinsic_matrix: np.ndarray = field(init=False) @matrix.validator @dist.validator @size.validator - @rvec.validator - @tvec.validator + @_rvec.validator + @_tvec.validator + @_extrinsic_matrix.validator def _validate_shape(self, attribute: attrs.Attribute, value): """Validate shape of attribute based on metadata. @@ -59,8 +65,9 @@ def _validate_shape(self, attribute: attrs.Attribute, value): "matrix": {"shape": (3, 3), "type": np.ndarray}, "dist": {"shape": (5,), "type": np.ndarray}, "size": {"shape": (2,), "type": tuple}, - "rvec": {"shape": (3,), "type": np.ndarray}, - "tvec": {"shape": (3,), "type": np.ndarray}, + "_rvec": {"shape": (3,), "type": np.ndarray}, + "_tvec": {"shape": (3,), "type": np.ndarray}, + "_extrinsic_matrix": {"shape": (4, 4), "type": np.ndarray}, } optional_attrs = ["size"] @@ -73,5 +80,151 @@ def _validate_shape(self, attribute: attrs.Attribute, value): expected_type = attr_metadata[attribute.name]["type"] if np.shape(value) != expected_shape: raise ValueError( - f"{attribute.name} must be a {expected_type} of size {expected_shape}" + f"{attribute.name} must be a {expected_type} of size {expected_shape}, " + f"but recieved shape: {np.shape(value)} and type: {type(value)} for " + f"value: {value}" ) + + def __attrs_post_init__(self): + """Initialize extrinsic matrix from rotation and translation vectors.""" + + # Initialize extrinsic matrix + self._extrinsic_matrix = np.eye(4, dtype="float64") + self._extrinsic_matrix[:3, :3] = cv2.Rodrigues(self._rvec)[0] + self._extrinsic_matrix[:3, 3] = self._tvec + + @property + def rvec(self) -> np.ndarray: + """Get rotation vector of camera. + + Returns: + Rotation vector of camera of size 3. + """ + + return self._rvec + + @rvec.setter + def rvec(self, value: np.ndarray): + """Set rotation vector and update extrinsic matrix. + + Args: + value: Rotation vector of size 3. + """ + self._rvec = value + + # Update extrinsic matrix + rotation_matrix, _ = cv2.Rodrigues(self._rvec) + self._extrinsic_matrix[:3, :3] = rotation_matrix + + @property + def tvec(self) -> np.ndarray: + """Get translation vector of camera. + + Returns: + Translation vector of camera of size 3. + """ + + return self._tvec + + @tvec.setter + def tvec(self, value: np.ndarray): + """Set translation vector and update extrinsic matrix. + + Args: + value: Translation vector of size 3. + """ + + self._tvec = value + + # Update extrinsic matrix + self._extrinsic_matrix[:3, 3] = self._tvec + + @property + def extrinsic_matrix(self) -> np.ndarray: + """Get extrinsic matrix of camera. + + Returns: + Extrinsic matrix of camera of size 4 x 4. + """ + + return self._extrinsic_matrix + + @extrinsic_matrix.setter + def extrinsic_matrix(self, value: np.ndarray): + """Set extrinsic matrix and update rotation and translation vectors. + + Args: + value: Extrinsic matrix of size 4 x 4. + """ + + self._extrinsic_matrix = value + + # Update rotation and translation vectors + self._rvec, _ = cv2.Rodrigues(self._extrinsic_matrix[:3, :3]) + self._tvec = self._extrinsic_matrix[:3, 3] + + def undistort_points(self, points: np.ndarray) -> np.ndarray: + """Undistort points using camera matrix and distortion coefficients. + + Args: + points: Points to undistort of shape (N, 2). + + Returns: + Undistorted points of shape (N, 2). + """ + + shape = points.shape + points = points.reshape(-1, 1, 2) + out = cv2.undistortPoints(points, self.matrix, self.dist) + return out.reshape(shape) + + def project(self, points: np.ndarray) -> np.ndarray: + """Project 3D points to 2D using camera matrix and distortion coefficients. + + Args: + points: 3D points to project of shape (N, 3) or (N, 1, 3). + + Returns: + Projected 2D points of shape (N, 1, 2). + """ + + points = points.reshape(-1, 1, 3) + out, _ = cv2.projectPoints( + points, + self.rvec, + self.tvec, + self.matrix, + self.dist, + ) + return out + + # TODO: Remove this when we implement triangulation without aniposelib + def __getattr__(self, name: str): + """Get attribute by name. + + Args: + name: Name of attribute to get. + + Returns: + Value of attribute. + + Raises: + AttributeError: If attribute does not exist. + """ + + if name in self.__attrs_attrs__: + return getattr(self, name) + + # The aliases for methods called when triangulate with sleap_anipose + method_aliases = { + "get_name": self.name, + "get_extrinsic_matrix": self.extrinsic_matrix, + } + + def return_callable_method_alias(): + return method_aliases[name] + + if name in method_aliases: + return return_callable_method_alias + + raise AttributeError(f"'Camera' object has no attribute or method '{name}'") From de1728e4ee7ecc224f78097f2a934e415a66311f Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:53:09 -0800 Subject: [PATCH 4/8] Add tests for extrinisic_matrix, undistort, project,and aliases --- tests/model/test_camera.py | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/model/test_camera.py b/tests/model/test_camera.py index 34183866..db70779d 100644 --- a/tests/model/test_camera.py +++ b/tests/model/test_camera.py @@ -1,5 +1,6 @@ """Tests for methods in the sleap_io.model.instance file.""" +import cv2 import numpy as np import pytest @@ -84,6 +85,25 @@ def test_camera_size(): camera.size = (100, 200, 300) +def construct_extrinsic_matrix(rvec, tvec): + """Construct extrinsic matrix from rotation and translation vectors. + + Args: + rvec: Rotation vector in unnormalized axis angle representation of size (3,) and + type float64. + tvec: Translation vector of size (3,) and type float64. + + Returns: + Extrinsic matrix of camera of size (4, 4) and type float64. + """ + + extrinsic_matrix = np.eye(4) + extrinsic_matrix[:3, :3] = cv2.Rodrigues(np.array(rvec))[0] + extrinsic_matrix[:3, 3] = tvec + + return extrinsic_matrix + + def test_camera_rvec(): """Test camera rotation vector converter and validator.""" @@ -92,15 +112,28 @@ def test_camera_rvec(): # During initialization camera = Camera(rvec=rvec) np.testing.assert_array_equal(camera.rvec, rvec) + extrinsic_matrix = construct_extrinsic_matrix(camera.rvec, camera.tvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + np.testing.assert_array_equal( + camera.extrinsic_matrix[:3, :3], cv2.Rodrigues(camera.rvec)[0] + ) with pytest.raises(ValueError): camera = Camera(rvec=[1, 2]) + np.testing.assert_array_equal(camera.rvec, rvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) # Test rvec validator camera = Camera() camera.rvec = rvec np.testing.assert_array_equal(camera.rvec, rvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + np.testing.assert_array_equal( + camera.extrinsic_matrix[:3, :3], cv2.Rodrigues(camera.rvec)[0] + ) with pytest.raises(ValueError): camera.rvec = [1, 2] + np.testing.assert_array_equal(camera.rvec, rvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) def test_camera_tvec(): @@ -111,12 +144,122 @@ def test_camera_tvec(): # During initialization camera = Camera(tvec=tvec) np.testing.assert_array_equal(camera.tvec, tvec) + extrinsic_matrix = construct_extrinsic_matrix(camera.rvec, camera.tvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + np.testing.assert_array_equal(camera.extrinsic_matrix[:3, 3], camera.tvec) with pytest.raises(ValueError): camera = Camera(tvec=[1, 2]) + np.testing.assert_array_equal(camera.tvec, tvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) # Test tvec validator camera = Camera() camera.tvec = tvec np.testing.assert_array_equal(camera.tvec, tvec) + extrinsic_matrix = construct_extrinsic_matrix(camera.rvec, camera.tvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + np.testing.assert_array_equal(camera.extrinsic_matrix[:3, 3], camera.tvec) with pytest.raises(ValueError): camera.tvec = [1, 2] + np.testing.assert_array_equal(camera.tvec, tvec) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + + +def test_camera_extrinsic_matrix(): + """Test camera extrinsic matrix method.""" + + # During initialization + + # ... with rvec and tvec + camera = Camera( + rvec=[1, 2, 3], + tvec=[1, 2, 3], + ) + extrinsic_matrix = camera.extrinsic_matrix + np.testing.assert_array_equal( + extrinsic_matrix[:3, :3], cv2.Rodrigues(camera.rvec)[0] + ) + np.testing.assert_array_equal(extrinsic_matrix[:3, 3], camera.tvec) + + # ... without rvec and tvec + camera = Camera() + extrinsic_matrix = camera.extrinsic_matrix + np.testing.assert_array_equal(extrinsic_matrix, np.eye(4)) + + # After initialization + + # Setting extrinsic matrix updates rvec and tvec + extrinsic_matrix = np.random.rand(4, 4) + camera.extrinsic_matrix = extrinsic_matrix + rvec = cv2.Rodrigues(camera.extrinsic_matrix[:3, :3])[0] + tvec = camera.extrinsic_matrix[:3, 3] + np.testing.assert_array_equal(camera.rvec, rvec.ravel()) + np.testing.assert_array_equal(camera.tvec, tvec) + + # Invalid extrinsic matrix doesn't update rvec and tvec or extrinsic matrix + with pytest.raises(ValueError): + camera.extrinsic_matrix = np.eye(3) + np.testing.assert_array_equal(camera.extrinsic_matrix, extrinsic_matrix) + np.testing.assert_array_equal(camera.rvec, rvec.ravel()) + np.testing.assert_array_equal(camera.tvec, tvec) + + +# TODO: Remove when implement triangulation without aniposelib +def test_camera_aliases(): + """Test camera aliases for attributes.""" + + camera = Camera( + matrix=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + dist=[[1], [2], [3], [4], [5]], + size=(100, 200), + rvec=[1, 2, 3], + tvec=[1, 2, 3], + name="camera", + ) + + # Test __getattr__ aliases + assert camera.get_name() == camera.name + np.testing.assert_array_equal( + camera.get_extrinsic_matrix(), camera.extrinsic_matrix + ) + + +def test_camera_undistort_points(): + """Test camera undistort points method.""" + + camera = Camera( + matrix=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], + dist=[[0], [0], [0], [0], [0]], + ) + + # Test with no distortion + points = np.array([[0, 0], [1, 1], [2, 2]], dtype=np.float32) + undistorted_points = camera.undistort_points(points) + np.testing.assert_array_equal(points, undistorted_points) + + # Test with distortion + camera.dist = [[1], [0], [0], [0], [0]] + undistorted_points = camera.undistort_points(points) + with pytest.raises(AssertionError): + np.testing.assert_array_equal(points, undistorted_points) + + +def test_camera_project(): + """Test camera project method.""" + + camera = Camera( + matrix=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], + dist=[[0], [0], [0], [0], [0]], + ) + + points = np.random.rand(10, 3) + projected_points = camera.project(points) + assert projected_points.shape == (points.shape[0], 1, 2) + + points = np.random.rand(10, 1, 3) + projected_points = camera.project(points) + assert projected_points.shape == (points.shape[0], 1, 2) + + +if __name__ == "__main__": + pytest.main([__file__]) From 553964e5efe064fd0665fc3e5868befee83273c8 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:57:03 -0800 Subject: [PATCH 5/8] Conform to pydocstyle --- sleap_io/model/camera.py | 10 ---------- tests/model/test_camera.py | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/sleap_io/model/camera.py b/sleap_io/model/camera.py index 8c902238..30a20acb 100644 --- a/sleap_io/model/camera.py +++ b/sleap_io/model/camera.py @@ -59,7 +59,6 @@ def _validate_shape(self, attribute: attrs.Attribute, value): Raises: ValueError: If attribute shape is not as expected. """ - # Define metadata for each attribute attr_metadata = { "matrix": {"shape": (3, 3), "type": np.ndarray}, @@ -87,7 +86,6 @@ def _validate_shape(self, attribute: attrs.Attribute, value): def __attrs_post_init__(self): """Initialize extrinsic matrix from rotation and translation vectors.""" - # Initialize extrinsic matrix self._extrinsic_matrix = np.eye(4, dtype="float64") self._extrinsic_matrix[:3, :3] = cv2.Rodrigues(self._rvec)[0] @@ -100,7 +98,6 @@ def rvec(self) -> np.ndarray: Returns: Rotation vector of camera of size 3. """ - return self._rvec @rvec.setter @@ -123,7 +120,6 @@ def tvec(self) -> np.ndarray: Returns: Translation vector of camera of size 3. """ - return self._tvec @tvec.setter @@ -133,7 +129,6 @@ def tvec(self, value: np.ndarray): Args: value: Translation vector of size 3. """ - self._tvec = value # Update extrinsic matrix @@ -146,7 +141,6 @@ def extrinsic_matrix(self) -> np.ndarray: Returns: Extrinsic matrix of camera of size 4 x 4. """ - return self._extrinsic_matrix @extrinsic_matrix.setter @@ -156,7 +150,6 @@ def extrinsic_matrix(self, value: np.ndarray): Args: value: Extrinsic matrix of size 4 x 4. """ - self._extrinsic_matrix = value # Update rotation and translation vectors @@ -172,7 +165,6 @@ def undistort_points(self, points: np.ndarray) -> np.ndarray: Returns: Undistorted points of shape (N, 2). """ - shape = points.shape points = points.reshape(-1, 1, 2) out = cv2.undistortPoints(points, self.matrix, self.dist) @@ -187,7 +179,6 @@ def project(self, points: np.ndarray) -> np.ndarray: Returns: Projected 2D points of shape (N, 1, 2). """ - points = points.reshape(-1, 1, 3) out, _ = cv2.projectPoints( points, @@ -211,7 +202,6 @@ def __getattr__(self, name: str): Raises: AttributeError: If attribute does not exist. """ - if name in self.__attrs_attrs__: return getattr(self, name) diff --git a/tests/model/test_camera.py b/tests/model/test_camera.py index db70779d..383f92e6 100644 --- a/tests/model/test_camera.py +++ b/tests/model/test_camera.py @@ -9,7 +9,6 @@ def test_camera_name(): """Test camera name converter always converts to string.""" - # During initialization camera = Camera(name=12) assert camera.name == "12" @@ -25,7 +24,6 @@ def test_camera_name(): def test_camera_matrix(): """Test camera matrix converter and validator.""" - matrix_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] matrix_array = np.array(matrix_list) @@ -48,7 +46,6 @@ def test_camera_matrix(): def test_camera_distortions(): """Test camera distortion converter and validator.""" - distortions_unraveled = [[1], [2], [3], [4], [5]] distortions_raveled = np.array(distortions_unraveled).ravel() @@ -68,7 +65,6 @@ def test_camera_distortions(): def test_camera_size(): """Test camera size converter and validator.""" - size = (100, 200) # During initialization @@ -96,7 +92,6 @@ def construct_extrinsic_matrix(rvec, tvec): Returns: Extrinsic matrix of camera of size (4, 4) and type float64. """ - extrinsic_matrix = np.eye(4) extrinsic_matrix[:3, :3] = cv2.Rodrigues(np.array(rvec))[0] extrinsic_matrix[:3, 3] = tvec @@ -106,7 +101,6 @@ def construct_extrinsic_matrix(rvec, tvec): def test_camera_rvec(): """Test camera rotation vector converter and validator.""" - rvec = [1, 2, 3] # During initialization @@ -138,7 +132,6 @@ def test_camera_rvec(): def test_camera_tvec(): """Test camera translation vector converter and validator.""" - tvec = [1, 2, 3] # During initialization @@ -167,7 +160,6 @@ def test_camera_tvec(): def test_camera_extrinsic_matrix(): """Test camera extrinsic matrix method.""" - # During initialization # ... with rvec and tvec @@ -207,7 +199,6 @@ def test_camera_extrinsic_matrix(): # TODO: Remove when implement triangulation without aniposelib def test_camera_aliases(): """Test camera aliases for attributes.""" - camera = Camera( matrix=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], dist=[[1], [2], [3], [4], [5]], @@ -226,7 +217,6 @@ def test_camera_aliases(): def test_camera_undistort_points(): """Test camera undistort points method.""" - camera = Camera( matrix=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], dist=[[0], [0], [0], [0], [0]], @@ -246,7 +236,6 @@ def test_camera_undistort_points(): def test_camera_project(): """Test camera project method.""" - camera = Camera( matrix=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], dist=[[0], [0], [0], [0], [0]], From 32367ab2ba4474983f6c82f6525e6c8d3f906871 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:40:40 -0800 Subject: [PATCH 6/8] Add method to link Camera to Video --- sleap_io/model/camera.py | 92 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/sleap_io/model/camera.py b/sleap_io/model/camera.py index 30a20acb..896bc4e5 100644 --- a/sleap_io/model/camera.py +++ b/sleap_io/model/camera.py @@ -7,8 +7,88 @@ import numpy as np from attrs import define, field +from sleap_io.model.video import Video + @define +class CameraGroup: + """A group of cameras used to record a multi-view `RecordingSession`. + + Attributes: + cameras: List of `Camera` objects in the group. + """ + + cameras: list[Camera] = field(factory=list) + + +@define(eq=False) # Set eq to false to make class hashable +class RecordingSession: + """A recording session with multiple cameras. + + Attributes: + camera_group: `CameraGroup` object containing cameras in the session. + _video_by_camera: Dictionary mapping `Camera` to `Video`. + _camera_by_video: Dictionary mapping `Video` to `Camera`. + """ + + camera_group: CameraGroup = field(factory=CameraGroup) + _video_by_camera: dict[Camera, Video] = field(factory=dict) + _camera_by_video: dict[Video, Camera] = field(factory=dict) + + def get_video(self, camera: Camera) -> Video | None: + """Get `Video` associated with `Camera`. + + Args: + camera: Camera to get video + + Returns: + Video associated with camera or None if not found + """ + return self._video_by_camera.get(camera, None) + + def add_video(self, video: Video, camera: Camera): + """Add `Video` to `RecordingSession` and mapping to `Camera`. + + Args: + video: `Video` object to add to `RecordingSession`. + camera: `Camera` object to associate with `Video`. + + Raises: + ValueError: If `Camera` is not in associated `CameraGroup`. + ValueError: If `Video` is not a `Video` object. + """ + # Raise ValueError if camera is not in associated camera group + self.camera_group.cameras.index(camera) + + # Raise ValueError if `Video` is not a `Video` object + if not isinstance(video, Video): + raise ValueError( + f"Expected `Video` object, but received {type(video)} object." + ) + + # Add camera to video mapping + self._video_by_camera[camera] = video + + # Add video to camera mapping + self._camera_by_video[video] = camera + + def remove_video(self, video: Video): + """Remove `Video` from `RecordingSession` and mapping to `Camera`. + + Args: + video: `Video` object to remove from `RecordingSession`. + + Raises: + ValueError: If `Video` is not in associated `RecordingSession`. + """ + # Remove video from camera mapping + camera = self._camera_by_video.pop(video) + + # Remove camera from video mapping + self._video_by_camera.pop(camera) + + +@define(eq=False) # Set eq to false to make class hashable class Camera: """A camera used to record in a multi-view `RecordingSession`. @@ -22,6 +102,7 @@ class Camera: tvec: Translation vector of size (3,) and type float64. extrinsic_matrix: Extrinsic matrix of camera of size (4, 4) and type float64. name: Camera name. + _video_by_session: Dictionary mapping `RecordingSession` to `Video`. """ matrix: np.ndarray = field( @@ -189,6 +270,17 @@ def project(self, points: np.ndarray) -> np.ndarray: ) return out + def get_video(self, session: RecordingSession) -> Video | None: + """Get video associated with recording session. + + Args: + session: Recording session to get video for. + + Returns: + Video associated with recording session or None if not found. + """ + return session.get_video(camera=self) + # TODO: Remove this when we implement triangulation without aniposelib def __getattr__(self, name: str): """Get attribute by name. From 6240f0da16e94328ea9eb421f7714595ea2e9c9e Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:41:12 -0800 Subject: [PATCH 7/8] Test Camera to Video link --- tests/model/test_camera.py | 61 ++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/tests/model/test_camera.py b/tests/model/test_camera.py index 383f92e6..243e672d 100644 --- a/tests/model/test_camera.py +++ b/tests/model/test_camera.py @@ -4,7 +4,8 @@ import numpy as np import pytest -from sleap_io.model.camera import Camera +from sleap_io.model.camera import Camera, CameraGroup, RecordingSession +from sleap_io.model.video import Video def test_camera_name(): @@ -196,25 +197,6 @@ def test_camera_extrinsic_matrix(): np.testing.assert_array_equal(camera.tvec, tvec) -# TODO: Remove when implement triangulation without aniposelib -def test_camera_aliases(): - """Test camera aliases for attributes.""" - camera = Camera( - matrix=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - dist=[[1], [2], [3], [4], [5]], - size=(100, 200), - rvec=[1, 2, 3], - tvec=[1, 2, 3], - name="camera", - ) - - # Test __getattr__ aliases - assert camera.get_name() == camera.name - np.testing.assert_array_equal( - camera.get_extrinsic_matrix(), camera.extrinsic_matrix - ) - - def test_camera_undistort_points(): """Test camera undistort points method.""" camera = Camera( @@ -250,5 +232,40 @@ def test_camera_project(): assert projected_points.shape == (points.shape[0], 1, 2) -if __name__ == "__main__": - pytest.main([__file__]) +def test_camera_get_video(): + """Test camera get video method.""" + camera = Camera() + camera_group = CameraGroup(cameras=[camera]) + + # Test with no video + session = RecordingSession(camera_group=camera_group) + video = camera.get_video(session) + assert video is None + + # Test with video + video = Video(filename="not/a/file.mp4") + session.add_video(video=video, camera=camera) + assert camera.get_video(session) is video + + # Remove video + session.remove_video(video) + assert camera.get_video(session) is None + + +# TODO: Remove when implement triangulation without aniposelib +def test_camera_aliases(): + """Test camera aliases for attributes.""" + camera = Camera( + matrix=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + dist=[[1], [2], [3], [4], [5]], + size=(100, 200), + rvec=[1, 2, 3], + tvec=[1, 2, 3], + name="camera", + ) + + # Test __getattr__ aliases + assert camera.get_name() == camera.name + np.testing.assert_array_equal( + camera.get_extrinsic_matrix(), camera.extrinsic_matrix + ) From 250a8c545aca6ddfba37a4d672c810f7c3e37341 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:34:48 -0800 Subject: [PATCH 8/8] Convert points to float before cv2.undistortPoints --- sleap_io/model/camera.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sleap_io/model/camera.py b/sleap_io/model/camera.py index 896bc4e5..530939f3 100644 --- a/sleap_io/model/camera.py +++ b/sleap_io/model/camera.py @@ -248,7 +248,7 @@ def undistort_points(self, points: np.ndarray) -> np.ndarray: """ shape = points.shape points = points.reshape(-1, 1, 2) - out = cv2.undistortPoints(points, self.matrix, self.dist) + out = cv2.undistortPoints(points.astype("float64"), self.matrix, self.dist) return out.reshape(shape) def project(self, points: np.ndarray) -> np.ndarray: