From a3dd87249f81d77331b7cb7b8144618a97524aa7 Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 21:12:20 +0800 Subject: [PATCH 1/6] Add __dir__ where needed to support tab completion. --- swiftgalaxy/reader.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/swiftgalaxy/reader.py b/swiftgalaxy/reader.py index a294c9a..54e93ec 100644 --- a/swiftgalaxy/reader.py +++ b/swiftgalaxy/reader.py @@ -196,6 +196,22 @@ def __init__(self, coordinates: Union[dict, cosmo_array], masks: dict) -> None: self._masks: dict = masks return + def __dir__(self) -> list[str]: + """ + Supply a list of attributes of the :class:`~swiftgalaxy.reader._CoordinateHelper`. + + The regular ``dir`` behaviour doesn't index the names of the coordinates + because these are stored in a ``dict`` held by the class, so we customize + the ``__dir__`` method to list the coordinate names. They will then appear in + tab completion, for example. + + Returns + ------- + out : list + List of coordinate name strings. + """ + return list(self._masks.keys()) + def __getattr__(self, attr: str) -> cosmo_array: """ Get a coordinate array using attribute (dot) syntax. @@ -321,6 +337,24 @@ def __repr__(self) -> str: """ return self.__str__() + def __dir__(self) -> list[str]: + """ + Supply a list of attributes of the + :class:`~swiftgalaxy.reader._SWIFTNamedColumnDatasetHelper`. + + The regular ``dir`` behaviour doesn't index the names of the named columns + because these are stored in the + :class:`~swiftsimio.reader.__SWIFTNamedColumnDataset` held by the class, so we + customize the ``__dir__`` method to list the named column names. They will then + appear in tab completion, for example. + + Returns + ------- + out : list + List of named column name strings. + """ + return dir(self._named_column_dataset) + def __getattribute__(self, attr: str) -> Any: """ Get an attribute of the object. @@ -547,6 +581,24 @@ class _SWIFTGroupDatasetHelper(object): mygalaxy.gas.cylindrical_velocities.z """ + def __dir__(self) -> list[str]: + """ + Supply a list of attributes of the + :class:`~swiftgalaxy.reader._SWIFTGroupDatasetHelper`. + + The regular ``dir`` behaviour doesn't index the names of the particle datasets + because these are stored in the + :class:`~swiftsimio.reader.__SWIFTGroupDataset` held by the class, so we + customize the ``__dir__`` method to list the particle dataset names. They will + then appear in tab completion, for example. + + Returns + ------- + out : list + List of named column name strings. + """ + return dir(self._particle_dataset) + def __init__( self, particle_dataset: "__SWIFTGroupDataset", swiftgalaxy: "SWIFTGalaxy" ) -> None: From 702841d4ef3ec7a4dd37944696804f978654887d Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 21:55:30 +0800 Subject: [PATCH 2/6] Also support tab completion for halo catalogues with __dir__. --- swiftgalaxy/halo_catalogues.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/swiftgalaxy/halo_catalogues.py b/swiftgalaxy/halo_catalogues.py index 5bdd3f9..c93af5d 100644 --- a/swiftgalaxy/halo_catalogues.py +++ b/swiftgalaxy/halo_catalogues.py @@ -337,6 +337,23 @@ def _check_multi(self) -> None: assert self.extra_mask in (None, "bound_only") return + def __dir__(self) -> list[str]: + """ + Supply a list of attributes of the halo catalogue. + + The regular ``dir`` behaviour doesn't index the names of catalogue attributes + because they're attached to the internally maintained ``_catalogue`` attribute, + so we custimize the ``__dir__`` method to list the attribute names. They will + then appear in tab completion, for example. + + Returns + ------- + out : list + The list of catalogue attribute names. + """ + # use getattr to default to None, e.g. for Standalone + return dir(getattr(self, "_catalogue", None)) + def __getattr__(self, attr: str) -> Any: """ Exposes the masked halo catalogue. @@ -1057,7 +1074,6 @@ def _load(self) -> None: of interest. """ import h5py - from velociraptor.catalogue.catalogue import Catalogue as VelociraptorCatalogue from velociraptor import load as load_catalogue from velociraptor.particles import load_groups @@ -1068,7 +1084,7 @@ def _load(self) -> None: else 1.0 ) - self._catalogue: "VelociraptorCatalogue" = load_catalogue( + self._catalogue = load_catalogue( self.velociraptor_files["properties"], mask=np.array(self._halo_index) ) groups = load_groups( From c91dec7f39a2063594cc86811e22cbe4f8d110fe Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 21:56:07 +0800 Subject: [PATCH 3/6] Add tests for tab completion features. --- tests/test_creation.py | 37 +++++++++++++++++++++++++++++++ tests/test_derived_coordinates.py | 12 ++++++++++ tests/test_halo_catalogues.py | 28 +++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/tests/test_creation.py b/tests/test_creation.py index 45880eb..d46ef7c 100644 --- a/tests/test_creation.py +++ b/tests/test_creation.py @@ -1,31 +1,68 @@ class TestSWIFTGalaxyCreation: def test_sg_creation(self, sg): + """ + Make sure we can create a SWIFTGalaxy without error. + """ pass # fixture created SWIFTGalaxy def test_soap_creation(self, soap): + """ + Make sure we can create a SOAP without error. + """ pass # fixture created SOAP interface def test_vr_creation(self, vr): + """ + Make sure we can create a Velociraptor without error. + """ pass # fixture created Velociraptor interface def test_caesar_creation(self, caesar): + """ + Make sure we can create a Caesar without error. + """ pass # fixture created Caesar interface def test_sa_creation(self, sa): + """ + Make sure we can create a Standalone without error. + """ pass # fixture created Standalone interface def test_sg_soap_creation(self, sg_soap): + """ + Make sure we can create a SWIFTGalaxy with SOAP without error. + """ pass # fixture created SWIFTGalaxy with SOAP interface def test_sg_vr_creation(self, sg_vr): + """ + Make sure we can create a SWIFTGalaxy with velociraptor without error. + """ pass # fixture created SWIFTGalaxy with Velociraptor interface def test_sg_caesar_creation(self, sg_caesar): + """ + Make sure we can create a SWIFTGalaxy with Caesar without error. + """ pass # fixture created SWIFTGalaxy with Caesar interface def test_sg_sa_creation(self, sg_sa): + """ + Make sure we can create a SWIFTGalaxy with Standalone without error. + """ pass # fixture created SWIFTGalaxy with Standalone interface + def test_dir_for_tab_completion(self, sg): + """ + Check that particle dataset names and named column names get injected into + the namespace for tab completion by the __dir__ methods of the relevant classes. + """ + for prop in ("coordinates", "masses", "hydrogen_ionization_fractions"): + assert prop in dir(sg.gas) + for prop in ("neutral", "ionized"): + assert prop in dir(sg.gas.hydrogen_ionization_fractions) + class TestSWIFTGalaxiesCreation: def test_sgs_creation(self, sgs): diff --git a/tests/test_derived_coordinates.py b/tests/test_derived_coordinates.py index 4eceb64..31b920d 100644 --- a/tests/test_derived_coordinates.py +++ b/tests/test_derived_coordinates.py @@ -596,3 +596,15 @@ def test_single_particle_produces_array_cylindrical( ).shape is not tuple() ) + + def test_dir_for_tab_completion(self, sg): + """ + Check that we can see coordinate names when using dir on the helper class. + We don't test for an exhaustive list, as long as some appear all will. + """ + for coord in ("x", "xyz"): + assert coord in dir(sg.gas.cartesian_coordinates) + for coord in ("r", "lon", "radius"): + assert coord in dir(sg.gas.spherical_coordinates) + for coord in ("rho", "z", "azimuth"): + assert coord in dir(sg.gas.cylindrical_coordinates) diff --git a/tests/test_halo_catalogues.py b/tests/test_halo_catalogues.py index a7b9d12..b49e7e8 100644 --- a/tests/test_halo_catalogues.py +++ b/tests/test_halo_catalogues.py @@ -508,6 +508,15 @@ def test_masking_catalogue(self, vr_multi): vr_multi._mask_multi_galaxy(0) self.test_catalogue_exposed(vr_multi) + def test_dir_for_tab_completion(self, vr): + """ + Check that we add catalogue properties to the namespace directory. + + Just check a couple, don't need to be exhaustive. + """ + for prop in ("energies", "metallicity", "temperature"): + assert prop in dir(vr) + class TestVelociraptorWithSWIFTGalaxy: """ @@ -671,6 +680,16 @@ def test_spatial_mask_applied(self, caesar, toysnap): )[particle_type] ) + def test_dir_for_tab_completion(self, caesar): + """ + Check that we add catalogue properties to the namespace directory. + + Just check a couple, don't need to be exhaustive. + """ + # picked these to be common between halo and galaxy catalogues: + for prop in ("glist", "pos", "radii"): + assert prop in dir(caesar) + class TestCaesarWithSWIFTGalaxy: """ @@ -859,6 +878,15 @@ def test_masking_catalogue(self, soap_multi): soap_multi._mask_multi_galaxy(0) self.test_catalogue_exposed(soap_multi) + def test_dir_for_tab_completion(self, soap): + """ + Check that we add catalogue properties to the namespace directory. + + Just check a couple, don't need to be exhaustive. + """ + for prop in ("exclusive_sphere_100kpc", "input_halos", "metadata"): + assert prop in dir(soap) + class TestSOAPWithSWIFTGalaxy: """ From 383af0f693ad692b4cdcf1a6eeb2ed0dffe8862b Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 22:35:05 +0800 Subject: [PATCH 4/6] Explicitly look up attribute lists to try to avoid reading all of the data. --- swiftgalaxy/halo_catalogues.py | 16 ++++++++++++++++ swiftgalaxy/reader.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/swiftgalaxy/halo_catalogues.py b/swiftgalaxy/halo_catalogues.py index c93af5d..dae9b18 100644 --- a/swiftgalaxy/halo_catalogues.py +++ b/swiftgalaxy/halo_catalogues.py @@ -902,6 +902,22 @@ def __repr__(self) -> str: """ return self._catalogue.__repr__() + def __dir__(self) -> list[str]: + """ + Supply a list of attributes of the halo catalogue. + + The regular ``dir`` behaviour doesn't index the names of catalogue attributes + because they're attached to the internally maintained ``_catalogue`` attribute, + so we custimize the ``__dir__`` method to list the attribute names. They will + then appear in tab completion, for example. + + Returns + ------- + out : list + The list of catalogue attribute names. + """ + return list(self._catalogue.group_metadata.field_names) + class Velociraptor(_HaloCatalogue): """ diff --git a/swiftgalaxy/reader.py b/swiftgalaxy/reader.py index 54e93ec..8078950 100644 --- a/swiftgalaxy/reader.py +++ b/swiftgalaxy/reader.py @@ -353,7 +353,7 @@ def __dir__(self) -> list[str]: out : list List of named column name strings. """ - return dir(self._named_column_dataset) + return self._named_column_dataset.named_columns def __getattribute__(self, attr: str) -> Any: """ @@ -597,7 +597,7 @@ def __dir__(self) -> list[str]: out : list List of named column name strings. """ - return dir(self._particle_dataset) + return self._particle_dataset.group_metadata.field_names def __init__( self, particle_dataset: "__SWIFTGroupDataset", swiftgalaxy: "SWIFTGalaxy" From b4d4e04a041948b92fb4b879a4362cb3b67e4606 Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 22:35:30 +0800 Subject: [PATCH 5/6] Document tab completion and it's unfortunate side effect. --- docs/source/getting_started/index.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/getting_started/index.rst b/docs/source/getting_started/index.rst index d30f4ae..d162344 100644 --- a/docs/source/getting_started/index.rst +++ b/docs/source/getting_started/index.rst @@ -95,4 +95,14 @@ In this case with :class:`~swiftgalaxy.halo_catalogues.Velociraptor`, we can get sg.halo_catalogue.masses.mvir +:mod:`swiftgalaxy` supports Python's tab completion features. This means that you can browse the available attributes of objects in an interactive interpreter by starting to type an attribute (or just a trailing dot) and pressing tab twice. A few examples to help start exploring: + + - ``sg.`` + - ``sg.gas.`` + - ``sg.halo_catalogue.`` + +.. warning:: + + As currently implemented, using tab completion accesses the attributes of the object that is being searched for attriubtes. This triggers :mod:`swiftsimio`'s lazy-loading functionality. The result is that using tab completion like ``sg.gas.`` reads *all* gas particle property arrays from the snapshot! This is obviously not desirable behaviour; a better implementation is being sought. + The further features of a :class:`~swiftgalaxy.reader.SWIFTGalaxy` are detailed in the next sections. From 3e7731a4a53b61f23a2a260db0c9844191d5f087 Mon Sep 17 00:00:00 2001 From: Kyle Oman Date: Sun, 1 Dec 2024 23:06:58 +0800 Subject: [PATCH 6/6] Access list of swiftsimio attributes correctly. --- swiftgalaxy/halo_catalogues.py | 2 +- tests/test_halo_catalogues.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/swiftgalaxy/halo_catalogues.py b/swiftgalaxy/halo_catalogues.py index dae9b18..62947b9 100644 --- a/swiftgalaxy/halo_catalogues.py +++ b/swiftgalaxy/halo_catalogues.py @@ -916,7 +916,7 @@ def __dir__(self) -> list[str]: out : list The list of catalogue attribute names. """ - return list(self._catalogue.group_metadata.field_names) + return list(self._catalogue.metadata.present_group_names) class Velociraptor(_HaloCatalogue): diff --git a/tests/test_halo_catalogues.py b/tests/test_halo_catalogues.py index b49e7e8..0a0b89e 100644 --- a/tests/test_halo_catalogues.py +++ b/tests/test_halo_catalogues.py @@ -884,7 +884,11 @@ def test_dir_for_tab_completion(self, soap): Just check a couple, don't need to be exhaustive. """ - for prop in ("exclusive_sphere_100kpc", "input_halos", "metadata"): + for prop in ( + "exclusive_sphere_100kpc", + "input_halos", + "spherical_overdensity_bn98", + ): assert prop in dir(soap)