From 59907f550ea27921ac4c6d92f523f6ff1c425bec Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Wed, 19 Apr 2023 16:11:31 -0400 Subject: [PATCH 01/11] Fixed imports --- triplespec_focus/triplespec_focus.py | 4 ++-- triplespec_focus/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/triplespec_focus/triplespec_focus.py b/triplespec_focus/triplespec_focus.py index 2e903c8..3e3fc35 100644 --- a/triplespec_focus/triplespec_focus.py +++ b/triplespec_focus/triplespec_focus.py @@ -16,8 +16,8 @@ from pandas import DataFrame from pandas import concat from pathlib import Path -from photutils import DAOStarFinder -from photutils import CircularAperture +from photutils.detection import DAOStarFinder +from photutils.aperture import CircularAperture from scipy import optimize from typing import List, Union diff --git a/triplespec_focus/utils.py b/triplespec_focus/utils.py index e1189a0..87c883d 100644 --- a/triplespec_focus/utils.py +++ b/triplespec_focus/utils.py @@ -15,7 +15,7 @@ from pandas import DataFrame from typing import Union, List -from photutils import CircularAperture, CircularAnnulus, ApertureStats +from photutils.aperture import CircularAperture, CircularAnnulus, ApertureStats log = logging.getLogger(__name__) From 94acbd1033c4714a1085e215d2cc9eeb8d1ed199 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Wed, 19 Apr 2023 16:15:40 -0400 Subject: [PATCH 02/11] updated format of passenv value --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 357c4c1..1a32be5 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ setenv = NPY_LAPACK_ORDER= # Pass through the following environment variables which may be needed for the CI -passenv = HOME WINDIR LC_ALL LC_CTYPE CC CI +passenv = HOME,WINDIR,LC_ALL,LC_CTYPE,CC,CI # Run the tests in a temporary directory to make sure that we don't import # this package from the source tree From 9d634a66922df65ddb20ac83d7f1871bdf29f87c Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Fri, 21 Apr 2023 18:02:59 -0400 Subject: [PATCH 03/11] Added autoclass_content = 'both' to display init method docstrings --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 69afe8c..bc7827a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,6 +44,7 @@ 'matplotlib.sphinxext.plot_directive', ] +autoclass_content = 'both' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 172a15cbff11e666751c4a0fe81ee25bee9b5fd9 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Fri, 21 Apr 2023 18:04:18 -0400 Subject: [PATCH 04/11] Changed flake8 line length to 120 characters, removed pip freeze command from tox configuration --- setup.cfg | 4 ++-- tox.ini | 3 --- triplespec_focus/tests/test_triplespec_focus.py | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 694249f..694a3e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,6 +71,6 @@ triplespec_focus = triplespec_focus.run_triplespec_focus # W504 line break after binary operator # W505 doc line too long [flake8] -max-line-length = 100 -max-doc-length = 79 +max-line-length = 120 +max-doc-length = 120 ignore = E126,E127,E128,E201,E202,E203,E221,E225,E226,E227,E231,E251,E501,E731,E741,W503,W504,W505 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 1a32be5..1e536ea 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,6 @@ extras = alldeps: all commands = - pip freeze !cov: pytest --pyargs triplespec_focus {toxinidir}/docs {posargs} cov: pytest --pyargs triplespec_focus {toxinidir}/docs --cov triplespec_focus --cov-config={toxinidir}/setup.cfg {posargs} @@ -75,7 +74,6 @@ changedir = docs description = invoke sphinx-build to build the HTML docs extras = docs commands = - pip freeze sphinx-build -W -b html . _build/html [testenv:linkcheck] @@ -83,7 +81,6 @@ changedir = docs description = check the links in the HTML docs extras = docs commands = - pip freeze sphinx-build -W -b linkcheck . _build/html [testenv:codestyle] diff --git a/triplespec_focus/tests/test_triplespec_focus.py b/triplespec_focus/tests/test_triplespec_focus.py index 9fcc3cd..bcad5ec 100644 --- a/triplespec_focus/tests/test_triplespec_focus.py +++ b/triplespec_focus/tests/test_triplespec_focus.py @@ -89,5 +89,3 @@ def test_call_with_file_list(self): results = self.tspec_focus(file_list=sorted(data_path.glob(pattern='*.fits'))) self.assertIsInstance(results, dict) self.assertEqual(len(results), 11) - - From 7b7c5a43df338d99d584456106889b273666a819 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Fri, 21 Apr 2023 18:05:42 -0400 Subject: [PATCH 05/11] fixed docstrings formatting --- triplespec_focus/triplespec_focus.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/triplespec_focus/triplespec_focus.py b/triplespec_focus/triplespec_focus.py index 3e3fc35..4642d73 100644 --- a/triplespec_focus/triplespec_focus.py +++ b/triplespec_focus/triplespec_focus.py @@ -59,7 +59,8 @@ def __init__(self, n_brightest (int): Number of the brightest sources to use for measuring source statistics. Default 5. saturation (float): Data value at which the detector saturates. Default 40000. plot_results (bool): If set to True will display information plots at the end. Default False. - debug_plots (bool): If set to True will display several plots useful for debugging or viewing the process. Default False. + debug_plots (bool): If set to True will display several plots useful for debugging or viewing the process. + Default False. """ self.best_fwhm = None @@ -113,12 +114,15 @@ def __call__(self, file_list (List, None): Optional file list with files to be used to obtain best focus. Default None. source_fwhm (float): Full width at half maximum to use for source detection and statistics. det_threshold (float): Number of standard deviation above median to use as detection threshold. Default 5.0. - mask_threshold (float): Number of standard deviation below median to use as a threshold for masking values. Default 1. + mask_threshold (float): Number of standard deviation below median to use as a threshold for masking values. + Default 1. n_brightest (int): Number of the brightest sources to use for measuring source statistics. Default 5. saturation_level (float): Data value at which the detector saturates. Default 40000. - show_mask (bool): If set to True will display masked values in red when debug_plots is also True: Default False. + show_mask (bool): If set to True will display masked values in red when debug_plots is also True: + Default False. plot_results (bool): If set to True will display information plots at the end. Default False. - debug_plots (bool): If set to True will display several plots useful for debugging or viewing the process. Default False. + debug_plots (bool): If set to True will display several plots useful for debugging or viewing the process. + Default False. print_all_data (bool): If set to True will print the entire dataset at the end. Returns: @@ -276,7 +280,8 @@ def detect_sources(self, ccd: CCDData, debug_plots: bool = False) -> QTable: Args: ccd (CCDData): An image with point sources. - debug_plots (bool): If set to True will display the image with the sources. Default False. + debug_plots (bool): If set to True will display the image with the sources. + Default False. Returns: An Astropy's QTable containing ids, centroids, focus value and image name. @@ -337,7 +342,8 @@ def get_best_focus(self, df: DataFrame, x_axis_size: int = 2000) -> List[np.ndar """Obtains the best focus for a single source Args: - df (DataFrame): Pandas DataFrame containing at least a 'focus' and a 'fwhm' column. The data should belong to a single source. + df (DataFrame): Pandas DataFrame containing at least a 'focus' and a 'fwhm' column. + The data should belong to a single source. x_axis_size (int): Size of the x-axis used to sample the fitted model. Is not an interpolation size. Returns: From 2800791100356d0b962ea1e46116779c5e936a11 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Fri, 21 Apr 2023 18:06:52 -0400 Subject: [PATCH 06/11] Added error handling --- triplespec_focus/triplespec_focus.py | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/triplespec_focus/triplespec_focus.py b/triplespec_focus/triplespec_focus.py index 4642d73..ed34e92 100644 --- a/triplespec_focus/triplespec_focus.py +++ b/triplespec_focus/triplespec_focus.py @@ -204,9 +204,11 @@ def __call__(self, for star_id in star_ids: star_phot = self.sources_df[self.sources_df['id'] == star_id] interpolated_data = self.get_best_focus(df=star_phot) - all_stars_photometry.append([star_phot, interpolated_data, self.best_focus]) - all_focus.append(self.best_focus) - all_fwhm.append(self.best_fwhm) + if interpolated_data: + all_stars_photometry.append([star_phot, interpolated_data, self.best_focus]) + if self.best_focus and self.best_fwhm: + all_focus.append(self.best_focus) + all_fwhm.append(self.best_fwhm) mean_focus = np.mean(all_focus) median_focus = np.median(all_focus) @@ -214,7 +216,8 @@ def __call__(self, mean_fwhm = np.mean(all_fwhm) best_image_overall = CCDData.read(self.best_image_overall, unit='adu') - self.best_image_fwhm = self.sources_df[self.sources_df['filename'] == os.path.basename(self.best_image_overall)]['fwhm'].mean() + self.best_image_fwhm = self.sources_df[self.sources_df['filename'] == os.path.basename( + self.best_image_overall)]['fwhm'].mean() focus_data = [] fwhm_data = [] @@ -359,11 +362,20 @@ def get_best_focus(self, df: DataFrame, x_axis_size: int = 2000) -> List[np.ndar modeled_data = self.fitted_model(x_axis) index_of_minimum = np.argmin(modeled_data) middle_point = x_axis[index_of_minimum] - - self.best_focus = optimize.brent(self.fitted_model, brack=(focus_start, middle_point, focus_end)) - self.best_fwhm = modeled_data[index_of_minimum] - - return [x_axis, modeled_data] + if middle_point == focus_start or middle_point == focus_end: + self.log.warning("The focus vs FWHM curve does not seem to have a V or U shape. Trying by forcing the " + "mean focus as the middle point for Brent's optimization bracket definition.") + middle_point = (focus_start + focus_end) / 2. + + self.log.debug(f"Brent optimization bracket, Start (xa): {focus_start} Middle (xb): {middle_point} End (xc): {focus_end}") + + try: + self.best_focus = optimize.brent(self.fitted_model, brack=(focus_start, middle_point, focus_end)) + self.best_fwhm = modeled_data[index_of_minimum] + self.log.info(f"Found best focus at {self.best_focus} with a fwhm of {self.best_fwhm}") + return [x_axis, modeled_data] + except ValueError as error: + self.log.error(error) if __name__ == '__main__': From bde1fcfb94af57b511adb9be1004fc51d93edde4 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Fri, 21 Apr 2023 18:13:21 -0400 Subject: [PATCH 07/11] Added test to be ignored. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index bc7827a..f439538 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'test_', 'tests'] # -- Options for HTML output ------------------------------------------------- From f53ee75e9331a88fb95fd56de7f0105643e36256 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Mon, 24 Apr 2023 13:21:03 -0400 Subject: [PATCH 08/11] added omit test files in coverage run --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 694a3e9..c91ac7e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,6 +49,9 @@ docs = sphinx sphinx-rtd-theme +[coverage:run] +omit=*/tests/*,*__init__.py + [options.entry_points] triplespec_focus = triplespec_focus.run_triplespec_focus From 68adce74993c03fdfa766a0980f80916cbf4f7f2 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Mon, 24 Apr 2023 22:29:33 -0400 Subject: [PATCH 09/11] Added test that checks that when the data does not have v or U shape it will return None. --- triplespec_focus/tests/test_triplespec_focus.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/triplespec_focus/tests/test_triplespec_focus.py b/triplespec_focus/tests/test_triplespec_focus.py index bcad5ec..00f7aca 100644 --- a/triplespec_focus/tests/test_triplespec_focus.py +++ b/triplespec_focus/tests/test_triplespec_focus.py @@ -70,6 +70,16 @@ def test_get_focus_with_data(self): self.assertIsInstance(self.tspec_focus.fitted_model, Model) self.assertIsInstance(results, list) + def test_get_focus_with_fwhm_monotonically_increasing(self): + focus_values = [-1300, -1200, -1100, -1000, -900, -800, -700] + fwhm_values = [1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7] + data = {'focus': focus_values, 'fwhm': fwhm_values} + df = DataFrame(data=data) + self.assertIsNone(self.tspec_focus.best_focus) + self.assertIsNone(self.tspec_focus.fitted_model) + results = self.tspec_focus.get_best_focus(df=df, x_axis_size=2000) + self.assertIsNone(results) + def test_detect_sources(self): ccd = CCDData.read(self.test_file_name, unit='adu') self.tspec_focus.show_mask = True From 34fbe370faf931ed9b55837eb97d4c5f95c2c07f Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Mon, 24 Apr 2023 22:30:30 -0400 Subject: [PATCH 10/11] Updated docs to specify that it could return None --- triplespec_focus/triplespec_focus.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/triplespec_focus/triplespec_focus.py b/triplespec_focus/triplespec_focus.py index ed34e92..c4aaa02 100644 --- a/triplespec_focus/triplespec_focus.py +++ b/triplespec_focus/triplespec_focus.py @@ -350,7 +350,8 @@ def get_best_focus(self, df: DataFrame, x_axis_size: int = 2000) -> List[np.ndar x_axis_size (int): Size of the x-axis used to sample the fitted model. Is not an interpolation size. Returns: - A list with the x-axis and the sampled data using the fitted model + A list with the x-axis and the sampled data using the fitted model, None if it is not possible to find the + focus. """ focus_start = df['focus'].min() From 3876ddaadc6670ae1d2b2b9a8b9fa266e78a0551 Mon Sep 17 00:00:00 2001 From: Simon Torres Date: Tue, 25 Apr 2023 10:49:50 -0400 Subject: [PATCH 11/11] Added api docs files --- docs/source/modules.rst | 7 +++++ docs/source/triplespec_focus.rst | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 docs/source/modules.rst create mode 100644 docs/source/triplespec_focus.rst diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..838ffcb --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +triplespec_focus +================ + +.. toctree:: + :maxdepth: 4 + + triplespec_focus diff --git a/docs/source/triplespec_focus.rst b/docs/source/triplespec_focus.rst new file mode 100644 index 0000000..4472308 --- /dev/null +++ b/docs/source/triplespec_focus.rst @@ -0,0 +1,45 @@ +triplespec\_focus package +========================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + triplespec_focus.tests + +Submodules +---------- + +triplespec\_focus.entrypoints module +------------------------------------ + +.. automodule:: triplespec_focus.entrypoints + :members: + :undoc-members: + :show-inheritance: + +triplespec\_focus.triplespec\_focus module +------------------------------------------ + +.. automodule:: triplespec_focus.triplespec_focus + :members: + :undoc-members: + :show-inheritance: + +triplespec\_focus.utils module +------------------------------ + +.. automodule:: triplespec_focus.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: triplespec_focus + :members: + :undoc-members: + :show-inheritance: