Merge pull request #202 from askap-vast/v2.0-rc.2

v2.0-rc.2

FINDSOURCES.md

@@ -138,6 +138,10 @@ optional arguments:
   --lc-min-detections LC_MIN_DETECTIONS
                         Minimum number of times a source must be detected (default: 0)
   --lc-mjd              Plot lightcurve in MJD rather than datetime. (default: False)
+  --lc-start-date LC_START_DATE
+                        Plot lightcurve in days from some start date, formatted
+                        as YYYY-MM-DD HH:MM:SS or any other form that is
+                        accepted by pd.to_datetime() (default: None)
   --lc-grid             Turn on the 'grid' in the lightcurve plot. (default: False)
   --lc-yaxis-start {auto,0}
                         Define where the y axis on the lightcurve plot starts from. 'auto' will let

bin/find_sources.py

@@ -263,6 +263,14 @@ def parse_args():
         '--lc-mjd',
         action="store_true",
         help='Plot lightcurve in MJD rather than datetime.')
+    parser.add_argument(
+        '--lc-start-date',
+        type=str,
+        help=(
+            "Plot lightcurve in days from some start date, "
+            "formatted as YYYY-MM-DD HH:MM:SS or any other form "
+            "that is accepted by pd.to_datetime()"
+        ))
     parser.add_argument(
         '--lc-grid',
         action="store_true",
@@ -371,6 +379,25 @@ def check_output_directory(args):
         )
         sys.exit()
 
+    if args.lc_use_forced_for_limits or args.lc_use_forced_for_all:
+        if not args.forced_fits:
+            logger.error(
+                "Forced fits requested for lightcurve, "
+                "please use --forced-fits flag and try again."
+            )
+            sys.exit()
+
+    if args.lc_start_date:
+        try:
+            args.lc_start_date = pd.to_datetime(args.lc_start_date)
+        except Exception as e:
+            logger.error(
+                "Invalid lightcurve start date, "
+                "please check input and try again"
+            )
+            logger.error("Error: %s", e)
+            sys.exit()
+
     output_ok = check_output_directory(args)
 
     if not output_ok:
@@ -477,6 +504,7 @@ def check_output_directory(args):
                 lc_min_points=args.lc_min_points,
                 lc_min_detections=args.lc_min_detections,
                 lc_mjd=args.lc_mjd,
+                lc_start_date=args.lc_start_date,
                 lc_grid=args.lc_grid,
                 lc_yaxis_start=args.lc_yaxis_start,
                 lc_peak_flux=(not args.lc_use_int_flux),

vasttools/pipeline.py

@@ -586,12 +586,10 @@ def check_for_planets(self):
         ap = ALLOWED_PLANETS
 
         planets_df = self.images.loc[:, [
-            'id',
             'datetime',
             'duration',
             'centre_ra',
             'centre_dec',
-            'xtr_radius'
         ]].rename(
             columns={
                 'datetime': 'DATEOBS',

vasttools/query.py

@@ -22,6 +22,7 @@
 import signal
 import numexpr
 import gc
+import time
 
 from multiprocessing import Pool, cpu_count
 from multiprocessing_logging import install_mp_handler
@@ -410,6 +411,7 @@ def _gen_all_source_products(
         lc_min_points=2,
         lc_min_detections=1,
         lc_mjd=False,
+        lc_start_date=None,
         lc_grid=False,
         lc_yaxis_start="auto",
         lc_peak_flux=True,
@@ -483,6 +485,9 @@ def _gen_all_source_products(
         :type lc_min_detections: int, optional
         :param lc_mjd: Use MJD for lightcurve x-axis, defaults to `False`
         :type lc_mjd: bool, optional
+        :param lc_start_date: Plot lightcurve in days from start date, \
+        defaults to None
+        :type lc_start_date: pandas datetime, optional
         :param lc_grid: Include grid on lightcurve plot, defaults to `False`
         :type lc_grid: bool, optional
         :param lc_yaxis_start: Start the lightcurve y-axis at 0 ("0") or use \
@@ -551,6 +556,7 @@ def _gen_all_source_products(
             lc_min_points=lc_min_points,
             lc_min_detections=lc_min_detections,
             lc_mjd=lc_mjd,
+            lc_start_date=lc_start_date,
             lc_grid=lc_grid,
             lc_yaxis_start=lc_yaxis_start,
             lc_peak_flux=lc_peak_flux,
@@ -587,10 +593,10 @@ def _gen_all_source_products(
             )
             workers.terminate()
             sys.exit()
-        else:
-            self.logger.debug("Normal termination")
+        finally:
+            self.logger.debug("Closing workers.")
             workers.close()
-            # workers.join()
+            workers.join()
 
     def _produce_source_products(
         self,
@@ -617,6 +623,7 @@ def _produce_source_products(
         lc_min_points=2,
         lc_min_detections=1,
         lc_mjd=False,
+        lc_start_date=None,
         lc_grid=False,
         lc_yaxis_start="auto",
         lc_peak_flux=True,
@@ -689,6 +696,9 @@ def _produce_source_products(
         :type lc_min_detections: int, optional
         :param lc_mjd: Use MJD for lightcurve x-axis, defaults to `False`
         :type lc_mjd: bool, optional
+        :param lc_start_date: Plot lightcurve in days from start date, \
+        defaults to None
+        :type lc_start_date: pandas datetime, optional
         :param lc_grid: Include grid on lightcurve plot, defaults to `False`
         :type lc_grid: bool, optional
         :param lc_yaxis_start: Start the lightcurve y-axis at 0 ("0") or use \
@@ -750,6 +760,7 @@ def _produce_source_products(
                 min_points=lc_min_points,
                 min_detections=lc_min_detections,
                 mjd=lc_mjd,
+                start_date=lc_start_date,
                 grid=lc_grid,
                 yaxis_start=lc_yaxis_start,
                 peak_flux=lc_peak_flux,
@@ -763,6 +774,9 @@ def _produce_source_products(
         if measurements:
             source.write_measurements(simple=measurements_simple)
 
+        # attempt to avoid join hangs
+        time.sleep(0.2)
+
         return
 
     def _summary_log(self):
@@ -1629,7 +1643,10 @@ def find_fields(self):
 
             self.fields_df[
                 ['epoch', 'field', 'sbid', 'dateobs']
-            ] = self.fields_df.field_per_epoch.apply(pd.Series)
+            ] = pd.DataFrame(
+                self.fields_df['field_per_epoch'].tolist(),
+                index=self.fields_df.index
+            )
 
             to_drop = [
                 'field_per_epoch',

vasttools/source.py

@@ -112,7 +112,7 @@ class Source:
 
     plot_lightcurve(
         sigma_thresh=5, figsize=(8, 4), min_points=2,
-        min_detections=0, mjd=False, grid=False,
+        min_detections=0, mjd=False, start_date=None, grid=False,
         yaxis_start="auto", peak_flux=True, save=False,
         outfile=None, use_forced_for_limits=False,
         use_forced_for_all=False, hide_legend=False
@@ -417,9 +417,10 @@ def write_measurements(self, simple=False, outfile=None):
 
     def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
                         min_points=2, min_detections=0, mjd=False,
-                        grid=False, yaxis_start="auto", peak_flux=True,
-                        save=False, outfile=None, use_forced_for_limits=False,
-                        use_forced_for_all=False, hide_legend=False):
+                        start_date=None, grid=False, yaxis_start="auto",
+                        peak_flux=True, save=False, outfile=None,
+                        use_forced_for_limits=False, use_forced_for_all=False,
+                        hide_legend=False):
         '''
         Plot source lightcurves and save to file
 
@@ -435,6 +436,8 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
         :type min_detections: float, optional
         :param mjd: Plot x-axis in MJD rather than datetime, defaults to False
         :type mjd: bool, optional
+        :param start_date: Plot in days from start date, defaults to None
+        :type start_date: pandas datetime, optional
         :param grid: Turn on matplotlib grid, defaults to False
         :type grid: bool, optional
         :param yaxis_start: Define where the y-axis begins from, either 'auto'
@@ -500,6 +503,8 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
         plot_dates = measurements['dateobs']
         if mjd:
             plot_dates = Time(plot_dates.to_numpy()).mjd
+        elif start_date:
+            plot_dates = (plot_dates-start_date)/pd.Timedelta(1, unit='d')
 
         fig = plt.figure(figsize=figsize)
         ax = fig.add_subplot(111)
@@ -542,15 +547,15 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
             uplims = False
             sigma_thresh = 1.0
             label = 'Forced'
-            markerfacecolor = 'k'
+            markerfacecolor = 'w'
         else:
             if use_forced_for_limits:
                 value_col = 'f_flux_peak'
                 err_value_col = 'f_flux_peak_err'
                 uplims = False
                 marker = "D"
                 sigma_thresh = 1.0
-                markerfacecolor = 'k'
+                markerfacecolor = 'w'
                 label = "Forced"
             else:
                 value_col = err_value_col = 'rms_image'
@@ -595,7 +600,7 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
 
         if use_forced_for_all:
             marker = "D"
-            markerfacecolor = 'k'
+            markerfacecolor = 'w'
             label = 'Forced'
         else:
             marker = 'o'
@@ -624,6 +629,8 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
 
         if mjd:
             ax.set_xlabel('Date (MJD)')
+        elif start_date:
+            ax.set_xlabel('Days since {}'.format(start_date))
         else:
             fig.autofmt_xdate()
             ax.set_xlabel('Date')
@@ -646,7 +653,7 @@ def plot_lightcurve(self, sigma_thresh=5, figsize=(8, 4),
                 ))
 
             elif not outfile.endswith(".png"):
-                outname += ".png"
+                outfile += ".png"
 
             if self.outdir != ".":
                 outfile = os.path.join(
@@ -1173,7 +1180,7 @@ def save_all_png_cutouts(
         percentile=99.9,
         zscale=False,
         contrast=0.2,
-        islands=True,
+        no_islands=True,
         no_colorbar=False,
         crossmatch_overlay=False,
         hide_beam=False,
@@ -1250,7 +1257,7 @@ def save_all_png_cutouts(
                 zscale,
                 contrast,
                 None,
-                islands,
+                no_islands,
                 "Source",
                 no_colorbar,
                 None,

vasttools/utils.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import logging
 import logging.handlers
 import logging.config