Skip to content

Commit

Permalink
Changed the variable names for the general "biggest source" and "bigg…
Browse files Browse the repository at this point in the history 
…est source fraction"

limits which were misleading, and added a new variable name to accommodate the RickerWavelet
biggest source limit.
   OLD                       NEW
   _rw2d_biggest_source  ==> _biggest_source_limit
   _rw2d_source_fraction ==> _biggest_source_fraction_limit
    new                  ==> _rw_biggest_source_limit

These changes required updating the catalog parameter JSON files, the code which reads
the parameter values, and parameter names in segmentation source support functions.

Clarified informational log messages when reporting the configuration limits in use for
the Segmentation catalog.  Moved the computation of the ratio between the "Gaussian
biggest source" and the "Rickerwavlet biggest source" to be computed, as well as applied,
for Round 2. Added more informational messages to keep the user better informed.  Changed
some limit values (though more changes will be needed for other detectors):

ACS/WFC ONLY
   negative_percent: 15.0 ==> 20.0
   rw2d_nsigma: 10.0  ==> 40.0
  • Loading branch information
@mdlpstsci
mdlpstsci committed Jul 31, 2023
1 parent b02b484 commit 64cfcbb
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 42 deletions.
67 changes: 39 additions & 28 deletions drizzlepac/haputils/catalog_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1516,8 +1516,9 @@ def __init__(self, image, param_dict, param_dict_qc, diagnostic_mode, tp_sources
self._nsigma = self.param_dict["sourcex"]["segm_nsigma"]
self._rw2d_size = self.param_dict["sourcex"]["rw2d_size"]
self._rw2d_nsigma = self.param_dict["sourcex"]["rw2d_nsigma"]
self._rw2d_biggest_source = self.param_dict["sourcex"]["rw2d_biggest_source"]
self._rw2d_source_fraction = self.param_dict["sourcex"]["rw2d_source_fraction"]
self._biggest_source_limit = self.param_dict["sourcex"]["biggest_source_limit"]
self._biggest_fraction_limit = self.param_dict["sourcex"]["biggest_fraction_limit"]
self._rw_biggest_source_limit = self.param_dict["sourcex"]["rw_biggest_source_limit"]
self._bs_deblend_limit = self.param_dict["sourcex"]["biggest_source_deblend_limit"]
self._sf_deblend_limit = self.param_dict["sourcex"]["source_fraction_deblend_limit"]
self._ratio_bigsource_limit = self.param_dict["sourcex"]["ratio_bigsource_limit"]
Expand Down Expand Up @@ -1584,10 +1585,13 @@ def identify_sources(self, **pars):
log.info("contrast (frac. flux for peak to be separate object, 0=max. deblend, 1=no deblend): {}".format(self._contrast))
log.info("RickerWavelet nsigma (threshold = nsigma * background_rms): {}".format(self._rw2d_nsigma))
log.info("RickerWavelet kernel X- and Y-dimension: {}".format(self._rw2d_size))
log.info("Percentage limit on biggest source (criterion for RickerWavelet kernel): {}".format(100.0 * self._rw2d_biggest_source))
log.info("Percentage limit on source fraction over the image (criterion for RickerWavelet kernel): {}".format(100.0 * self._rw2d_source_fraction))
log.info("Percentage limit on biggest source deblending limit: {}".format(100.0 * self._bs_deblend_limit))
log.info("Percentage limit on source fraction deblending limit: {}".format(100.0 * self._sf_deblend_limit))
log.info("Maximum % limit on acceptable biggest source: {}".format(100.0 * self._biggest_source_limit))
log.info("Maximum % limit on acceptable source fraction over the image: {}".format(100.0 * self._biggest_fraction_limit))
log.info("Maximum % limit on acceptable RickerWavelet biggest source: {}".format(100.0 * self._rw_biggest_source_limit))
log.info("Minimum acceptable Gaussian vs RickerWavelet kernel big source ratio limit (overlapping PSFs): {}".format(self._ratio_bigsource_limit))
log.info("Maximum % limit on acceptable biggest source (RW deblend retry): {}".format(100.0 * self._bs_deblend_limit))
log.info("Maximum % limit on acceptable source fraction (RW deblend retry): {}".format(100.0 * self._sf_deblend_limit))
log.info("Ignore prohibitively big sources identified by mimimum ratio of the areas (biggest/biggest-1): {}".format(self._ratio_bigsource_deblend_limit))
log.info("Scaling parameter of the Kron radius: {}".format(self._kron_scaling_radius))
log.info("Kron minimum circular radius: {}".format(self._kron_minimum_radius))
log.info("")
Expand Down Expand Up @@ -1617,6 +1621,7 @@ def identify_sources(self, **pars):
# Round 1
ncount = 0
log.info("")
log.info("ROUND 1")
log.info("Using Custom kernel or Gaussian to generate a segmentation map.")
g_segm_img, g_is_big_crowded, g_bs, g_sf = self.detect_and_eval_segments(imgarr,
g2d_kernel,
Expand All @@ -1626,8 +1631,8 @@ def identify_sources(self, **pars):
self.image.bkg_background_ra,
self.image.bkg_rms_ra,
check_big_island_only=False,
rw2d_biggest_source=self._rw2d_biggest_source,
rw2d_source_fraction=self._rw2d_source_fraction)
biggest_source=self._biggest_source_limit,
source_fraction=self._biggest_fraction_limit)
segm_img_orig = copy.deepcopy(g_segm_img)

# If the science field via the segmentation map is deemed crowded or has big sources/islands, compute the
Expand Down Expand Up @@ -1658,14 +1663,8 @@ def identify_sources(self, **pars):
self.image.bkg_background_ra,
self.image.bkg_rms_ra,
check_big_island_only=True,
rw2d_biggest_source=self._rw2d_biggest_source,
rw2d_source_fraction=self._rw2d_source_fraction)

# Compute the ratio of big sources/islands using Custom/Gaussian kernel vs Rickerwavelet kernel
# This value can be used as a discriminant between overlapping point sources and nebulousity fields
ratio_cg2rw_bigsource = 3.0
if rw_bs > 0.0:
ratio_cg2rw_bigsource = g_bs / rw_bs
biggest_source=self._rw_biggest_source_limit,
source_fraction=self._biggest_fraction_limit)

# Check if the RickerWavelet segmentation image still seems to be problematic
if rw_is_big_crowded and rw_segm_img:
Expand Down Expand Up @@ -1731,6 +1730,7 @@ def identify_sources(self, **pars):
# Round 2
ncount += 1
log.info("")
log.info("ROUND 2")
log.info("With alternate background...using Custom/Gaussian kernel to generate a segmentation map.")
del g_segm_img
g_segm_img, g_is_big_crowded, g_bs, g_sf = self.detect_and_eval_segments(imgarr,
Expand All @@ -1741,8 +1741,8 @@ def identify_sources(self, **pars):
self.image.bkg_background_ra,
self.image.bkg_rms_ra,
check_big_island_only=False,
rw2d_biggest_source=self._rw2d_biggest_source,
rw2d_source_fraction=self._rw2d_source_fraction)
biggest_source=self._biggest_source_limit,
source_fraction=self._biggest_fraction_limit)

# Check again for big sources/islands or a large source fraction
if g_is_big_crowded:
Expand All @@ -1763,8 +1763,14 @@ def identify_sources(self, **pars):
self.image.bkg_background_ra,
self.image.bkg_rms_ra,
check_big_island_only=False,
rw2d_biggest_source=self._bs_deblend_limit,
rw2d_source_fraction=self._sf_deblend_limit)
biggest_source=self._bs_deblend_limit,
source_fraction=self._sf_deblend_limit)

# Compute the ratio of big sources/islands using Custom/Gaussian kernel vs Rickerwavelet kernel
# This value can be used as a discriminant between overlapping point sources and nebulousity fields
ratio_cg2rw_bigsource = 3.0
if rw_bs > 0.0:
ratio_cg2rw_bigsource = g_bs / rw_bs

Check warning on line 1773 in drizzlepac/haputils/catalog_utils.py

View check run for this annotation

Codecov / codecov/patch

drizzlepac/haputils/catalog_utils.py#L1771-L1773 

Added lines #L1771 - L1773 were not covered by tests

# Last chance - The larger "deblend" limits were used in this last detection
# attempt based upon the the statistics of processing lots of data - looking
Expand Down Expand Up @@ -1923,7 +1929,7 @@ def identify_sources(self, **pars):

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsigma_above_bkg, background_img, background_rms, check_big_island_only=False, rw2d_biggest_source=0.015, rw2d_source_fraction=0.075):
def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsigma_above_bkg, background_img, background_rms, check_big_island_only=False, biggest_source=0.015, source_fraction=0.075):

# Compute the threshold to use for source detection
threshold = self.compute_threshold(nsigma_above_bkg, background_img, background_rms)
Expand Down Expand Up @@ -1962,8 +1968,8 @@ def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsig
is_big_crowded, big_island, source_fraction = self._evaluate_segmentation_image(segm_img,
imgarr,
big_island_only=check_big_island_only,
max_biggest_source=rw2d_biggest_source,
max_source_fraction=rw2d_source_fraction)
max_biggest_source=biggest_source,
max_source_fraction=source_fraction)

return segm_img, is_big_crowded, big_island, source_fraction

Expand Down Expand Up @@ -2053,10 +2059,12 @@ def detect_segments(self, imgarr, threshold, ncount, filter_kernel=None, source_
img_bkg_sub = np.clip(img_bkg_sub, 0, img_bkg_sub.max())
# Now set threshold to 0, since it has already been applied to the input data array
thresh0 = np.zeros_like(img_bkg_sub)
log.info("Setting threshold to zero as data has been background subtracted.")
else:
# Use inputs provided with the kernel
img_bkg_sub = imgarr
thresh0 = threshold
log.info("Using computed threshold which includes [bkg + nsigma * rms].")

# Note: SExtractor has "connectivity=8" which is the default for detect_sources().
segm_img = None
Expand Down Expand Up @@ -2689,8 +2697,7 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
max_biggest_source : float, optional
Maximum limit on the single largest detected "source".
max_source_fraction : float, optional
Maximum limit on the fraction of pixels identified as part of a "source".
max_source_fraction : float, optional Maximum limit on the fraction of pixels identified as part of a "source".
Returns
-------
Expand Down Expand Up @@ -2728,7 +2735,6 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
n, binedges = np.histogram(segm_img.data, range=(1, nbins))
real_pixels = (image_data != 0).sum()
biggest_source = n.max()/float(real_pixels)
log.info("Biggest_source: %f", biggest_source)

# Compute which segments are larger than the kernel.
deb_limit = self.kernel.size
Expand All @@ -2745,8 +2751,10 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
# and is_poor_quality should be set to True. The is_poor_quality is only an indicator that
# a different kernel type or background computation could be tried for improved results.
if biggest_source > max_biggest_source:
log.info("Biggest source %.4f percent exceeds %f percent of the image", (100.0*biggest_source), (100.0*max_biggest_source))
log.info("Biggest source %.4f percent exceeds %f percent of the image.", (100.0*biggest_source), (100.0*max_biggest_source))
is_poor_quality = True
else:
log.info("Biggest source %.4f percent is WITHIN THE ACCEPTABLE limit of %f percent.", (100.0*biggest_source), (100.0*max_biggest_source))

# Filter the big_segments array to remove the prohibitively large segments
if big_segments.size > 0:
Expand Down Expand Up @@ -2793,11 +2801,14 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
# big_island_only parameter allows control over whether the source_fraction should
# or should not be ignored.
source_fraction = n.sum()/float(real_pixels)
log.info("Source_fraction: %f", source_fraction)
if not big_island_only:
if source_fraction > max_source_fraction:
log.info("Total source fraction %.4f percent exceeds %f percent of the image.", (100.0*source_fraction), (100.0*max_source_fraction))
is_poor_quality = True
else:
log.info("Total source fraction %.4f percent is WITHIN THE ACCEPTABLE limit of %f percent.", (100.0*source_fraction), (100.0*max_source_fraction))
else:
log.info("Total source fraction %.4f percent is computed, but not set to be considered as a rejection criterion.", (100.0*source_fraction))

return is_poor_quality, biggest_source, source_fraction

Expand Down
5 changes: 3 additions & 2 deletions drizzlepac/pars/hap_pars/svm_parameters/acs/hrc/acs_hrc_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@
"border": 10,
"rw2d_size": 15,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.045,
"rw2d_source_fraction": 0.15,
"biggest_source_limit": 0.045,
"biggest_fraction_limit": 0.15,
"biggest_source_deblend_limit": 0.15,
"rw_biggest_source_limit": 0.05,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
"ratio_bigsource_deblend_limit": 10,
Expand Down
5 changes: 3 additions & 2 deletions drizzlepac/pars/hap_pars/svm_parameters/acs/sbc/acs_sbc_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
"border": 10,
"rw2d_size": 23,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.045,
"rw2d_source_fraction": 0.15,
"biggest_source_limit": 0.045,
"biggest_fraction_limit": 0.15,
"rw_biggest_source_limit": 0.05,
"biggest_source_deblend_limit": 0.15,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
Expand Down
9 changes: 5 additions & 4 deletions drizzlepac/pars/hap_pars/svm_parameters/acs/wfc/acs_wfc_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"flag_trim_value": 5,
"simple_bkg": false,
"zero_percent": 25.0,
"negative_percent": 15.0,
"negative_percent": 20.0,
"nsigma_clip": 3.0,
"maxiters": 3,
"bkg_skew_threshold": 0.5,
Expand All @@ -36,9 +36,10 @@
"contrast": 0.001,
"border": 10,
"rw2d_size": 15,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.015,
"rw2d_source_fraction": 0.075,
"rw2d_nsigma": 40.0,
"biggest_source_limit": 0.015,
"biggest_fraction_limit": 0.075,
"rw_biggest_source_limit": 0.05,
"biggest_source_deblend_limit": 0.15,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
Expand Down
5 changes: 3 additions & 2 deletions drizzlepac/pars/hap_pars/svm_parameters/wfc3/ir/wfc3_ir_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
"border": 10,
"rw2d_size": 11,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.045,
"rw2d_source_fraction": 0.15,
"biggest_source_limit": 0.045,
"biggest_fraction_limit": 0.15,
"rw_biggest_source_limit": 0.05,
"biggest_source_deblend_limit": 0.15,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
Expand Down
5 changes: 3 additions & 2 deletions drizzlepac/pars/hap_pars/svm_parameters/wfc3/uvis/wfc3_uvis_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
"border": 10,
"rw2d_size": 15,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.015,
"rw2d_source_fraction": 0.075,
"biggest_source_limit": 0.015,
"biggest_fraction_limit": 0.075,
"rw_biggest_source_limit": 0.05,
"biggest_source_deblend_limit": 0.15,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
Expand Down
5 changes: 3 additions & 2 deletions drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
"border": 10,
"rw2d_size": 11,
"rw2d_nsigma": 10.0,
"rw2d_biggest_source": 0.045,
"rw2d_source_fraction": 0.15,
"biggest_source_limit": 0.045,
"biggest_fraction_limit": 0.15,
"rw_biggest_source_limit": 0.05,
"biggest_source_deblend_limit": 0.15,
"source_fraction_deblend_limit": 0.20,
"ratio_bigsource_limit": 2,
Expand Down

0 comments on commit 64cfcbb

Please sign in to comment.