diff --git a/CHANGES.md b/CHANGES.md index c45968953..2444377e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +0.21.1 (2019-03-28) +------------------- +- Added a quality control stage that checks to ensure images are not + primarily comprised of 0s, which indicates a camera failure + 0.21.0 (2019-03-25) ------------------- - Significant refactor to the pipeline context and settings files. We have now diff --git a/banzai/qc/__init__.py b/banzai/qc/__init__.py index 1d1b36616..c2f5f4758 100644 --- a/banzai/qc/__init__.py +++ b/banzai/qc/__init__.py @@ -3,6 +3,7 @@ from banzai.qc.sinistro_1000s import ThousandsTest from banzai.qc.pattern_noise import PatternNoiseDetector from banzai.qc.header_checker import HeaderSanity +from banzai.qc.zeros import ZerosTest __all__ = ['SaturationTest', 'PointingTest', 'ThousandsTest', - 'PatternNoiseDetector', 'HeaderSanity'] + 'PatternNoiseDetector', 'HeaderSanity', 'ZerosTest'] diff --git a/banzai/qc/zeros.py b/banzai/qc/zeros.py new file mode 100644 index 000000000..127c98725 --- /dev/null +++ b/banzai/qc/zeros.py @@ -0,0 +1,44 @@ +import logging + +import numpy as np + +from banzai.stages import Stage +from banzai.utils import qc + +logger = logging.getLogger(__name__) + + +class ZerosTest(Stage): + """ + Reject any images that have ZEROS_THRESHOLD or more of their pixels in a single amp exactly equal to 0. + + Notes + ===== + Sometimes when a camera fails, all pixels have a value of 0. + """ + ZEROS_THRESHOLD = 0.95 + + def __init__(self, runtime_context): + super(ZerosTest, self).__init__(runtime_context) + + def do_stage(self, image): + fraction_0s_list = [] + for i_amp in range(image.get_n_amps()): + npixels = np.product(image.data[i_amp].shape) + fraction_0s_list.append(float(np.sum(image.data[i_amp] == 0)) / npixels) + + has_0s_error = any([fraction_0s > self.ZEROS_THRESHOLD for fraction_0s in fraction_0s_list]) + logging_tags = {'FRAC0': fraction_0s_list, + 'threshold': self.ZEROS_THRESHOLD} + qc_results = {'zeros_test.failed': has_0s_error, + 'zeros_test.fraction': fraction_0s_list, + 'zeros_test.threshold': self.ZEROS_THRESHOLD} + if has_0s_error: + logger.error('Image is mostly 0s. Rejecting image', image=image, extra_tags=logging_tags) + qc_results['rejected'] = True + return None + else: + logger.info('Measuring fraction of 0s.', image=image, extra_tags=logging_tags) + qc.save_qc_results(self.runtime_context, qc_results, image) + + return image diff --git a/banzai/settings.py b/banzai/settings.py index f553f1ab6..c51941a67 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -24,6 +24,7 @@ def telescope_to_filename(image): ORDERED_STAGES = ['banzai.bpm.BPMUpdater', 'banzai.qc.HeaderSanity', 'banzai.qc.ThousandsTest', + 'banzai.qc.ZerosTest', 'banzai.qc.SaturationTest', 'banzai.bias.OverscanSubtractor', 'banzai.crosstalk.CrosstalkCorrector', diff --git a/banzai/tests/test_zeros_qc.py b/banzai/tests/test_zeros_qc.py new file mode 100644 index 000000000..1581c4bef --- /dev/null +++ b/banzai/tests/test_zeros_qc.py @@ -0,0 +1,71 @@ +import pytest +import numpy as np + +from banzai.tests.utils import FakeImage +from banzai.qc import ZerosTest + + +@pytest.fixture(scope='module') +def set_random_seed(): + np.random.seed(81232385) + + +def get_random_pixel_pairs(nx, ny, fraction): + return np.unravel_index(np.random.choice(range(nx * ny), size=int(fraction * nx * ny)), (ny, nx)) + + +def test_null_input_image(): + tester = ZerosTest(None) + image = tester.run(None) + assert image is None + + +def test_no_pixels_0_not_rejected(): + tester = ZerosTest(None) + image = tester.do_stage(FakeImage(image_multiplier=5)) + assert image is not None + + +def test_image_all_0s_rejected(): + tester = ZerosTest(None) + image = FakeImage() + image.data[:, :] = 0 + image = tester.do_stage(image) + assert image is None + + +def test_image_50_percent_0_not_rejected(set_random_seed): + tester = ZerosTest(None) + nx = 101 + ny = 103 + image = FakeImage(nx=nx, ny=ny) + random_pixels = get_random_pixel_pairs(nx, ny, 0.5) + for j,i in zip(random_pixels[0], random_pixels[1]): + image.data[i,i] = 0 + image = tester.do_stage(image) + assert image is not None + + +def test_image_99_percent_0_rejected(set_random_seed): + tester = ZerosTest(None) + nx = 101 + ny = 103 + image = FakeImage(nx=nx, ny=ny) + random_pixels = get_random_pixel_pairs(nx, ny, 0.99) + for j,i in zip(random_pixels[0], random_pixels[1]): + image.data[i] = 0 + image = tester.do_stage(image) + assert image is None + + +def test_single_amp_99_percent_0_rejected(set_random_seed): + tester = ZerosTest(None) + nx = 101 + ny = 103 + n_amps = 4 + image = FakeImage(nx=nx, ny=ny, n_amps=n_amps) + random_pixels = get_random_pixel_pairs(nx, ny, 0.99) + for j,i in zip(random_pixels[0], random_pixels[1]): + image.data[0][i] = 0 + image = tester.do_stage(image) + assert image is None diff --git a/setup.cfg b/setup.cfg index 7cb40f051..a8fb273fd 100755 --- a/setup.cfg +++ b/setup.cfg @@ -77,7 +77,7 @@ edit_on_github = True github_project = lcogt/banzai # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 0.21.0 +version = 0.21.1 [entry_points] banzai = banzai.main:main