From bfaaea520cec36aa3ff988eae64aa14f2a8032e3 Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 11:03:50 +0000 Subject: [PATCH 1/9] Fix #47: Shallower output folder structure --- src/correct_images/corrector.py | 146 +++++++++++++------------------- 1 file changed, 59 insertions(+), 87 deletions(-) diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 3d287c18..3c24dc34 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -8,7 +8,6 @@ import os import random -from datetime import datetime from pathlib import Path import imageio @@ -66,10 +65,9 @@ def __init__(self, force=False, camera=None, correct_config=None, path=None): # Members for folder paths self.output_dir_path = None - self.bayer_numpy_dir_path = None self.attenuation_parameters_folder = None - self.memmap_folder = None self.output_images_folder = None + self.suffix = None # Placeholders for process self.image_attenuation_parameters = None @@ -157,26 +155,13 @@ def __init__(self, force=False, camera=None, correct_config=None, path=None): # Define basic filepaths if self.correction_method == "colour_correction": - self.attenuation_params_filepath = ( - Path(self.attenuation_parameters_folder) - / "attenuation_parameters.npy" - ) - self.correction_gains_filepath = ( - Path(self.attenuation_parameters_folder) / "correction_gains.npy" - ) - self.corrected_mean_filepath = ( - Path(self.attenuation_parameters_folder) - / "image_corrected_mean.npy" - ) - self.corrected_std_filepath = ( - Path(self.attenuation_parameters_folder) / "image_corrected_std.npy" - ) - self.raw_mean_filepath = ( - Path(self.attenuation_parameters_folder) / "image_raw_mean.npy" - ) - self.raw_std_filepath = ( - Path(self.attenuation_parameters_folder) / "image_raw_std.npy" - ) + p = Path(self.attenuation_parameters_folder) + self.attenuation_params_filepath = (p / "attenuation_parameters.npy") + self.correction_gains_filepath = (p / "correction_gains.npy") + self.corrected_mean_filepath = (p / "image_corrected_mean.npy") + self.corrected_std_filepath = (p / "image_corrected_std.npy") + self.raw_mean_filepath = (p / "image_raw_mean.npy") + self.raw_std_filepath = (p / "image_raw_std.npy") # Define image loader # Use default loader @@ -277,82 +262,68 @@ def process(self): # numpy files, correction parameters and corrected output images def create_output_directories(self): """Handle the creation of output directories for each camera""" + self.output_dir_path = self.path_processed / "images" + self.output_dir_path /= self.camera_name - # create output directory path - image_path = Path(self.camera.image_list[0]).resolve() - image_parent_path = image_path.parent - output_dir_path = get_processed_folder(image_parent_path) - self.output_dir_path = output_dir_path / "attenuation_correction" - if not self.output_dir_path.exists(): - self.output_dir_path.mkdir(parents=True) - - if self.correction_method == "colour_correction": - # create path for parameters files - attenuation_parameters_folder_name = "params_" + self.camera.name - self.attenuation_parameters_folder = ( - self.output_dir_path / attenuation_parameters_folder_name - ) - if not self.attenuation_parameters_folder.exists(): - self.attenuation_parameters_folder.mkdir(parents=True) - - # create path for output images - output_images_folder_name = "developed_" + self.camera.name - self.output_images_folder = self.output_dir_path / output_images_folder_name - if not self.output_images_folder.exists(): - self.output_images_folder.mkdir(parents=True) + # Create output directories depending on the correction method + parameters_folder_str = "params_" + developed_folder_str = "developed_" - # create folder name for parameters based on correction method - sub_directory_name = "unknown_sub_directory_name" - output_folder_name = "unknown_output_folder_name" + subfolder_name = None if self.correction_method == "colour_correction": if self.distance_metric == "none": - sub_directory_name = "greyworld_corrected" + parameters_folder_str += "ps" + developed_folder_str += "ps" elif self.distance_metric == "altitude": - sub_directory_name = "altitude_corrected" + parameters_folder_str += "alt" + developed_folder_str += "alt" elif self.distance_metric == "depth_map": - sub_directory_name = "depth_map_corrected" - - output_folder_name = ( + parameters_folder_str += "dm" + developed_folder_str += "dm" + subfolder_name = ( "m" + str(int(self.brightness)) + "_std" + str(int(self.contrast)) ) - - # appending params path with sub directory and output folder - self.attenuation_parameters_folder = ( - self.attenuation_parameters_folder / sub_directory_name - ) - if not self.attenuation_parameters_folder.exists(): - self.attenuation_parameters_folder.mkdir(parents=True) - else: - dir_temp = self.attenuation_parameters_folder - file_list = list(dir_temp.glob("*.npy")) - if len(file_list) > 0: - if not self.force: - Console.quit( - "Parameters exist for current configuration.", - "Run parse with Force (-F flag)...", - ) - else: - Console.warn( - "Code will overwrite existing parameters for ", - "current configuration...", - ) elif self.correction_method == "manual_balance": - sub_directory_name = "manually_corrected" - temp1 = str(datetime.now()) - temp2 = temp1.split(":") - temp3 = temp2[0].split(" ") - temp4 = temp3[1] + temp2[1] - output_folder_name = "developed_" + temp4 - - # appending developed images path with sub directory and output folder - self.output_images_folder = ( - self.output_images_folder / sub_directory_name / output_folder_name - ) + parameters_folder_str += "man" + developed_folder_str += "man" + subfolder_name = ("g_" + + str(self.color_gain_matrix_rgb[0]) + "_" + + str(self.color_gain_matrix_rgb[4]) + "_" + + str(self.color_gain_matrix_rgb[8]) + "_s_" + + str(self.subtractors_rgb[0]) + "_" + + str(self.subtractors_rgb[1]) + "_" + + str(self.subtractors_rgb[2]) + ) + + # Accept suffixes for the output directories + if self.suffix: + parameters_folder_str += "_" + self.suffix + developed_folder_str += "_" + self.suffix + + self.attenuation_parameters_folder = self.output_dir_path / parameters_folder_str + self.output_images_folder = self.output_dir_path / developed_folder_str + self.output_images_folder /= subfolder_name + + if not self.attenuation_parameters_folder.exists(): + self.attenuation_parameters_folder.mkdir(parents=True) + else: + file_list = list(self.attenuation_parameters_folder.glob("*.npy")) + if len(file_list) > 0: + if not self.force: + Console.quit( + "Parameters exist for current configuration.", + "Run parse with Force (-F flag)...", + ) + else: + Console.warn( + "Code will overwrite existing parameters for ", + "current configuration...", + ) + if not self.output_images_folder.exists(): self.output_images_folder.mkdir(parents=True) else: - dir_temp = self.output_images_folder - file_list = list(dir_temp.glob("*.*")) + file_list = list(self.output_images_folder.glob("*.*")) if len(file_list) > 0: if not self.force: Console.quit( @@ -365,6 +336,7 @@ def create_output_directories(self): "current configuration...", ) + def get_camera_idx(self): idx = [ i From 8c9695b1af4ddf15ab78afe47eabad9708193bda Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 11:31:51 +0000 Subject: [PATCH 2/9] Fix #53: Allow use of suffixes --- src/correct_images/correct_images.py | 51 +++++++++++++++++++++++----- src/correct_images/corrector.py | 7 ++-- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/correct_images/correct_images.py b/src/correct_images/correct_images.py index a51ce5f1..5f2aed10 100644 --- a/src/correct_images/correct_images.py +++ b/src/correct_images/correct_images.py @@ -9,6 +9,7 @@ import argparse import os +import string import sys import time from pathlib import Path @@ -52,6 +53,12 @@ def main(args=None): action="store_true", help="Force overwrite if correction parameters already exist.", ) + subparser_correct.add_argument( + "--suffix", + dest="suffix", + default="", + help="Expected suffix for correct_images configuration and output folders.", + ) subparser_correct.set_defaults(func=call_correct) # subparser parse @@ -66,6 +73,12 @@ def main(args=None): action="store_true", help="Force overwrite if correction parameters already exist.", ) + subparser_parse.add_argument( + "--suffix", + dest="suffix", + default="", + help="Expected suffix for correct_images configuration and output folders.", + ) subparser_parse.set_defaults(func=call_parse) # subparser process @@ -80,6 +93,12 @@ def main(args=None): action="store_true", help="Force overwrite if correction parameters already exist.", ) + subparser_process.add_argument( + "--suffix", + dest="suffix", + default="", + help="Expected suffix for correct_images configuration and output folders.", + ) subparser_process.set_defaults(func=call_process) # subparser rescale image @@ -87,6 +106,12 @@ def main(args=None): "rescale", help="Rescale processed images" ) subparser_rescale.add_argument("path", help="Path to raw folder") + subparser_rescale.add_argument( + "--suffix", + dest="suffix", + default="", + help="Expected suffix for correct_images configuration and output folders.", + ) subparser_rescale.set_defaults(func=call_rescale) if len(sys.argv) == 1 and args is None: @@ -99,7 +124,13 @@ def main(args=None): args.func(args) else: args = parser.parse_args() - args.func(args) + + # Check suffix is only text, digits, dash and underscores + allowed_chars = string.ascii_letters+ "-" + "_" + string.digits + if all([c in allowed_chars for c in args.suffix]): + args.func(args) + else: + Console.error("Suffix must only contain letters, digits, dash and underscores") def call_parse(args): @@ -120,7 +151,7 @@ def call_parse(args): / ("log/" + time_string + "_correct_images_parse.log") ) - correct_config, camerasystem = load_configuration_and_camera_system(path) + correct_config, camerasystem = load_configuration_and_camera_system(path, args.suffix) for camera in camerasystem.cameras: Console.info("Parsing for camera", camera.name) @@ -129,7 +160,7 @@ def call_parse(args): Console.info("No images found for the camera at the path provided...") continue else: - corrector = Corrector(args.force, camera, correct_config, path) + corrector = Corrector(args.force, args.suffix, camera, correct_config, path) if corrector.camera_found: corrector.parse() @@ -157,7 +188,7 @@ def call_process(args): / ("log/" + time_string + "_correct_images_process.log") ) - correct_config, camerasystem = load_configuration_and_camera_system(path) + correct_config, camerasystem = load_configuration_and_camera_system(path, args.suffix) for camera in camerasystem.cameras: Console.info("Processing for camera", camera.name) @@ -166,7 +197,7 @@ def call_process(args): Console.info("No images found for the camera at the path provided...") continue else: - corrector = Corrector(args.force, camera, correct_config, path) + corrector = Corrector(args.force, args.suffix, camera, correct_config, path) if corrector.camera_found: corrector.process() Console.info("Process completed for all cameras...") @@ -193,7 +224,7 @@ def call_rescale(args): / ("log/" + time_string + "_correct_images_rescale.log") ) - correct_config, camerasystem = load_configuration_and_camera_system(path) + correct_config, camerasystem = load_configuration_and_camera_system(path, args.suffix) # install freeimage plugins if not installed imageio.plugins.freeimage.download() @@ -206,7 +237,7 @@ def call_rescale(args): Console.info("Rescaling completed for all cameras ...") -def load_configuration_and_camera_system(path): +def load_configuration_and_camera_system(path, suffix=None): """Generate correct_config and camera system objects from input config yaml files @@ -327,7 +358,11 @@ def load_configuration_and_camera_system(path): ) # check for correct_config yaml path - path_correct_images = path_config_folder / "correct_images.yaml" + path_correct_images = None + if suffix == "" or suffix is None: + path_correct_images = path_config_folder / "correct_images.yaml" + else: + path_correct_images = path_config_folder / ("correct_images_" + suffix + ".yaml") if path_correct_images.exists(): Console.info( "Configuration file correct_images.yaml file found at", path_correct_images, diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 3c24dc34..128c664f 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -40,7 +40,7 @@ class Corrector: - def __init__(self, force=False, camera=None, correct_config=None, path=None): + def __init__(self, force=False, suffix=None, camera=None, correct_config=None, path=None): """Constructor for the Corrector class Parameters @@ -67,7 +67,10 @@ def __init__(self, force=False, camera=None, correct_config=None, path=None): self.output_dir_path = None self.attenuation_parameters_folder = None self.output_images_folder = None - self.suffix = None + if suffix is not None and suffix != "": + self.suffix = suffix + else: + self.suffix = None # Placeholders for process self.image_attenuation_parameters = None From 2c7fefc9ae8354821931d6fe309c2f00e7ff5bcb Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 11:59:22 +0000 Subject: [PATCH 3/9] Fix consistent folder naming --- src/correct_images/corrector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 128c664f..6bc375d2 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -265,7 +265,7 @@ def process(self): # numpy files, correction parameters and corrected output images def create_output_directories(self): """Handle the creation of output directories for each camera""" - self.output_dir_path = self.path_processed / "images" + self.output_dir_path = self.path_processed / "image" self.output_dir_path /= self.camera_name # Create output directories depending on the correction method From bbf21887f350fd2bb0c1e28dea54f9aab11a890e Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 12:14:20 +0000 Subject: [PATCH 4/9] Fix #46: Populate default values in all cases --- src/correct_images/correct_images.py | 147 +++++++++++++-------------- 1 file changed, 71 insertions(+), 76 deletions(-) diff --git a/src/correct_images/correct_images.py b/src/correct_images/correct_images.py index 5f2aed10..299d5ca6 100644 --- a/src/correct_images/correct_images.py +++ b/src/correct_images/correct_images.py @@ -260,93 +260,88 @@ def load_configuration_and_camera_system(path, suffix=None): else: Console.quit("File mission.yaml file not found at", path_raw_folder) - # load mission parameters - mission = Mission(path_mission) + acfr_std_camera_file = "auv_nav/default_yaml/ts1/SSK17-01/camera.yaml" + sx3_camera_file = "auv_nav/default_yaml/ae2000/YK17-23C/camera.yaml" + biocam_camera_file = "auv_nav/default_yaml/as6/DY109/camera.yaml" + biocam4000_15c_camera_file = "auv_nav/default_yaml/alr/jc220/camera.yaml" + hybis_camera_file = "auv_nav/default_yaml/hybis/camera.yaml" + ntnu_camera_file = "auv_nav/default_yaml/ntnu_stereo/tautra21/camera.yaml" + rosbag_extracted_camera_file = ( + "auv_nav/default_yaml/rosbag/grassmap/camera.yaml" + ) - # resolve path to camera.yaml file - temp_path = path_raw_folder / "camera.yaml" + acfr_std_correct_config_file = ( + "correct_images/default_yaml/acfr/correct_images.yaml" + ) + sx3_std_correct_config_file = ( + "correct_images/default_yaml/sx3/correct_images.yaml" + ) + biocam_std_correct_config_file = ( + "correct_images/default_yaml/biocam/correct_images.yaml" + ) + biocam4000_15c_std_correct_config_file = ( + "correct_images/default_yaml/biocam4000_15c/correct_images.yaml" + ) + hybis_std_correct_config_file = ( + "correct_images/default_yaml/hybis/correct_images.yaml" + ) + ntnu_std_correct_config_file = ( + "correct_images/default_yaml/ntnu_stereo/correct_images.yaml" + ) + rosbag_extracted_images_std_correct_config_file = ( + "correct_images/default_yaml/rosbag_extracted_images/correct_images.yaml" + ) - default_file_path_correct_config = None camera_yaml_path = None + default_file_path_correct_config = None - if not temp_path.exists(): - Console.info( - "Not found camera.yaml file in /raw folder...Using default ", - "camera.yaml file...", - ) - # find out default yaml paths - root = Path(__file__).resolve().parents[1] - - acfr_std_camera_file = "auv_nav/default_yaml/ts1/SSK17-01/camera.yaml" - sx3_camera_file = "auv_nav/default_yaml/ae2000/YK17-23C/camera.yaml" - biocam_camera_file = "auv_nav/default_yaml/as6/DY109/camera.yaml" - biocam4000_15c_camera_file = "auv_nav/default_yaml/alr/jc220/camera.yaml" - hybis_camera_file = "auv_nav/default_yaml/hybis/camera.yaml" - ntnu_camera_file = "auv_nav/default_yaml/ntnu_stereo/tautra21/camera.yaml" - rosbag_extracted_camera_file = ( - "auv_nav/default_yaml/rosbag/grassmap/camera.yaml" - ) + # load mission parameters + mission = Mission(path_mission) - acfr_std_correct_config_file = ( - "correct_images/default_yaml/acfr/correct_images.yaml" - ) - sx3_std_correct_config_file = ( - "correct_images/default_yaml/sx3/correct_images.yaml" + # find out default yaml paths + root = Path(__file__).resolve().parents[1] + + if mission.image.format == "acfr_standard": + camera_yaml_path = root / acfr_std_camera_file + default_file_path_correct_config = root / acfr_std_correct_config_file + elif mission.image.format == "seaxerocks_3": + camera_yaml_path = root / sx3_camera_file + default_file_path_correct_config = root / sx3_std_correct_config_file + elif mission.image.format == "biocam": + camera_yaml_path = root / biocam_camera_file + default_file_path_correct_config = root / biocam_std_correct_config_file + elif mission.image.format == "biocam4000_15c": + camera_yaml_path = root / biocam4000_15c_camera_file + default_file_path_correct_config = ( + root / biocam4000_15c_std_correct_config_file ) - biocam_std_correct_config_file = ( - "correct_images/default_yaml/biocam/correct_images.yaml" + elif mission.image.format == "hybis": + camera_yaml_path = root / hybis_camera_file + default_file_path_correct_config = root / hybis_std_correct_config_file + elif mission.image.format == "ntnu_stereo": + camera_yaml_path = root / ntnu_camera_file + default_file_path_correct_config = root / ntnu_std_correct_config_file + elif mission.image.format == "rosbag_extracted_images": + camera_yaml_path = root / rosbag_extracted_camera_file + default_file_path_correct_config = ( + root / rosbag_extracted_images_std_correct_config_file ) - biocam4000_15c_std_correct_config_file = ( - "correct_images/default_yaml/biocam4000_15c/correct_images.yaml" - ) - hybis_std_correct_config_file = ( - "correct_images/default_yaml/hybis/correct_images.yaml" - ) - ntnu_std_correct_config_file = ( - "correct_images/default_yaml/ntnu_stereo/correct_images.yaml" - ) - rosbag_extracted_images_std_correct_config_file = ( - "correct_images/default_yaml/rosbag_extracted_images/correct_images.yaml" + else: + Console.quit( + "Image system in camera.yaml does not match with mission.yaml", + "Provide correct camera.yaml in /raw folder... ", ) - Console.info("Image format:", mission.image.format) - - if mission.image.format == "acfr_standard": - camera_yaml_path = root / acfr_std_camera_file - default_file_path_correct_config = root / acfr_std_correct_config_file - elif mission.image.format == "seaxerocks_3": - camera_yaml_path = root / sx3_camera_file - default_file_path_correct_config = root / sx3_std_correct_config_file - elif mission.image.format == "biocam": - camera_yaml_path = root / biocam_camera_file - default_file_path_correct_config = root / biocam_std_correct_config_file - elif mission.image.format == "biocam4000_15c": - camera_yaml_path = root / biocam4000_15c_camera_file - default_file_path_correct_config = ( - root / biocam4000_15c_std_correct_config_file - ) - elif mission.image.format == "hybis": - camera_yaml_path = root / hybis_camera_file - default_file_path_correct_config = root / hybis_std_correct_config_file - elif mission.image.format == "ntnu_stereo": - camera_yaml_path = root / ntnu_camera_file - default_file_path_correct_config = root / ntnu_std_correct_config_file - elif mission.image.format == "rosbag_extracted_images": - camera_yaml_path = root / rosbag_extracted_camera_file - default_file_path_correct_config = ( - root / rosbag_extracted_images_std_correct_config_file - ) - else: - Console.quit( - "Image system in camera.yaml does not match with mission.yaml", - "Provide correct camera.yaml in /raw folder... ", - ) + # resolve path to camera.yaml file + expected_camera_yaml_path = path_raw_folder / "camera.yaml" + if not expected_camera_yaml_path.exists(): + Console.info( + "Not found camera.yaml file in /raw folder...Using default ", + "camera.yaml file...", + ) else: Console.info("Found camera.yaml file in /raw folder...") - camera_yaml_path = temp_path - - Console.info("camera.yaml:", camera_yaml_path) - Console.info("raw folder:", path_raw_folder) + camera_yaml_path = expected_camera_yaml_path # instantiate the camera system and setup cameras from mission and # config files / auv_nav From 08dbb1fbd0f2bb303444093a5d059fa7fadae4e6 Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 12:22:50 +0000 Subject: [PATCH 5/9] Fix #41 and #42: Do not expect all sections in correct_images.yaml --- src/correct_images/correct_images.py | 5 + src/correct_images/corrector.py | 6 + src/correct_images/parser.py | 567 ++++++++++++++------------- 3 files changed, 296 insertions(+), 282 deletions(-) diff --git a/src/correct_images/correct_images.py b/src/correct_images/correct_images.py index 299d5ca6..fe1b1d14 100644 --- a/src/correct_images/correct_images.py +++ b/src/correct_images/correct_images.py @@ -229,6 +229,11 @@ def call_rescale(args): # install freeimage plugins if not installed imageio.plugins.freeimage.download() + if correct_config.camerarescale is None: + Console.error("Camera rescale configuration not found") + Console.error("Please populate the correct_images.yaml file with a rescale configuration") + Console.quit("Malformed correct_images.yaml file") + # obtain parameters for rescale from correct_config rescale_cameras = correct_config.camerarescale.rescale_cameras diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 6bc375d2..ecd60858 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -99,6 +99,12 @@ def __init__(self, force=False, suffix=None, camera=None, correct_config=None, p """Load general configuration parameters""" self.correction_method = self.correct_config.method if self.correction_method == "colour_correction": + if self.correct_config.color_correction is None: + Console.error( + "No color correction parameters found in the config file" + ) + Console.error("Please populate the color_correction section") + Console.quit("Malformed correct_images.yaml file") self.distance_metric = ( self.correct_config.color_correction.distance_metric ) diff --git a/src/correct_images/parser.py b/src/correct_images/parser.py index 444ceec6..c755a222 100644 --- a/src/correct_images/parser.py +++ b/src/correct_images/parser.py @@ -1,282 +1,285 @@ -# -*- coding: utf-8 -*- -""" -Copyright (c) 2020, University of Southampton -All rights reserved. -Licensed under the BSD 3-Clause License. -See LICENSE.md file in the project root for full license information. -""" - -import numpy as np -import yaml - -from oplab import Console - - -class ColourCorrection: - """class ColourCorrection creates an object for generic colour correction - parameters from correct_images.yaml file - - Attributes - ----------- - distance_metric : string - what mode of distance values will be used - metric_path : string - path to the file containing distance values - altitude_max : int - maximum permissible height for an image for use in calculating - attenuation coefficients - altitude_min : int - minimum permissible height for an image for use in calculating - attenuation coefficients - smoothing : string - method of sampling intensity values - window_size : int - control how noisy the parameters can be - outlier_reject : bool - flag for choosing filtering image outliers - """ - - def __init__(self, node): - """__init__ is the constructor function - - Parameters - ----------- - node : cdict - dictionary object for an entry in correct_images.yaml file - """ - self.distance_metric = node["distance_metric"] - self.metric_path = node["metric_path"] - self.altitude_max = node["altitude_filter"]["max_m"] - self.altitude_min = node["altitude_filter"]["min_m"] - self.smoothing = node["smoothing"] - self.window_size = node["window_size"] - self.outlier_reject = node["curve_fitting_outlier_rejection"] - - -class CameraConfig: - """class Config creates an object for camera specific configuration - parameters from correct_images.yaml file - - Attributes - ----------- - camera_name : string - name of camera - imagefilelist : list - list of paths to images provided by user - brightness : int - target mean for the image intensities - contrast : int - target std for the image intensities - subtractors_rgb : matrix - parameters for manual balance of images - color_correct_matrix_rgb : matrix - parameters for manual balance of images - """ - - def __init__(self, node): - """__init__ is the constructor function - - Parameters - ----------- - node : cdict - dictionary object for an entry in correct_images.yaml file - """ - - camera_name = node["camera_name"] - - imagefilelist_parse = "none" - imagefilelist_process = "none" - if "image_file_list" in node: - imagefilelist_parse = node.get("image_file_list", {}).get("parse", "none") - imagefilelist_process = node.get("image_file_list", {}).get( - "process", "none" - ) - - brightness = 30.0 - contrast = 3.0 - if "colour_correction" in node: - brightness = node.get("colour_correction", {}).get("brightness", brightness) - contrast = node.get("colour_correction", {}).get("contrast", contrast) - - subtractors_rgb = np.array([0, 0, 0]) - color_gain_matrix_rgb = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - if "manual_balance" in node: - subtractors_rgb = node.get("manual_balance", {}).get( - "subtractors_rgb", subtractors_rgb - ) - color_gain_matrix_rgb = node.get("manual_balance", {}).get( - "colour_gain_matrix_rgb", color_gain_matrix_rgb - ) - - self.camera_name = camera_name - self.imagefilelist_parse = imagefilelist_parse - self.imagefilelist_process = imagefilelist_process - self.brightness = brightness - self.contrast = contrast - self.subtractors_rgb = subtractors_rgb - self.color_gain_matrix_rgb = color_gain_matrix_rgb - - -class CameraConfigs: - - """class OutputSettings creates an object for output_settings parameters - from the correct_images.yaml file - - Attributes - ---------- - camera_configs : Config object - object stores camera specific configuration parameters - num_cameras : int - number of cameras in the system - """ - - def __init__(self, node): - """__init__ is the constructor function - - Parameters - ----------- - node : cdict - dictionary object for an entry in correct_images.yaml file - """ - self.camera_configs = [] - self.num_cameras = len(node) - for i in range(self.num_cameras): - self.camera_configs.append(CameraConfig(node[i])) - - -class OutputSettings: - - """class OutputSettings creates an object for output_settings parameters - from the correct_images.yaml file - - Attributes - ---------- - undistort_flag : bool - flag denotes if images need to be corrected for distortion - compression_parameter : str - output format in which images need to be saved - """ - - def __init__(self, node): - - """__init__ is the constructor function - - Parameters - ----------- - node : cdict - dictionary object for an entry in correct_images.yaml file - """ - - self.undistort_flag = node["undistort"] - self.compression_parameter = node["compression_parameter"] - - -class RescaleImage: - def __init__( - self, - camera_name, - path, - distance_path, - interpolate_method, - target_pixel_size, - maintain_pixels, - output_folder, - ): - - self.camera_name = camera_name - self.path = path - self.distance_path = distance_path - self.interpolate_method = interpolate_method - self.target_pixel_size = target_pixel_size / 100 - self.maintain_pixels = maintain_pixels - self.output_folder = output_folder - - -class CameraRescale: - def __init__(self, node): - """__init__ is the constructor function - - Parameters - ----------- - node : cdict - dictionary object for an entry in correct_images.yaml file - """ - self.rescale_cameras = [] - self.num_cameras = len(node) - for i in range(self.num_cameras): - self.rescale_cameras.append( - RescaleImage( - node[i]["camera_name"], - node[i]["path"], - node[i]["distance_path"], - node[i]["interpolate_method"], - node[i]["target_pixel_size"], - node[i]["maintain_pixels"], - node[i]["output_folder"], - ) - ) - - -class CorrectConfig: - """class CorrectConfig creates an object from the correct_images.yaml file - - Attributes - ---------- - version : str - version of the correct_images.yaml - method : str - method of correction to be used - color_correction : ColourCorrection Object - object contains parameters for colour based corrections - configs : CameraConfigs Object - object contains camera specific configuration parameters - output_settings : OutputSettings Object - object contains parameters for output settings - - """ - - def __init__(self, filename=None): - - """__init__ is the constructor function - - Parameters - ----------- - filename : Path - path to the correct_images.yaml file - """ - - if filename is None: - return - with filename.open("r") as stream: - data = yaml.safe_load(stream) - - if "version" not in data: - Console.error( - "It seems you are using an old correct_images.yaml.", - "You will have to delete it and run this software again.", - ) - Console.error("Delete the file with:") - Console.error(" rm ", filename) - Console.quit("Wrong correct_images.yaml format") - self.version = data["version"] - - valid_methods = ["manual_balance", "colour_correction"] - self.method = data["method"] - - if self.method not in valid_methods: - Console.quit( - "The method requested (", - self.method, - ") is not supported or implemented. The valid methods", - "are:", - " ".join(m for m in valid_methods), - ) - - node = data["colour_correction"] - self.color_correction = ColourCorrection(node) - node = data["cameras"] - self.configs = CameraConfigs(node) - node = data["output_settings"] - self.output_settings = OutputSettings(node) - node = data["rescale"] - self.camerarescale = CameraRescale(node) +# -*- coding: utf-8 -*- +""" +Copyright (c) 2020, University of Southampton +All rights reserved. +Licensed under the BSD 3-Clause License. +See LICENSE.md file in the project root for full license information. +""" + +import numpy as np +import yaml + +from oplab import Console + + +class ColourCorrection: + """class ColourCorrection creates an object for generic colour correction + parameters from correct_images.yaml file + + Attributes + ----------- + distance_metric : string + what mode of distance values will be used + metric_path : string + path to the file containing distance values + altitude_max : int + maximum permissible height for an image for use in calculating + attenuation coefficients + altitude_min : int + minimum permissible height for an image for use in calculating + attenuation coefficients + smoothing : string + method of sampling intensity values + window_size : int + control how noisy the parameters can be + outlier_reject : bool + flag for choosing filtering image outliers + """ + + def __init__(self, node): + """__init__ is the constructor function + + Parameters + ----------- + node : cdict + dictionary object for an entry in correct_images.yaml file + """ + self.distance_metric = node["distance_metric"] + self.metric_path = node["metric_path"] + self.altitude_max = node["altitude_filter"]["max_m"] + self.altitude_min = node["altitude_filter"]["min_m"] + self.smoothing = node["smoothing"] + self.window_size = node["window_size"] + self.outlier_reject = node["curve_fitting_outlier_rejection"] + + +class CameraConfig: + """class Config creates an object for camera specific configuration + parameters from correct_images.yaml file + + Attributes + ----------- + camera_name : string + name of camera + imagefilelist : list + list of paths to images provided by user + brightness : int + target mean for the image intensities + contrast : int + target std for the image intensities + subtractors_rgb : matrix + parameters for manual balance of images + color_correct_matrix_rgb : matrix + parameters for manual balance of images + """ + + def __init__(self, node): + """__init__ is the constructor function + + Parameters + ----------- + node : cdict + dictionary object for an entry in correct_images.yaml file + """ + + camera_name = node["camera_name"] + + imagefilelist_parse = "none" + imagefilelist_process = "none" + if "image_file_list" in node: + imagefilelist_parse = node.get("image_file_list", {}).get("parse", "none") + imagefilelist_process = node.get("image_file_list", {}).get( + "process", "none" + ) + + brightness = 30.0 + contrast = 3.0 + if "colour_correction" in node: + brightness = node.get("colour_correction", {}).get("brightness", brightness) + contrast = node.get("colour_correction", {}).get("contrast", contrast) + + subtractors_rgb = np.array([0, 0, 0]) + color_gain_matrix_rgb = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + if "manual_balance" in node: + subtractors_rgb = node.get("manual_balance", {}).get( + "subtractors_rgb", subtractors_rgb + ) + color_gain_matrix_rgb = node.get("manual_balance", {}).get( + "colour_gain_matrix_rgb", color_gain_matrix_rgb + ) + + self.camera_name = camera_name + self.imagefilelist_parse = imagefilelist_parse + self.imagefilelist_process = imagefilelist_process + self.brightness = brightness + self.contrast = contrast + self.subtractors_rgb = subtractors_rgb + self.color_gain_matrix_rgb = color_gain_matrix_rgb + + +class CameraConfigs: + + """class OutputSettings creates an object for output_settings parameters + from the correct_images.yaml file + + Attributes + ---------- + camera_configs : Config object + object stores camera specific configuration parameters + num_cameras : int + number of cameras in the system + """ + + def __init__(self, node): + """__init__ is the constructor function + + Parameters + ----------- + node : cdict + dictionary object for an entry in correct_images.yaml file + """ + self.camera_configs = [] + self.num_cameras = len(node) + for i in range(self.num_cameras): + self.camera_configs.append(CameraConfig(node[i])) + + +class OutputSettings: + + """class OutputSettings creates an object for output_settings parameters + from the correct_images.yaml file + + Attributes + ---------- + undistort_flag : bool + flag denotes if images need to be corrected for distortion + compression_parameter : str + output format in which images need to be saved + """ + + def __init__(self, node): + + """__init__ is the constructor function + + Parameters + ----------- + node : cdict + dictionary object for an entry in correct_images.yaml file + """ + + self.undistort_flag = node["undistort"] + self.compression_parameter = node["compression_parameter"] + + +class RescaleImage: + def __init__( + self, + camera_name, + path, + distance_path, + interpolate_method, + target_pixel_size, + maintain_pixels, + output_folder, + ): + + self.camera_name = camera_name + self.path = path + self.distance_path = distance_path + self.interpolate_method = interpolate_method + self.target_pixel_size = target_pixel_size / 100 + self.maintain_pixels = maintain_pixels + self.output_folder = output_folder + + +class CameraRescale: + def __init__(self, node): + """__init__ is the constructor function + + Parameters + ----------- + node : cdict + dictionary object for an entry in correct_images.yaml file + """ + self.rescale_cameras = [] + self.num_cameras = len(node) + for i in range(self.num_cameras): + self.rescale_cameras.append( + RescaleImage( + node[i]["camera_name"], + node[i]["path"], + node[i]["distance_path"], + node[i]["interpolate_method"], + node[i]["target_pixel_size"], + node[i]["maintain_pixels"], + node[i]["output_folder"], + ) + ) + + +class CorrectConfig: + """class CorrectConfig creates an object from the correct_images.yaml file + + Attributes + ---------- + version : str + version of the correct_images.yaml + method : str + method of correction to be used + color_correction : ColourCorrection Object + object contains parameters for colour based corrections + configs : CameraConfigs Object + object contains camera specific configuration parameters + output_settings : OutputSettings Object + object contains parameters for output settings + + """ + + def __init__(self, filename=None): + + """__init__ is the constructor function + + Parameters + ----------- + filename : Path + path to the correct_images.yaml file + """ + + if filename is None: + return + with filename.open("r") as stream: + data = yaml.safe_load(stream) + + self.color_correction = None + self.camerarescale = None + + if "version" not in data: + Console.error( + "It seems you are using an old correct_images.yaml.", + "You will have to delete it and run this software again.", + ) + Console.error("Delete the file with:") + Console.error(" rm ", filename) + Console.quit("Wrong correct_images.yaml format") + self.version = data["version"] + + valid_methods = ["manual_balance", "colour_correction"] + self.method = data["method"] + + if self.method not in valid_methods: + Console.quit( + "The method requested (", + self.method, + ") is not supported or implemented. The valid methods", + "are:", + " ".join(m for m in valid_methods), + ) + + if "colour_correction" in data: + self.color_correction = ColourCorrection(data["colour_correction"]) + node = data["cameras"] + self.configs = CameraConfigs(node) + node = data["output_settings"] + self.output_settings = OutputSettings(node) + if "rescale" in data: + self.camerarescale = CameraRescale(data["rescale"]) From d516d82cddfe4d6b713fb0dd8e91534a664fd30c Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Tue, 18 Jan 2022 12:45:46 +0000 Subject: [PATCH 6/9] Implement adrian-bodenmann proposed changes in folder naming --- src/correct_images/corrector.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index ecd60858..7c57e5db 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -276,7 +276,7 @@ def create_output_directories(self): # Create output directories depending on the correction method parameters_folder_str = "params_" - developed_folder_str = "developed_" + developed_folder_str = "corrected_" subfolder_name = None if self.correction_method == "colour_correction": @@ -290,15 +290,15 @@ def create_output_directories(self): parameters_folder_str += "dm" developed_folder_str += "dm" subfolder_name = ( - "m" + str(int(self.brightness)) + "_std" + str(int(self.contrast)) + "mean_" + str(int(self.brightness)) + "_std_" + str(int(self.contrast)) ) elif self.correction_method == "manual_balance": parameters_folder_str += "man" developed_folder_str += "man" - subfolder_name = ("g_" + subfolder_name = ("gain_" + str(self.color_gain_matrix_rgb[0]) + "_" + str(self.color_gain_matrix_rgb[4]) + "_" - + str(self.color_gain_matrix_rgb[8]) + "_s_" + + str(self.color_gain_matrix_rgb[8]) + "_sub_" + str(self.subtractors_rgb[0]) + "_" + str(self.subtractors_rgb[1]) + "_" + str(self.subtractors_rgb[2]) From 835776a74f7dc773b506aba6f02bfc515c637980 Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Wed, 19 Jan 2022 12:00:31 +0000 Subject: [PATCH 7/9] Change to shallower folders, longer names, uniform instead of none --- src/correct_images/corrector.py | 61 ++++----- .../default_yaml/acfr/correct_images.yaml | 100 +++++++------- .../default_yaml/biocam/correct_images.yaml | 102 +++++++-------- .../biocam4000_15c/correct_images.yaml | 98 +++++++------- .../default_yaml/hybis/correct_images.yaml | 2 +- .../ntnu_stereo/correct_images.yaml | 2 +- .../default_yaml/sx3/correct_images.yaml | 122 +++++++++--------- src/correct_images/parser.py | 11 ++ src/correct_images/tools/curve_fitting.py | 21 ++- 9 files changed, 257 insertions(+), 262 deletions(-) diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 7c57e5db..0bd4727c 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -214,7 +214,7 @@ def process(self): self.attenuation_params_filepath ) else: - if self.distance_metric != "none": + if self.distance_metric != "uniform": Console.quit( "Code does not find attenuation_parameters.npy...", "Please run parse before process...", @@ -222,7 +222,7 @@ def process(self): if self.correction_gains_filepath.exists(): self.correction_gains = np.load(self.correction_gains_filepath) else: - if self.distance_metric != "none": + if self.distance_metric != "uniform": Console.quit( "Code does not find correction_gains.npy...", "Please run parse before process...", @@ -232,7 +232,7 @@ def process(self): self.corrected_mean_filepath ).squeeze() else: - if self.distance_metric != "none": + if self.distance_metric != "uniform": Console.quit( "Code does not find image_corrected_mean.npy...", "Please run parse before process...", @@ -242,21 +242,21 @@ def process(self): self.corrected_std_filepath ).squeeze() else: - if self.distance_metric != "none": + if self.distance_metric != "uniform": Console.quit( "Code does not find image_corrected_std.npy...", "Please run parse before process...", ) - if self.raw_mean_filepath.exists() and self.distance_metric == "none": + if self.raw_mean_filepath.exists() and self.distance_metric == "uniform": self.image_raw_mean = np.load(self.raw_mean_filepath).squeeze() - elif self.distance_metric == "none": + elif self.distance_metric == "uniform": Console.quit( "Code does not find image_raw_mean.npy...", "Please run parse before process...", ) - if self.raw_std_filepath.exists() and self.distance_metric == "none": + if self.raw_std_filepath.exists() and self.distance_metric == "uniform": self.image_raw_std = np.load(self.raw_std_filepath).squeeze() - elif self.distance_metric == "none": + elif self.distance_metric == "uniform": Console.quit( "Code does not find image_raw_std.npy...", "Please run parse before process...", @@ -278,24 +278,16 @@ def create_output_directories(self): parameters_folder_str = "params_" developed_folder_str = "corrected_" - subfolder_name = None if self.correction_method == "colour_correction": - if self.distance_metric == "none": - parameters_folder_str += "ps" - developed_folder_str += "ps" - elif self.distance_metric == "altitude": - parameters_folder_str += "alt" - developed_folder_str += "alt" - elif self.distance_metric == "depth_map": - parameters_folder_str += "dm" - developed_folder_str += "dm" - subfolder_name = ( - "mean_" + str(int(self.brightness)) + "_std_" + str(int(self.contrast)) + parameters_folder_str += self.distance_metric + developed_folder_str += self.distance_metric + developed_folder_str += ( + "_mean_" + str(int(self.brightness)) + "_std_" + str(int(self.contrast)) ) elif self.correction_method == "manual_balance": - parameters_folder_str += "man" - developed_folder_str += "man" - subfolder_name = ("gain_" + parameters_folder_str += "manual" + developed_folder_str += "manual" + developed_folder_str += ("_gain_" + str(self.color_gain_matrix_rgb[0]) + "_" + str(self.color_gain_matrix_rgb[4]) + "_" + str(self.color_gain_matrix_rgb[8]) + "_sub_" @@ -311,7 +303,6 @@ def create_output_directories(self): self.attenuation_parameters_folder = self.output_dir_path / parameters_folder_str self.output_images_folder = self.output_dir_path / developed_folder_str - self.output_images_folder /= subfolder_name if not self.attenuation_parameters_folder.exists(): self.attenuation_parameters_folder.mkdir(parents=True) @@ -428,7 +419,7 @@ def get_imagelist(self): def get_altitude_and_depth_maps(self): """Generate distance matrix numpy files and save them""" # read altitude / depth map depending on distance_metric - if self.distance_metric == "none": + if self.distance_metric == "uniform": Console.info("Null distance matrix created") return @@ -466,7 +457,7 @@ def get_altitude_and_depth_maps(self): filtered_dataframe = dataframe.iloc[valid_idx] filtered_dataframe.reset_index(drop=True) # Warning: if the column does not contain any 'None' entry, it will be parsed as float, and the .str() accesor will fail - filtered_dataframe["altitude [m]"] = filtered_dataframe["altitude [m]"].astype('string') + filtered_dataframe["altitude [m]"] = filtered_dataframe["altitude [m]"].astype(str) filtered_dataframe=filtered_dataframe[~filtered_dataframe["altitude [m]"].str.contains("None")] # drop rows with None altitude distance_list = filtered_dataframe["altitude [m]"].tolist() self.camera_image_list = [] @@ -821,26 +812,24 @@ def compute_distance_bin( del memmap_handle os.remove(memmap_filename) + base_path = Path(self.attenuation_parameters_folder) + fig = plt.figure() plt.imshow(bin_images_sample) plt.colorbar() plt.title("Image bin " + str(idx_bin)) - plt.savefig( - Path(self.attenuation_parameters_folder) - / ("bin_images_sample_" + str(idx_bin) + ".png"), - dpi=600, - ) + fig_name = base_path / ("bin_images_sample_" + str(idx_bin) + ".png") + #Console.info("Saved figure at", fig_name) + plt.savefig(fig_name, dpi=600) plt.close(fig) fig = plt.figure() plt.imshow(bin_distances_sample) plt.colorbar() plt.title("Distance bin " + str(idx_bin)) - plt.savefig( - Path(self.attenuation_parameters_folder) - / ("bin_distances_sample_" + str(idx_bin) + ".png"), - dpi=600, - ) + fig_name = base_path / ("bin_distances_sample_" + str(idx_bin) + ".png") + #Console.info("Saved figure at", fig_name) + plt.savefig(fig_name, dpi=600) plt.close(fig) images_map[idx_bin] = bin_images_sample.reshape( diff --git a/src/correct_images/default_yaml/acfr/correct_images.yaml b/src/correct_images/default_yaml/acfr/correct_images.yaml index 19dceaff..e17941c5 100644 --- a/src/correct_images/default_yaml/acfr/correct_images.yaml +++ b/src/correct_images/default_yaml/acfr/correct_images.yaml @@ -1,50 +1,50 @@ -# Yaml 1.0 -version: 1 - -method: 'colour_correction' - -colour_correction : - distance_metric : 'altitude' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run - metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric - altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters - max_m : 12 - min_m : 2 - smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model - window_size : 3 # increase if the attenuation correction parameter looks noisy. - curve_fitting_outlier_rejection : False - -cameras : - - camera_name : 'LC' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance : - subtractors_rgb : [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - - - camera_name : 'RC' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance: - subtractors_rgb: [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - -output_settings : - undistort : False - compression_parameter : 'png' # 'tiff' - -rescale : - - camera_name : 'LC' - path : # path to images relative to processed folder - distance_path : # path to json_renav* folder for auv_ekf_.csv file - interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos - target_pixel_size : 0.01 # in meters - maintain_pixels : True - output_folder : # output path relative to processed images +# Yaml 1.0 +version: 1 + +method: 'colour_correction' + +colour_correction : + distance_metric : 'altitude' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric + altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters + max_m : 12 + min_m : 2 + smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model + window_size : 3 # increase if the attenuation correction parameter looks noisy. + curve_fitting_outlier_rejection : False + +cameras : + - camera_name : 'LC' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance : + subtractors_rgb : [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + + - camera_name : 'RC' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance: + subtractors_rgb: [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + +output_settings : + undistort : False + compression_parameter : 'png' # 'tiff' + +rescale : + - camera_name : 'LC' + path : # path to images relative to processed folder + distance_path : # path to json_renav* folder for auv_ekf_.csv file + interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos + target_pixel_size : 0.01 # in meters + maintain_pixels : True + output_folder : # output path relative to processed images diff --git a/src/correct_images/default_yaml/biocam/correct_images.yaml b/src/correct_images/default_yaml/biocam/correct_images.yaml index a8dbbeb9..8daa7a71 100644 --- a/src/correct_images/default_yaml/biocam/correct_images.yaml +++ b/src/correct_images/default_yaml/biocam/correct_images.yaml @@ -1,51 +1,51 @@ -# Yaml 1.0 -version: 1 - -method: 'colour_correction' # ['manual_balance', 'colour_correction'] - -colour_correction : - distance_metric : 'altitude' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run - metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric - altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters - max_m : 12 - min_m : 2 - smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model - window_size : 3 # increase if the attenuation correction parameter looks noisy. - curve_fitting_outlier_rejection : False - -cameras : - - camera_name : 'cam61003146' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance : - subtractors_rgb : [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - - - camera_name : 'cam61004444' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance: - subtractors_rgb: [0] # monochrome camera -> single value (but still needs to be stored as list) - colour_gain_matrix_rgb: [[1]] # monochrome camera -> single value (but still needs to be stored as list of list) - -output_settings : - undistort : False - compression_parameter : 'png' # 'tiff' - -rescale : - - camera_name : 'cam61003146' - path : # path to images relative to processed folder - distance_path : # path to json_renav* folder for auv_ekf_.csv file - interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos - target_pixel_size : 0.01 # in meters - maintain_pixels : True - output_folder : # output path relative to processed images - +# Yaml 1.0 +version: 1 + +method: 'colour_correction' # ['manual_balance', 'colour_correction'] + +colour_correction : + distance_metric : 'altitude' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric + altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters + max_m : 12 + min_m : 2 + smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model + window_size : 3 # increase if the attenuation correction parameter looks noisy. + curve_fitting_outlier_rejection : False + +cameras : + - camera_name : 'cam61003146' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance : + subtractors_rgb : [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + + - camera_name : 'cam61004444' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance: + subtractors_rgb: [0] # monochrome camera -> single value (but still needs to be stored as list) + colour_gain_matrix_rgb: [[1]] # monochrome camera -> single value (but still needs to be stored as list of list) + +output_settings : + undistort : False + compression_parameter : 'png' # 'tiff' + +rescale : + - camera_name : 'cam61003146' + path : # path to images relative to processed folder + distance_path : # path to json_renav* folder for auv_ekf_.csv file + interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos + target_pixel_size : 0.01 # in meters + maintain_pixels : True + output_folder : # output path relative to processed images + diff --git a/src/correct_images/default_yaml/biocam4000_15c/correct_images.yaml b/src/correct_images/default_yaml/biocam4000_15c/correct_images.yaml index 96966318..1301f6b4 100644 --- a/src/correct_images/default_yaml/biocam4000_15c/correct_images.yaml +++ b/src/correct_images/default_yaml/biocam4000_15c/correct_images.yaml @@ -1,49 +1,49 @@ -# Yaml 1.0 -version: 1 - -method: 'manual_balance' # ['manual_balance', 'colour_correction'] - -colour_correction : - distance_metric : 'none' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run - metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric - altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters - max_m : 12 - min_m : 2 - smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model - window_size : 3 # increase if the attenuation correction parameter looks noisy. - curve_fitting_outlier_rejection : False - -cameras : - - camera_name : 'cam61008031' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance : - subtractors_rgb : [0] # monochrome camera -> single value (but still needs to be stored as list) - colour_gain_matrix_rgb : [[8]] # monochrome camera -> single value (but still needs to be stored as list of list) - - camera_name : 'cam61008269' #in line with vehicle.yaml - image_file_list : - parse : 'none' # Only inidicate this if you want to limit the images used - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance: - subtractors_rgb: [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb: [[8, 0, 0], [0, 16.7, 0], [0, 0, 26]] # diagonal terms corresponding to r gain, g gain, b gain - -output_settings : - undistort : False - compression_parameter : 'png' # 'tiff' - -rescale : - - camera_name : 'cam61008269' - path : # path to images relative to processed folder - distance_path : # path to json_renav* folder for auv_ekf_.csv file - interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos - target_pixel_size : 0.01 # in meters - maintain_pixels : True - output_folder : # output path relative to processed images +# Yaml 1.0 +version: 1 + +method: 'manual_balance' # ['manual_balance', 'colour_correction'] + +colour_correction : + distance_metric : 'uniform' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric + altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters + max_m : 12 + min_m : 2 + smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model + window_size : 3 # increase if the attenuation correction parameter looks noisy. + curve_fitting_outlier_rejection : False + +cameras : + - camera_name : 'cam61008031' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used. folder is in the image path + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance : + subtractors_rgb : [0] # monochrome camera -> single value (but still needs to be stored as list) + colour_gain_matrix_rgb : [[8]] # monochrome camera -> single value (but still needs to be stored as list of list) + - camera_name : 'cam61008269' #in line with vehicle.yaml + image_file_list : + parse : 'none' # Only inidicate this if you want to limit the images used + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance: + subtractors_rgb: [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb: [[8, 0, 0], [0, 16.7, 0], [0, 0, 26]] # diagonal terms corresponding to r gain, g gain, b gain + +output_settings : + undistort : False + compression_parameter : 'png' # 'tiff' + +rescale : + - camera_name : 'cam61008269' + path : # path to images relative to processed folder + distance_path : # path to json_renav* folder for auv_ekf_.csv file + interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos + target_pixel_size : 0.01 # in meters + maintain_pixels : True + output_folder : # output path relative to processed images diff --git a/src/correct_images/default_yaml/hybis/correct_images.yaml b/src/correct_images/default_yaml/hybis/correct_images.yaml index 30e277ea..ea678621 100644 --- a/src/correct_images/default_yaml/hybis/correct_images.yaml +++ b/src/correct_images/default_yaml/hybis/correct_images.yaml @@ -4,7 +4,7 @@ version: 1 method: 'colour_correction' colour_correction : - distance_metric : 'altitude' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + distance_metric : 'altitude' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters max_m : 5.5 diff --git a/src/correct_images/default_yaml/ntnu_stereo/correct_images.yaml b/src/correct_images/default_yaml/ntnu_stereo/correct_images.yaml index 672c3d95..b5fdc0ba 100644 --- a/src/correct_images/default_yaml/ntnu_stereo/correct_images.yaml +++ b/src/correct_images/default_yaml/ntnu_stereo/correct_images.yaml @@ -4,7 +4,7 @@ version: 1 method: 'colour_correction' colour_correction : - distance_metric : 'altitude' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + distance_metric : 'altitude' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters max_m : 4 diff --git a/src/correct_images/default_yaml/sx3/correct_images.yaml b/src/correct_images/default_yaml/sx3/correct_images.yaml index 28f19b04..f174130f 100644 --- a/src/correct_images/default_yaml/sx3/correct_images.yaml +++ b/src/correct_images/default_yaml/sx3/correct_images.yaml @@ -1,62 +1,62 @@ -# Yaml 1.0 - -version: 1 - -method: 'colour_correction' - -colour_correction : - distance_metric : 'altitude' # ['none', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run - metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric - altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters - max_m : 12 - min_m : 2 - smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model - window_size : 3 # increase if the attenuation correction parameter looks noisy. - curve_fitting_outlier_rejection : False - -cameras : - - camera_name : 'Cam51707923' #in line with vehicle.yaml - image_file_list : # Only inidicate this if you want to limit the images used. folder is in the image path - parse : 'none' - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance : - subtractors_rgb : [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - - - camera_name : 'Cam51707925' #in line with vehicle.yaml - image_file_list : # Only inidicate this if you want to limit the images used - parse : 'none' - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance: - subtractors_rgb: [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - - - camera_name : 'LM165' #in line with vehicle.yaml - image_file_list : # Only inidicate this if you want to limit the images used - parse : 'none' - process : 'none' - colour_correction : - brightness : 30 # % target across dataset - contrast : 3 # % target accross dataset - manual_balance: - subtractors_rgb: [0, 0, 0] # -c values (zero capped) - colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain - -output_settings : - undistort : False - compression_parameter : 'png' # 'tiff' - -rescale : - - camera_name : 'Cam51707923' - path : # path to images relative to processed folder - distance_path : # path to json_renav* folder for auv_ekf_.csv file - interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos - target_pixel_size : 0.01 # in meters - maintain_pixels : True +# Yaml 1.0 + +version: 1 + +method: 'colour_correction' + +colour_correction : + distance_metric : 'altitude' # ['uniform', 'altitude', 'depth_map'] assumes constant altitude, flat seafloor, 3d structure respectively. depth_map requires **** to have been run + metric_path : 'json_renav_*' # json nav path in case of altitude / depth map path in case of depth map metric + altitude_filter : # only use images in the altitude range below to calculate attenuation_modelling parameters + max_m : 12 + min_m : 2 + smoothing : 'median' # 'mean' / 'median' / 'mean_trimmed' sampling colour intensity values from window_size to develop model + window_size : 3 # increase if the attenuation correction parameter looks noisy. + curve_fitting_outlier_rejection : False + +cameras : + - camera_name : 'Cam51707923' #in line with vehicle.yaml + image_file_list : # Only inidicate this if you want to limit the images used. folder is in the image path + parse : 'none' + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance : + subtractors_rgb : [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb : [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + + - camera_name : 'Cam51707925' #in line with vehicle.yaml + image_file_list : # Only inidicate this if you want to limit the images used + parse : 'none' + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance: + subtractors_rgb: [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + + - camera_name : 'LM165' #in line with vehicle.yaml + image_file_list : # Only inidicate this if you want to limit the images used + parse : 'none' + process : 'none' + colour_correction : + brightness : 30 # % target across dataset + contrast : 3 # % target accross dataset + manual_balance: + subtractors_rgb: [0, 0, 0] # -c values (zero capped) + colour_gain_matrix_rgb: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # diagonal terms corresponding to r gain, g gain, b gain + +output_settings : + undistort : False + compression_parameter : 'png' # 'tiff' + +rescale : + - camera_name : 'Cam51707923' + path : # path to images relative to processed folder + distance_path : # path to json_renav* folder for auv_ekf_.csv file + interpolate_method : 'bicubic' # bicubic, nearest_neighbour, bilinear, lanczos + target_pixel_size : 0.01 # in meters + maintain_pixels : True output_folder : # output path relative to processed images \ No newline at end of file diff --git a/src/correct_images/parser.py b/src/correct_images/parser.py index c755a222..9f5c0a07 100644 --- a/src/correct_images/parser.py +++ b/src/correct_images/parser.py @@ -44,7 +44,18 @@ def __init__(self, node): node : cdict dictionary object for an entry in correct_images.yaml file """ + valid_distance_metrics = ["uniform", "altitude", "depth_map"] self.distance_metric = node["distance_metric"] + + if self.distance_metric == "none": + self.distance_metric = "uniform" + Console.warn("Distance metric is set to none. This is deprecated.", + "Please use uniform instead.") + if self.distance_metric not in valid_distance_metrics: + Console.error("Distance metric not valid. Please use one of the following:", + valid_distance_metrics) + Console.quit("Invalid distance metric: {}".format(self.distance_metric)) + self.metric_path = node["metric_path"] self.altitude_max = node["altitude_filter"]["max_m"] self.altitude_min = node["altitude_filter"]["min_m"] diff --git a/src/correct_images/tools/curve_fitting.py b/src/correct_images/tools/curve_fitting.py index 2458192c..33b0bcb9 100644 --- a/src/correct_images/tools/curve_fitting.py +++ b/src/correct_images/tools/curve_fitting.py @@ -7,6 +7,7 @@ """ import matplotlib.pyplot as plt +from pathlib import Path import numpy as np from numba import njit from scipy import optimize @@ -120,18 +121,6 @@ def curve_fitting(altitudes: np.ndarray, intensities: np.ndarray) -> np.ndarray: # for debug save = False - """ - i = 0 - fn = None - while i < 100: - fn = Path("least_squares_good" + str(i) + ".png") - i += 1 - if fn.exists(): - continue - else: - save = True - break - """ try: tmp_params = optimize.least_squares( @@ -143,7 +132,13 @@ def curve_fitting(altitudes: np.ndarray, intensities: np.ndarray) -> np.ndarray: bounds=(bound_lower, bound_upper), ) if save: - fn = "Intensities_curve.png" + i = 0 + while i < 100: + fn = Path("Intensities_curve_" + str(i) + ".png") + i += 1 + if not fn.exists(): + break + fig = plt.figure() plt.plot(altitudes_filt, intensities_filt, "c.") xs = np.arange(1, 5, 0.1) From 5781ae855582ba686f300ba63c7b95df96a0fcf1 Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Thu, 20 Jan 2022 11:10:09 +0000 Subject: [PATCH 8/9] Use numpy median --- src/correct_images/tools/numerical.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/src/correct_images/tools/numerical.py b/src/correct_images/tools/numerical.py index d66fe1b4..822b11d0 100644 --- a/src/correct_images/tools/numerical.py +++ b/src/correct_images/tools/numerical.py @@ -92,39 +92,16 @@ def median_array(data: np.ndarray) -> np.ndarray: # print("median_array", data.shape) if len(data.shape) == 3: # Monochrome - return median_array_impl(data) + return np.median(data, axis=0) elif len(data.shape) == 4: # Colour [n, a, b, c] = data.shape median_array = np.zeros((a, b, c), dtype=np.float32) for c in range(data.shape[3]): - median_array[:, :, c] = median_array_impl(data[:, :, :, c]) + median_array[:, :, c] = np.median(data[:, :, :, c], axis=0) return median_array -def median_array_impl(data: np.ndarray) -> np.ndarray: - # print("median_array_impl", data.shape) - n = data.shape[0] - a = data.shape[1] - b = data.shape[2] - median_array = np.zeros((a, b), dtype=np.float32) - for i in range(a): - for j in range(b): - lst = [0] * n - for k in range(n): - lst[k] = data[k, i, j] - s = sorted(lst) - median = 0 - if n % 2 == 0: - for k in range(n // 2 - 1, n // 2 + 1): - median += s[k] - median /= 2.0 - else: - median = s[n // 2] - median_array[i, j] = median - return median_array - - @njit def mean_std_array(data: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: # print("mean_std_array", data.shape) From 06a4fe6483221bf8a0d0e8ab5c1ab012eff36fae Mon Sep 17 00:00:00 2001 From: Miquel Massot Date: Thu, 20 Jan 2022 11:10:30 +0000 Subject: [PATCH 9/9] Fix bug in altitude bins and curve fitting --- src/correct_images/corrector.py | 32 +++++++++++--------- src/correct_images/tools/curve_fitting.py | 36 ++++++++++++++++++++--- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/correct_images/corrector.py b/src/correct_images/corrector.py index 0bd4727c..6ca17f21 100644 --- a/src/correct_images/corrector.py +++ b/src/correct_images/corrector.py @@ -557,10 +557,12 @@ def generate_attenuation_correction_parameters(self): self.bin_band = 0.1 hist_bins = np.arange(self.altitude_min, self.altitude_max, self.bin_band) + # Watch out: need to substract 1 to get the correct number of bins + # because the last bin is not included in the range images_fn, images_map = open_memmap( shape=( - len(hist_bins), + len(hist_bins) - 1, self.image_height * self.image_width, self.image_channels, ), @@ -568,28 +570,30 @@ def generate_attenuation_correction_parameters(self): ) distances_fn, distances_map = open_memmap( - shape=(len(hist_bins), self.image_height * self.image_width), + shape=(len(hist_bins) - 1, self.image_height * self.image_width), dtype=np.float32, ) - # Fill with NaN not to use empty bins - # images_map.fill(np.nan) - # distances_map.fill(np.nan) - distance_vector = None - if self.depth_map_list is not None: - Console.info("Computing depth map histogram with", hist_bins.size, " bins") + Console.info("Computing depth map histogram with", hist_bins.size - 1, " bins") distance_vector = np.zeros((len(self.depth_map_list), 1)) for i, dm_file in enumerate(self.depth_map_list): dm_np = depth_map.loader(dm_file, self.image_width, self.image_height) distance_vector[i] = dm_np.mean(axis=1) elif self.altitude_list is not None: - Console.info("Computing altitude histogram with", hist_bins.size, " bins") + Console.info("Computing altitude histogram with", hist_bins.size - 1, " bins:") distance_vector = np.array(self.altitude_list) if distance_vector is not None: - idxs = np.digitize(distance_vector, hist_bins) + idxs = np.digitize(distance_vector, hist_bins) - 1 + + # Display histogram in console + for idx_bin in range(hist_bins.size - 1): + tmp_idxs = np.where(idxs == idx_bin)[0] + Console.info(" Bin", idx_bin, "(", hist_bins[idx_bin], + "m < x <", hist_bins[idx_bin + 1], "m):", len(tmp_idxs), "images") + with tqdm_joblib( tqdm(desc="Computing altitude histogram", total=hist_bins.size - 1,) ): @@ -603,7 +607,7 @@ def generate_attenuation_correction_parameters(self): max_bin_size_gb, distance_vector, ) - for idx_bin in range(1, hist_bins.size) + for idx_bin in range(hist_bins.size - 1) ) # calculate attenuation parameters per channel @@ -764,7 +768,7 @@ def compute_distance_bin( ): dimensions = [self.image_height, self.image_width, self.image_channels] tmp_idxs = np.where(idxs == idx_bin)[0] - # Console.info("In bin", idx_bin, "there are", len(tmp_idxs), "images") + if len(tmp_idxs) > 2: bin_images = [self.camera_image_list[i] for i in tmp_idxs] bin_distances_sample = None @@ -818,7 +822,7 @@ def compute_distance_bin( plt.imshow(bin_images_sample) plt.colorbar() plt.title("Image bin " + str(idx_bin)) - fig_name = base_path / ("bin_images_sample_" + str(idx_bin) + ".png") + fig_name = base_path / ("bin_images_sample_" + str(distance_bin_sample) + "m.png") #Console.info("Saved figure at", fig_name) plt.savefig(fig_name, dpi=600) plt.close(fig) @@ -827,7 +831,7 @@ def compute_distance_bin( plt.imshow(bin_distances_sample) plt.colorbar() plt.title("Distance bin " + str(idx_bin)) - fig_name = base_path / ("bin_distances_sample_" + str(idx_bin) + ".png") + fig_name = base_path / ("bin_distances_sample_" + str(distance_bin_sample) + "m.png") #Console.info("Saved figure at", fig_name) plt.savefig(fig_name, dpi=600) plt.close(fig) diff --git a/src/correct_images/tools/curve_fitting.py b/src/correct_images/tools/curve_fitting.py index 33b0bcb9..f3e0414a 100644 --- a/src/correct_images/tools/curve_fitting.py +++ b/src/correct_images/tools/curve_fitting.py @@ -77,17 +77,41 @@ def curve_fitting(altitudes: np.ndarray, intensities: np.ndarray) -> np.ndarray: altitudes = altitudes[np.isfinite(altitudes)] intensities = intensities[np.isfinite(intensities)] + if altitudes.size == 0: + print("---------\naltitudes: ", altitudes, "\nintensities: ", intensities) + print("ERROR: Empty non-nan altitudes in curve fitting") + return np.array([1, 0, 0]) + if intensities.size == 0: + print("---------\naltitudes: ", altitudes, "\nintensities: ", intensities) + print("ERROR: Empty non-nan intensities in curve fitting") + return np.array([1, 0, 0]) + altitudes_filt = [] intensities_filt = [] for x, y in zip(altitudes, intensities): - if x > 0 and y > 0: + if x >= 0 and y >= 0: altitudes_filt.append(x) intensities_filt.append(y) + if not altitudes_filt or not intensities_filt: + if not altitudes_filt: + print("---------\naltitudes: ", altitudes, "\nintensities: ", intensities, "\naltitudes_filt: ", altitudes_filt, + "\nintensities_filt: ", intensities_filt) + print("ERROR: Altitudes are negative in curve fitting") + if not intensities_filt: + print("---------\naltitudes: ", altitudes, "\nintensities: ", intensities, "\naltitudes_filt: ", altitudes_filt, + "\nintensities_filt: ", intensities_filt) + print("ERROR: Intensities are negative in curve fitting") + return np.array([1, 0, 0]) + altitudes_filt = np.array(altitudes_filt) intensities_filt = np.array(intensities_filt) - c_upper_bound = intensities_filt.min() + try: + c_upper_bound = intensities_filt.min() + except ValueError: # raised if it is empty. + c_upper_bound = np.finfo(float).eps + if c_upper_bound <= 0: # c should be slightly greater than zero to avoid error # 'Each lower bound must be strictly less than each upper bound.' @@ -109,7 +133,11 @@ def curve_fitting(altitudes: np.ndarray, intensities: np.ndarray) -> np.ndarray: # Avoid zero divisions b = 0.0 - c = intensities_filt.min() * 0.5 + try: + c = intensities_filt.min() * 0.5 + except ValueError: # raised if it is empty. + c = np.finfo(float).eps + if intensities_filt[idx_1] != 0: b = (np.log((int_0 - c) / (int_1 - c))) / (alt_0 - alt_1) a = (int_1 - c) / np.exp(b * alt_1) @@ -152,5 +180,5 @@ def curve_fitting(altitudes: np.ndarray, intensities: np.ndarray) -> np.ndarray: return tmp_params.x except (ValueError, UnboundLocalError) as e: print("ERROR: Value Error due to Overflow", a, b, c) - print("Parameters calculated are unoptimised because of Value Error", e) + print("Parameters calculated are unoptimised because of error", e) return init_params