From f4bc676b2b3dd2b93250a15821c0c8221c1cbd62 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 29 Jun 2022 16:32:32 -0500 Subject: [PATCH 01/12] add _find_hdr --- plantcv/plantcv/hyperspectral/read_data.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index 0909947bf..ae8b06745 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -1,6 +1,7 @@ # Read in a hyperspectral data cube as an array and parse metadata from the header file import os +import re import cv2 import numpy as np from plantcv.plantcv import params @@ -90,6 +91,32 @@ def _make_pseudo_rgb(spectral_array): return pseudo_rgb +def _find_hdr(filename): + """Find a header file paired with an hyperspectral data file. + + Keyword arguments: + filename = File path/name of a hyperspectral data file. + + Returns: + hdrfile = File path/name of hyperspectral header file. + + :param filename: str + :return hdrfile: str + """ + # Split the filename into the path and name + dat_path, dat_filename = os.path.split(filename) + # Extract the base of the data file name and the extension, if any + dat_basename, dat_ext = os.path.splitext(dat_filename) + # Create a regular expression for the header file + # The data file extension is included optionally + hdr_regex = re.compile(f"{dat_basename}({dat_ext})?.hdr") + # List all the files in the data file directory + for fname in os.listdir(dat_path): + # If the filename matches, return the header file path + if hdr_regex.match(fname): + return os.path.join(dat_path, fname) + + def read_data(filename): """Read hyperspectral image data from file. Inputs: From dad12a9bdac895f5913eeab639e61db90bdd08a2 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 29 Jun 2022 16:33:59 -0500 Subject: [PATCH 02/12] implement helper function --- plantcv/plantcv/hyperspectral/read_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index ae8b06745..fe541de50 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -132,8 +132,7 @@ def read_data(filename): header_dict = {} # Remove any file extension and set .hdr filename - filename_base = os.path.splitext(filename)[0] - headername = filename_base + ".hdr" + headername = _find_hdr(filename=filename) with open(headername, "r") as f: # Replace characters for easier parsing From 9467a9d2b0dce66492df9db9842b6f71c4f74f1a Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Thu, 30 Jun 2022 07:26:26 -0500 Subject: [PATCH 03/12] limit numpy version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d867203a1..222b323bc 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ name: plantcv dependencies: - python=3.9 - matplotlib>=1.5 - - numpy>=1.11 + - numpy>=1.11,<1.23 - pandas - python-dateutil - scipy From da8e5c6da8611bede7c39dd0d53c8f91ef342c3b Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Thu, 30 Jun 2022 08:36:55 -0500 Subject: [PATCH 04/12] add to env --- environment.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/environment.yml b/environment.yml index 222b323bc..1ac9c41d6 100644 --- a/environment.yml +++ b/environment.yml @@ -18,6 +18,11 @@ dependencies: - statsmodels - mkdocs - pytest + - pytest-cov + - flake8 + - ipympl + - nodejs + - jupyterlab channels: - conda-forge - defaults From 400c59918b3fbf6d7cf89a733cad18fde4d448a7 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Fri, 1 Jul 2022 10:22:06 -0500 Subject: [PATCH 05/12] update environment.txt and restore yml --- environment.yml | 6 +----- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index 1ac9c41d6..4a02c7f61 100644 --- a/environment.yml +++ b/environment.yml @@ -18,11 +18,7 @@ dependencies: - statsmodels - mkdocs - pytest - - pytest-cov - - flake8 - - ipympl - - nodejs - - jupyterlab + channels: - conda-forge - defaults diff --git a/requirements.txt b/requirements.txt index 966b96532..e96a7a139 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ matplotlib>=1.5 -numpy>=1.11 +numpy>=1.11,<1.23 pandas python-dateutil scipy From c2e15e434d92e39f6f47e3717032efb7cffc5e8d Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 6 Jul 2022 10:57:56 -0500 Subject: [PATCH 06/12] change hdr parsing method remove white space, split on "=" or ":" and remove whitespace from keys to match new strings --- plantcv/plantcv/hyperspectral/read_data.py | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index fe541de50..17d3d65f9 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -147,13 +147,16 @@ def read_data(filename): hdata = hdata.split("\n") # Loop through and create a dictionary from the header file + # Try to reformat strings by replacing all " = " with '=' and " : " for string in hdata: - if ' = ' in string: - header_data = string.split(" = ") + # Remove white space for consistency across header file formats + string = string.replace(' ', '') + if '=' in string: + header_data = string.split("=") header_data[0] = header_data[0].lower() header_dict.update({header_data[0].rstrip(): header_data[1].rstrip()}) - elif ' : ' in string: - header_data = string.split(" : ") + elif ':' in string: + header_data = string.split(":") header_data[0] = header_data[0].lower() header_dict.update({header_data[0].rstrip(): header_data[1].rstrip()}) @@ -171,10 +174,10 @@ def read_data(filename): # Replace datatype ID number with the numpy datatype dtype_dict = {"1": np.uint8, "2": np.int16, "3": np.int32, "4": np.float32, "5": np.float64, "6": np.complex64, "9": np.complex128, "12": np.uint16, "13": np.uint32, "14": np.int64, "15": np.uint64} - header_dict["data type"] = dtype_dict[header_dict["data type"]] + header_dict["datatype"] = dtype_dict[header_dict["datatype"]] # Read in the data from the file - raw_data = np.fromfile(filename, header_dict["data type"], -1) + raw_data = np.fromfile(filename, header_dict["datatype"], -1) # Reshape the raw data into a datacube array data_format = { @@ -205,16 +208,16 @@ def read_data(filename): # Check for default bands (that get used to make pseudo_rgb image) default_bands = None - if "default bands" in header_dict: - header_dict["default bands"] = header_dict["default bands"].replace("{", "") - header_dict["default bands"] = header_dict["default bands"].replace("}", "") - default_bands = header_dict["default bands"].split(",") + if "defaultbands" in header_dict: + header_dict["defaultbands"] = header_dict["defaultbands"].replace("{", "") + header_dict["defaultbands"] = header_dict["defaultbands"].replace("}", "") + default_bands = header_dict["defaultbands"].split(",") # Find array min and max values max_pixel = float(np.amax(array_data)) min_pixel = float(np.amin(array_data)) - wavelength_units = header_dict.get("wavelength units") + wavelength_units = header_dict.get("wavelengthunits") if wavelength_units is None: wavelength_units = "nm" @@ -223,7 +226,7 @@ def read_data(filename): max_wavelength=float(str(header_dict["wavelength"][-1]).rstrip()), min_wavelength=float(str(header_dict["wavelength"][0]).rstrip()), max_value=max_pixel, min_value=min_pixel, - d_type=header_dict["data type"], + d_type=header_dict["datatype"], wavelength_dict=wavelength_dict, samples=int(header_dict["samples"]), lines=int(header_dict["lines"]), interleave=header_dict["interleave"], wavelength_units=wavelength_units, array_type="datacube", From b68578803509172c35f8d17bfdc0a96f1c4f0624 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 6 Jul 2022 15:48:07 -0500 Subject: [PATCH 07/12] else return none for deepsource --- plantcv/plantcv/hyperspectral/read_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index 17d3d65f9..e33f47926 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -115,6 +115,8 @@ def _find_hdr(filename): # If the filename matches, return the header file path if hdr_regex.match(fname): return os.path.join(dat_path, fname) + else: + return None def read_data(filename): From 6c9c4a7f270c220ac0648670f7502244702a69ca Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 6 Jul 2022 15:52:14 -0500 Subject: [PATCH 08/12] update logic --- plantcv/plantcv/hyperspectral/read_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index e33f47926..440f1115b 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -115,8 +115,7 @@ def _find_hdr(filename): # If the filename matches, return the header file path if hdr_regex.match(fname): return os.path.join(dat_path, fname) - else: - return None + return None def read_data(filename): From 3254b5cb5283bfb1b48470d7ec8fd886b4954b34 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 6 Jul 2022 15:59:25 -0500 Subject: [PATCH 09/12] add test for bad filename --- tests/plantcv/hyperspectral/conftest.py | 1 + tests/plantcv/hyperspectral/test_read_data.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests/plantcv/hyperspectral/conftest.py b/tests/plantcv/hyperspectral/conftest.py index 83d6f9c9c..3b1141ff1 100644 --- a/tests/plantcv/hyperspectral/conftest.py +++ b/tests/plantcv/hyperspectral/conftest.py @@ -16,6 +16,7 @@ def __init__(self): self.envi_no_default = os.path.join(self.datadir, "darkReference2") self.envi_appox_pseudo = os.path.join(self.datadir, "darkReference3") self.envi_bad_interleave = os.path.join(self.datadir, "darkReference4") + self.bad_filename = os.path.join(self.datadir, "darkReference0") self.hsi_file = os.path.join(self.datadir, "hsi.pkl") self.hsi_mask_file = os.path.join(self.datadir, "hsi_mask.png") self.hsi_whiteref_file = os.path.join(self.datadir, "hsi_whiteref.pkl") diff --git a/tests/plantcv/hyperspectral/test_read_data.py b/tests/plantcv/hyperspectral/test_read_data.py index 0075204f3..5cc999df2 100644 --- a/tests/plantcv/hyperspectral/test_read_data.py +++ b/tests/plantcv/hyperspectral/test_read_data.py @@ -25,3 +25,8 @@ def test_read_data_bad_interleave(hyperspectral_test_data): """Test for PlantCV.""" with pytest.raises(RuntimeError): _ = read_data(filename=hyperspectral_test_data.envi_bad_interleave) + +def test_read_data_bad_filename(hyperspectral_test_data): + """Test for PlantCV.""" + with pytest.raises(RuntimeError): + _ = read_data(filename=hyperspectral_test_data.bad_filename) \ No newline at end of file From 29db99ddadd86bfb491eacf3cc543bc8244b5971 Mon Sep 17 00:00:00 2001 From: HaleySchuhl Date: Wed, 6 Jul 2022 16:05:35 -0500 Subject: [PATCH 10/12] add fatal error --- plantcv/plantcv/hyperspectral/read_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index 440f1115b..dba8290f5 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -135,6 +135,9 @@ def read_data(filename): # Remove any file extension and set .hdr filename headername = _find_hdr(filename=filename) + if headername is None: + fatal_error("Unable to find the header file corresponding to " + filename) + with open(headername, "r") as f: # Replace characters for easier parsing hdata = f.read() From 20393a110f6342283d6983f0f7fd452194614b85 Mon Sep 17 00:00:00 2001 From: nfahlgren Date: Thu, 7 Jul 2022 11:35:26 -0500 Subject: [PATCH 11/12] Replace string concatenation with an f-string --- plantcv/plantcv/hyperspectral/read_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index dba8290f5..b04e76abd 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -136,7 +136,7 @@ def read_data(filename): headername = _find_hdr(filename=filename) if headername is None: - fatal_error("Unable to find the header file corresponding to " + filename) + fatal_error(f"Unable to find the header file corresponding to {filename}") with open(headername, "r") as f: # Replace characters for easier parsing From 29dda173e0ccb593e37c258ad27eb546fa8bedca Mon Sep 17 00:00:00 2001 From: nfahlgren Date: Thu, 7 Jul 2022 11:39:55 -0500 Subject: [PATCH 12/12] Fix spacing Two line returns between functions and one blank line at the end of a file --- tests/plantcv/hyperspectral/test_read_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/plantcv/hyperspectral/test_read_data.py b/tests/plantcv/hyperspectral/test_read_data.py index 5cc999df2..f3659dbb2 100644 --- a/tests/plantcv/hyperspectral/test_read_data.py +++ b/tests/plantcv/hyperspectral/test_read_data.py @@ -26,7 +26,8 @@ def test_read_data_bad_interleave(hyperspectral_test_data): with pytest.raises(RuntimeError): _ = read_data(filename=hyperspectral_test_data.envi_bad_interleave) + def test_read_data_bad_filename(hyperspectral_test_data): """Test for PlantCV.""" with pytest.raises(RuntimeError): - _ = read_data(filename=hyperspectral_test_data.bad_filename) \ No newline at end of file + _ = read_data(filename=hyperspectral_test_data.bad_filename)