Merge pull request #434 from nasa/release/3.0.0-rc.2.1

Release 3.0.0-RC.2.1

.ci/scripts/disp_s1/build_disp_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3.1"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

.ci/scripts/dswx_s1/build_dswx_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4.2"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

.ci/scripts/dswx_s1/dswx_s1_calval_0.4_runconfig.yaml → .ci/scripts/dswx_s1/dswx_s1_calval_0.4.2_runconfig.yaml

@@ -63,7 +63,7 @@ RunConfig:
 
  PrimaryExecutable:
  ProductIdentifier: DSWX_S1
- ProductVersion: "0.1"
+ ProductVersion: "0.4"
  ProgramPath: python3
  ProgramOptions:
  - /home/dswx_user/OPERA/DSWX-SAR/src/dswx_sar/dswx_s1.py
@@ -157,7 +157,7 @@ RunConfig:
  product_path: /home/dswx_user/output_dir
  scratch_path: /home/dswx_user/scratch_dir
  sas_output_path: /home/dswx_user/output_dir
- product_version: 0.1
+ product_version: "0.4"
  output_imagery_format: 'COG'
 
  browse_image_group:
@@ -170,5 +170,6 @@ RunConfig:
  set_hand_mask_to_nodata: False
  set_layover_shadow_to_nodata: False
  set_ocean_masked_to_nodata: False
+ save_tif_to_output: True
 
  log_file: /home/dswx_user/output_dir/dswx-s1.log

.ci/scripts/dswx_s1/test_int_dswx_s1.sh

@@ -28,9 +28,9 @@ SAMPLE_TIME=1
 # RUNCONFIG should be the name of the runconfig in s3://operasds-dev-pge/dswx_s1/
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath "$(dirname "$(realpath "$0")")"/../../..)
 [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev"
-[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4_expected_input.zip"
-[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4_expected_output.zip"
-[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4_runconfig.yaml"
+[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4.2_expected_input.zip"
+[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4.2_expected_output.zip"
+[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4.2_runconfig.yaml"
 [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT"
 
 # Create the test output directory in the work space
@@ -124,7 +124,7 @@ else
  echo "output_file $output_file"
  output_file=$(basename -- "$output_file")
 
- if [[ "${output_file##*/}" == *.tif* ]]
+ if [[ "${output_file##*/}" == *B0*.tif* ]]
  then
  for potential_product in B01_WTR B02_BWTR B03_CONF B04_DIAG
  do
@@ -142,7 +142,7 @@ else
 
  echo "tile code is $tile_code"
 
- for potential_file in "$expected_data_dir"/*.tif*
+ for potential_file in "$expected_data_dir"/*B0*.tif*
  do
  if [[ "$potential_file" == *"$tile_code"*"$product"* ]]; then
  echo "expected file is $potential_file"

examples/disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml → examples/disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.0
+# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 

examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml → examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.0
+# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 
@@ -209,7 +209,7 @@ RunConfig:
  # SAS writes the output product(s) to the following file
  sas_output_path: /home/dswx_user/output_dir
 
- product_version: 0.1
+ product_version: "0.1"
 
  # DSWx-S1 product format
  output_imagery_format: 'COG'

src/opera/_package.py

@@ -7,7 +7,7 @@
 Package information for the OPERA PGE repository.
 """
 
-__version__ = "3.0.0-rc.2.0"
+__version__ = "3.0.0-rc.2.1"
 __title__ = "opera-sds-pge"
 __summary__ = "OPERA SDS Product Generation Executable (PGE) Repository"
 __uri__ = "https://github.com/nasa/opera-sds-pge"

src/opera/pge/base/runconfig.py

@@ -442,7 +442,7 @@ def get_ancillary_filenames(self):
  for ancillary_mapping in self.ancillary_file_map.values():
  if isinstance(ancillary_mapping, list):
  result.extend(ancillary_mapping)
- else:
+ elif ancillary_mapping is not None:
  result.append(ancillary_mapping)
 
  return result

src/opera/pge/disp_s1/disp_s1_pge.py

@@ -337,7 +337,7 @@ def _browse_filename(self, inter_filename):
  The file name to assign to browse image created by this PGE.
 
  """
- browse_image_filename = f"{self._core_filename(inter_filename)}.png"
+ browse_image_filename = f"{self._core_filename(inter_filename)}_BROWSE.png"
 
  return browse_image_filename
 
@@ -373,9 +373,11 @@ def _compressed_cslc_filename(self, inter_filename):
  Returns the file name to use for compressed CSLC files produced by the
  DISP-S1 PGE.
 
- The current convention is to maintain the filename assigned by the DISP-S1
- SAS, but this function is still required to ensure the compressed CSLC products
- are moved to the output directory defined by the RunConfig.
+ The compressed CSLC filename for the DISP-S1 PGE consists of:
+
+ <Project>_<Level>_COMPRESSED-CSLC-S1_<BurstID>_<ReferenceDateTime>_\
+ <FirstDateTime>_<LastDateTime>_<ProductGenerationDateTime>_\
+ <Polarization>_<ProductVersion>.h5
 
  Parameters
  ----------

src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2

@@ -976,6 +976,7 @@
  </gmd:description>
  </gmd:LI_Source>
  </gmd:source>
+ {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files is not none %}
  {%- for amplitude_dispersion_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files %}
  <gmd:source>
  <gmd:LI_Source>
@@ -985,6 +986,8 @@
  </gmd:LI_Source>
  </gmd:source>
  {%- endfor %}
+ {%- endif %}
+ {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files is not none %}
  {%- for amplitude_mean_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files %}
  <gmd:source>
  <gmd:LI_Source>
@@ -994,6 +997,8 @@
  </gmd:LI_Source>
  </gmd:source>
  {%- endfor %}
+ {%- endif %}
+ {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files is not none %}
  {%- for static_layers_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files %}
  <gmd:source>
  <gmd:LI_Source>
@@ -1003,6 +1008,8 @@
  </gmd:LI_Source>
  </gmd:source>
  {%- endfor %}
+ {%- endif %}
+ {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files is not none %}
  {%- for ionosphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files %}
  <gmd:source>
  <gmd:LI_Source>
@@ -1012,6 +1019,8 @@
  </gmd:LI_Source>
  </gmd:source>
  {%- endfor %}
+ {%- endif %}
+ {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files is not none %}
  {%- for troposphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files %}
  <gmd:source>
  <gmd:LI_Source>
@@ -1021,6 +1030,7 @@
  </gmd:LI_Source>
  </gmd:source>
  {%- endfor %}
+ {%- endif %}
  </gmd:LI_Lineage>
  </gmd:lineage>
  </gmd:DQ_DataQuality>

src/opera/pge/dswx_s1/dswx_s1_pge.py

@@ -10,6 +10,7 @@
 """
 
 import glob
+import os.path
 from datetime import datetime
 from os.path import abspath, basename, dirname, exists, getsize, join, splitext
 
@@ -150,12 +151,12 @@ def _validate_output(self):
  Evaluates the output file(s) generated from SAS execution to ensure:
  - That the file(s) contains some content (size is greater than 0).
  - That the .tif output files (band data) end with 'B01_WTR',
- 'B02_BWTR', 'B03_CONF' or 'B04_DIAG'
+ 'B02_BWTR', 'B03_CONF', 'B04_DIAG' or 'BROWSE'
  - That the there are the same number of each type of file, implying
  3 output bands per tile
 
  """
- EXPECTED_NUM_BANDS: int = 4
+ EXPECTED_NUM_BANDS: int = 5
  band_dict = {}
  num_bands = []
  output_extension = '.tif'
@@ -388,6 +389,9 @@ def _browse_filename(self, inter_filename):
  The file name to assign to browse image created by this PGE.
 
  """
+ # Save the extension of the intermediate filename
+ inter_ext = os.path.splitext(inter_filename)[-1]
+
  # Find one of the tif files corresponding to the provided browse image
  # based on the tile ID
  tile_id = basename(inter_filename).split('_')[3]
@@ -407,7 +411,7 @@ def _browse_filename(self, inter_filename):
  tile_filename = self._tile_filename(tif_files[0])
 
  # Add the portion specific to browse images
- return f"{tile_filename}_BROWSE.png"
+ return f"{tile_filename}_BROWSE{inter_ext}"
 
  def _ancillary_filename(self):
  """
@@ -785,7 +789,7 @@ class DSWxS1Executor(DSWxS1PreProcessorMixin, DSWxS1PostProcessorMixin, PgeExecu
  LEVEL = "L3"
  """Processing Level for DSWx-S1 Products"""
 
- SAS_VERSION = "0.4" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4
+ SAS_VERSION = "0.4.2" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4.2
  """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
  def __init__(self, pge_name, runconfig_path, **kwargs):
@@ -794,6 +798,6 @@ def __init__(self, pge_name, runconfig_path, **kwargs):
  # Used in base_pge.py to rename and keep track of files
  # renamed by the PGE
  self.rename_by_pattern_map = {
- '*.tif*': self._geotiff_filename,
- '*.png': self._browse_filename
+ '*B0*.tif*': self._geotiff_filename,
+ '*BROWSE*': self._browse_filename
  }

src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml

@@ -72,7 +72,7 @@ runconfig:
  # according to proper file naming conventions.
  sas_output_path: str()
 
- product_version: num(required=False)
+ product_version: str(required=False)
 
  # DSWx-S1 product format (default is 'COG')
  output_imagery_format: enum('GTiff', 'COG', required=False)
@@ -99,21 +99,24 @@ runconfig:
  browse_image_width: int(min=1, required=False)
 
  # Flag to collapse water classes if set to True. Default is True.
- flag_collapse_wtr_classes:  bool(required=False)
+ flag_collapse_wtr_classes: bool(required=False)
 
  # Flag to exclude inundated vegetation from processing if set to True.
- exclude_inundated_vegetation:  bool(required=False)
+ exclude_inundated_vegetation: bool(required=False)
 
  # Flag to set non-water pixels to NoData value if set to True.
- set_not_water_to_nodata:  bool(required=False)
+ set_not_water_to_nodata: bool(required=False)
 
  # Flag to set HAND mask pixels to NoData value if set to True.
- set_hand_mask_to_nodata:  bool(required=False)
+ set_hand_mask_to_nodata: bool(required=False)
 
  # Flag to set layover and shadow pixels to NoData value if set to True.
- set_layover_shadow_to_nodata:  bool(required=False)
+ set_layover_shadow_to_nodata: bool(required=False)
 
  # Flag to set ocean-masked pixels to NoData value if set to True.
- set_ocean_masked_to_nodata:  bool(required=False)
+ set_ocean_masked_to_nodata: bool(required=False)
 
- log_file: str(required=False)
+ # Flag to save Geotiff to output directory if set to True.
+ save_tif_to_output: bool(required=False)
+
+ log_file: str(required=False)

src/opera/test/data/invalid_runconfig.yaml

@@ -13,6 +13,7 @@ RunConfig:
  DynamicAncillaryFilesGroup:
  AncillaryFileMap:
  DEMFile: input/input_dem.vrt
+ IonosphereFiles:
 
  ProductPathGroup:
  OutputProductPath: outputs/

src/opera/test/data/test_dswx_s1_config.yaml

@@ -27,6 +27,7 @@ RunConfig:
  - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B03_CONF.tif;
  - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B04_DIAG.tif;
  - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.png;
+ - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.tif;
  - /bin/echo DSWx-S1 invoked with RunConfig
  ErrorCodeBase: 400000
  SchemaPath: pge/dswx_s1/schema/dswx_s1_sas_schema.yaml

src/opera/test/data/valid_runconfig_extra_fields.yaml

@@ -18,6 +18,7 @@ RunConfig:
  DynamicAncillaryFilesGroup:
  AncillaryFileMap:
  DEMFile: input/input_dem.vrt
+ IonosphereFiles:
 
  ProductPathGroup:
  OutputProductPath: outputs/

src/opera/test/data/valid_runconfig_full.yaml

@@ -15,6 +15,7 @@ RunConfig:
  DynamicAncillaryFilesGroup:
  AncillaryFileMap:
  DEMFile: input/input_dem.vrt
+ IonosphereFiles:
 
  ProductPathGroup:
  OutputProductPath: outputs/

src/opera/test/data/valid_runconfig_no_sas.yaml

@@ -15,6 +15,7 @@ RunConfig:
  DynamicAncillaryFilesGroup:
  AncillaryFileMap:
  DEMFile: input/input_dem.vrt
+ IonosphereFiles:
 
  ProductPathGroup:
  OutputProductPath: outputs/

src/opera/test/pge/base/test_runconfig.py

@@ -43,7 +43,7 @@ def _compare_runconfig_to_expected(self, runconfig):
  self.assertEqual(runconfig.name, "OPERA-SAMPLE-PGE-SAS-CONFIG")
  self.assertEqual(runconfig.pge_name, "EXAMPLE_PGE")
  self.assertListEqual(runconfig.input_files, ["input/input_file01.h5", "input/input_file02.h5"])
- self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt"})
+ self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt", 'IonosphereFiles': None})
  self.assertEqual(runconfig.output_product_path, "outputs/")
  self.assertEqual(runconfig.scratch_path, "temp/")
  self.assertEqual(runconfig.product_identifier, "EXAMPLE")
@@ -82,6 +82,13 @@ def test_full_pge_config_parse_and_validate(self):
  # this test
  self.assertIsInstance(runconfig.sas_config, dict)
 
+ # Ensure empty ancillary fields are filtered out of list returned
+ # from get_ancillary_filenames()
+ ancillary_filenames = runconfig.get_ancillary_filenames()
+
+ self.assertIn("input/input_dem.vrt", ancillary_filenames)
+ self.assertNotIn(None, ancillary_filenames)
+
  def test_pge_only_config_parse_and_validate(self):
  """
  Test basic parsing and validation of an input RunConfig that only
@@ -104,6 +111,13 @@ def test_pge_only_config_parse_and_validate(self):
  # Check that None was assigned for SAS config section
  self.assertIsNone(runconfig.sas_config)
 
+ # Ensure empty ancillary fields are filtered out of list returned
+ # from get_ancillary_filenames()
+ ancillary_filenames = runconfig.get_ancillary_filenames()
+
+ self.assertIn("input/input_dem.vrt", ancillary_filenames)
+ self.assertNotIn(None, ancillary_filenames)
+
  def test_strict_mode_validation(self):
  """Test validation of a RunConfig with strict_mode both enabled and disabled"""
  # Parse a valid runconfig, but modify it with fields not in the base