Delete test data from repo and download it from CDSE

2 GB of test data seemed to large to keep in the repository.

.gitignore

@@ -139,9 +139,11 @@ Makefile
 tests/
 
 test_data/fusion_results
+test_data/S2/raw
 test_data/S2/processed
 test_data/S3/reprojected
 test_data/S3/composites
 test_data/S3/calibrated
 test_data/S3/binning
 test_data/S3/blurred
+test_data/S3/raw

efast/s2_processing.py

@@ -76,7 +76,7 @@ def extract_mask_s2_bands(
     -------
     None
     """
-    for p in input_dir.iterdir():
+    for p in input_dir.glob("*.SAFE"):
         band_paths = [
             list(p.glob(f"GRANULE/*/IMG_DATA/R{resolution}m/*{band}*.jp2"))[0]
             for band in bands

requirements.txt

@@ -1,11 +1,13 @@
-setuptools
+astropy
+ipdb
 numpy
-scipy
-tqdm
-scikit-learn
-rasterio
 pandas
-ipdb
-astropy
 python-dateutil
+rasterio
+scipy
+scikit-learn
+setuptools
+tqdm
+zipfile
 git+https://github.com/DHI-GRAS/snap-graph
+git+https://github.com/DHI-GRAS/creodias-finder

run_efast.py

@@ -30,12 +30,20 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 
+from creodias_finder import query, download
 from dateutil import rrule
+import zipfile
 
 import efast.efast as efast
 import efast.s2_processing as s2
 import efast.s3_processing as s3
 
+# CDSE credentials to download Sentinel-2 and Sentinel-3 imagery
+CREDENTIALS = {
+    "username": "my-cdse-email",
+    "password": "my-cdse-password"
+}
+
 # Test parameters
 path = Path("./test_data").absolute()
 s3_download_dir = path / "S3/raw"
@@ -52,11 +60,13 @@
 def main(
     start_date: str,
     end_date: str,
+    aoi_geometry: str,
     s3_sensor: str,
     s3_bands: list,
     s2_bands: list,
     mosaic_days: int,
     step: int,
+    cdse_credentials: dict,
     snap_gpt_path: str = "gpt",
 ):
     # Transform parameters
@@ -69,16 +79,27 @@ def main(
 
     # Create directories if necessary
     for folder in [
+        s3_download_dir,
         s3_binning_dir,
         s3_composites_dir,
         s3_blured_dir,
         s3_calibrated_dir,
         s3_reprojected_dir,
         s2_processed_dir,
+        s2_download_dir,
         fusion_dir,
     ]:
         folder.mkdir(parents=True, exist_ok=True)
 
+    # Download the data from CDSE
+    download_from_cdse(
+        start_date,
+        end_date,
+        aoi_geometry,
+        s2_download_dir,
+        s3_download_dir,
+        cdse_credentials)
+
     # Sentinel-2 pre-processing
     s2.extract_mask_s2_bands(
         s2_download_dir,
@@ -145,28 +166,71 @@ def main(
         )
 
 
+def download_from_cdse(
+        start_date,
+        end_date,
+        aoi_geometry,
+        s2_download_dir,
+        s3_download_dir,
+        credentials):
+
+    # First download Sentinel-3 SYN data
+    results = query.query('Sentinel3',
+                          start_date=start_date,
+                          end_date=end_date,
+                          geometry=aoi_geometry,
+                          instrument="SYNERGY",
+                          productType="SY_2_SYN___",
+                          timeliness="NT")
+    download.download_list([result['id'] for result in results.values()],
+                           outdir=s3_download_dir,
+                           threads=3,
+                           **credentials)
+    for zip_file in s3_download_dir.glob("*.zip"):
+        with zipfile.ZipFile(zip_file, "r") as zip_ref:
+            zip_ref.extractall(s3_download_dir)
+
+    # Then download Sentinel-2 L2A data
+    results = query.query('Sentinel2',
+                          start_date=start_date,
+                          end_date=end_date,
+                          geometry=aoi_geometry,
+                          productType="L2A")
+    download.download_list([result['id'] for result in results.values()],
+                           outdir=s2_download_dir,
+                           threads=3,
+                           **credentials)
+    for zip_file in s2_download_dir.glob("*.zip"):
+        with zipfile.ZipFile(zip_file, "r") as zip_ref:
+            zip_ref.extractall(s2_download_dir)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--start-date", default="2023-09-11")
     parser.add_argument("--end-date", default="2023-09-21")
+    parser.add_argument("--aoi-geometry", default="POINT (-15.432283 15.402828)")  # Dahra EC tower
     parser.add_argument("--s3-sensor", default="SYN")
     parser.add_argument(
         "--s3-bands", default=["SDR_Oa04", "SDR_Oa06", "SDR_Oa08", "SDR_Oa17"]
     )
     parser.add_argument("--s2-bands", default=["B02", "B03", "B04", "B8A"])
     parser.add_argument("--mosaic-days", default=100)
     parser.add_argument("--step", required=False, default=2)
+    parser.add_argument("--cdse-credentials", default=CREDENTIALS)
     parser.add_argument("--snap-gpt-path", required=False, default="gpt")
 
     args = parser.parse_args()
 
     main(
         start_date=args.start_date,
         end_date=args.end_date,
+        aoi_geometry=args.aoi_geometry,
         s3_sensor=args.s3_sensor,
         s3_bands=args.s3_bands,
         s2_bands=args.s2_bands,
         step=args.step,
         mosaic_days=args.mosaic_days,
-        snap_gpt_path=args.snap_gpt_path,
+        cdse_credentials=args.cdse_credentials,
+        snap_gpt_path=args.snap_gpt_path
     )