added 20m as asset export

-added 20m as asset export
swisstopo · Dec 20, 2023 · 3bc2db0 · 3bc2db0
1 parent 3410a6c
commit 3bc2db0
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 39 deletions.
diff --git a/configuration/dev_config.py b/configuration/dev_config.py
@@ -51,27 +51,27 @@
 # NDVI product parameters
 PRODUCT_NDVI_MAX = {
     "prefix": "Sentinel_NDVI-MAX_SR_CloudFree_crop",
-    "image_collection": "COPERNICUS/S2_SR_HARMONIZED",
+    "image_collection": "COPERNICUS/S2_SR_HARMONIZED",  # might be obsolete with step0?
     "temporal_coverage": 8,  # Days
     "spatial_scale_export": 10,  # Meters
     "band_names": [{'NIR': "B8", 'RED': "B4"}],
     "product_name": "NDVI-MAX",
-    "step0_collection": "projects/geetest-386915/assets/col_s2_sr"
+    "step0_collection": "projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS"
 }
 
 PRODUCT_S2_LEVEL_2A = {
     "prefix": "S2_L2A_SR",
-    "image_collection": "COPERNICUS/S2_SR_HARMONIZED",
+    "image_collection": "COPERNICUS/S2_SR_HARMONIZED",  # might be obsolete with step0?
     "temporal_coverage": 1,  # Days
     "spatial_scale_export": 10,  # Meters
     "spatial_scale_export_qa60": 60,  # Meters
     "product_name": "S2_LEVEL_2A",
-    "step0_collection": "projects/geetest-386915/assets/col_s2_sr"
+    "step0_collection": "projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS"
 }
 
 PRODUCT_S2_LEVEL_1C = {
     "prefix": "S2_L1C_TOA",
-    "image_collection": "COPERNICUS/S2_HARMONIZED",
+    "image_collection": "COPERNICUS/S2_HARMONIZED",  # might be obsolete with step0?
     "temporal_coverage": 30,  # Days
     "spatial_scale_export": 10,  # Meters
     "spatial_scale_export_mask60": 60,
@@ -81,7 +81,7 @@
 
 PRODUCT_NDVI_MAX_TOA = {
     "prefix": "Sentinel_NDVI-MAX_TOA_CloudFree_crop",
-    "image_collection": "COPERNICUS/S2_HARMONIZED",
+    "image_collection": "COPERNICUS/S2_HARMONIZED",  # might be obsolete with step0?
     "temporal_coverage": 1,  # Days
     "spatial_scale_export": 1,  # Meters
     "band_names": [{'NIR': "B8", 'RED': "B4"}],
@@ -92,13 +92,13 @@
 # dictionary used to manage custom asset,
 # for example to clear old images not used anymore.
 
-
+# Additional personal collections
 step0 = {
-    'projects/satromo-exolabs/assets/col_s2_toa': {
-        'step0_function': 'step0_processor_s2_toa.generate_s2_toa_mosaic_for_single_date',
-        # cleaning_older_than: 2 # entry used to clean assets
-    },
-    'projects/satromo-exolabs/assets/col_s2_sr': {
+    # 'projects/satromo-exolabs/assets/col_s2_toa': {
+    #    'step0_function': 'step0_processor_s2_toa.generate_s2_toa_mosaic_for_single_date',
+    #    # cleaning_older_than: 2 # entry used to clean assets
+    # },
+    'projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS': {
         'step0_function': 'step0_processor_s2_sr.generate_s2_sr_mosaic_for_single_date'
         # cleaning_older_than: 2 # entry used to clean assets
     }

diff --git a/satromo_processor.py b/satromo_processor.py
@@ -177,8 +177,10 @@ def get_collection_info(collection):
     last_image = sorted_collection.sort('system:time_start', False).first()
 
     # Get the dates of the first and last image
-    first_date = ee.Date(first_image.get('system:time_start')).format('YYYY-MM-dd').getInfo()
-    last_date = ee.Date(last_image.get('system:time_start')).format('YYYY-MM-dd').getInfo()
+    first_date = ee.Date(first_image.get('system:time_start')
+                         ).format('YYYY-MM-dd').getInfo()
+    last_date = ee.Date(last_image.get('system:time_start')
+                        ).format('YYYY-MM-dd').getInfo()
 
     # Get the count of images in the filtered collection
     image_count = collection.size()
@@ -544,7 +546,8 @@ def process_NDVI_MAX(roi):
         timestamp = timestamp.strftime('%Y%m%dT235959')
 
         # Generate the filename
-        filename = config.PRODUCT_NDVI_MAX['prefix'] + '_' + timestamp + '_10m.tif'
+        filename = config.PRODUCT_NDVI_MAX['prefix'] + \
+            '_' + timestamp + '_10m.tif'
         print(filename)
 
         # Check if there is at least 1 scene to be defined (if minimal scene count is required) TODO: is this necessary?
@@ -637,7 +640,8 @@ def add_footprint(image, lst):
                        multiband_export, sensor_stats, current_date_str)
 
         # Export QA60 band as a separate GeoTIFF with '_QA60'
-        masks_export = clipped_image.select(['terrainShadowMask', 'cloudAndCloudShadowMask'])
+        masks_export = clipped_image.select(
+            ['terrainShadowMask', 'cloudAndCloudShadowMask'])
         masks_export_name = mosaic_id.replace('_bands-10m', '_masks-10m')
         prepare_export(clipped_image_bounding_box, mosaic_sensing_timestamp, masks_export_name,
                        config.PRODUCT_S2_LEVEL_2A['product_name'],
@@ -726,13 +730,15 @@ def add_footprint(image, lst):
                        multiband_export, sensor_stats, current_date_str)
 
         # Export QA60 band as a separate GeoTIFF with '_QA60'
-        masks_export = clipped_image.select(['terrainShadowMask', 'cloudAndCloudShadowMask'])
+        masks_export = clipped_image.select(
+            ['terrainShadowMask', 'cloudAndCloudShadowMask'])
         masks_export_name = mosaic_id.replace('_bands-10m', '_masks-10m')
         prepare_export(clipped_image_bounding_box, mosaic_sensing_timestamp, masks_export_name,
                        config.PRODUCT_S2_LEVEL_1C['product_name'],
                        config.PRODUCT_S2_LEVEL_1C['spatial_scale_export_mask'], masks_export,
                        sensor_stats, current_date_str)
 
+
 def process_NDVI_MAX_TOA(roi):
     """
     Process the NDVI MAX product for TOA.
@@ -781,7 +787,8 @@ def process_NDVI_MAX_TOA(roi):
         timestamp = timestamp.strftime('%Y%m%dT235959')
 
         # Generate the filename
-        filename = config.PRODUCT_NDVI_MAX_TOA['prefix'] + '_' + timestamp + '_10m.tif'
+        filename = config.PRODUCT_NDVI_MAX_TOA['prefix'] + \
+            '_' + timestamp + '_10m.tif'
         print(filename)
 
         # Start the export
@@ -801,17 +808,17 @@ def process_NDVI_MAX_TOA(roi):
     current_date_str = datetime.datetime.today().strftime('%Y-%m-%d')
 
     # For debugging
-    # current_date_str = "2023-06-12"
-    # print("*****************************\n")
-    # print("using a manual set Date: "+current_date_str)
-    # print("*****************************\n")
+    current_date_str = "2023-11-01"
+    print("*****************************\n")
+    print("using a manual set Date: "+current_date_str)
+    print("*****************************\n")
 
     current_date = ee.Date(current_date_str)
 
     roi = ee.Geometry.Rectangle(config.ROI_RECTANGLE)
     step0_product_dict = get_step0_dict()
     print(step0_product_dict)
-
+    
     collections_ready_for_processors = step0_main(
         step0_product_dict, current_date_str)
     print(collections_ready_for_processors)
@@ -825,7 +832,8 @@ def process_NDVI_MAX_TOA(roi):
                 result = process_NDVI_MAX(roi)
 
             elif product_to_be_processed == 'PRODUCT_S2_LEVEL_2A':
-                border = ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017").filter(ee.Filter.eq("country_co", "SZ"))
+                border = ee.FeatureCollection(
+                    "USDOS/LSIB_SIMPLE/2017").filter(ee.Filter.eq("country_co", "SZ"))
                 roi = border.geometry().buffer(config.ROI_BORDER_BUFFER)
                 # roi = ee.Geometry.Rectangle( [ 7.075402, 46.107098, 7.100894, 46.123639])
                 result = process_S2_LEVEL_2A(roi)

diff --git a/step0_processors/step0_processor_s2_sr.py b/step0_processors/step0_processor_s2_sr.py
@@ -45,8 +45,8 @@ def generate_s2_sr_mosaic_for_single_date(day_to_process: str, collection: str,
     # Export switches
     # options': True, 'False - defines if 10-m-bands are exported': 'B2','B3','B4','B8'
     export10mBands = True
-    # options': True, 'False - defines if 20-m-bands are exported': 'B5','B6','B7','B8A','B11','B12'
-    # export20mBands = False  # NOTEJS: ununsed, export function commented in the script below
+    # options': True, 'False - defines if 20-m-bands are exported':  select from 'B5','B6','B7','B8A','B11','B12'below
+    export20mBands = True
     # options': True, 'False - defines if 60-m-bands are exported': 'B1','B9','B10'
     # export60mBands = False  # NOTEJS: ununsed, export function commented in the script below
     # options': True, 'False - defines if registration layers are exported': 'reg_dx','reg_dy', 'reg_confidence'
@@ -70,7 +70,8 @@ def generate_s2_sr_mosaic_for_single_date(day_to_process: str, collection: str,
     # processing: reprojected in QGIS to epsg32632
     aoi_CH = ee.FeatureCollection(
         "users/wulf/SATROMO/swissBOUNDARIES3D_1_4_TLM_LANDESGEBIET_epsg32632").geometry()
-    aoi_CH_simplified = ee.FeatureCollection("users/wulf/SATROMO/CH_boundaries_buffer_5000m_epsg32632").geometry()
+    aoi_CH_simplified = ee.FeatureCollection(
+        "users/wulf/SATROMO/CH_boundaries_buffer_5000m_epsg32632").geometry()
 
     ##############################
     # REFERENCE DATA
@@ -399,7 +400,8 @@ def S2regFunc(image):
         reg_dx = reg_dx.multiply(100).round().toInt16()
         reg_dy = displacement.select('dy').rename('reg_dy')
         reg_dy = reg_dy.multiply(100).round().toInt16()
-        reg_confidence = displacement.select('confidence').rename('reg_confidence')
+        reg_confidence = displacement.select(
+            'confidence').rename('reg_confidence')
         reg_confidence = reg_confidence.multiply(100).round().toUint8()
 
         # Compute image offset and direction.
@@ -427,10 +429,8 @@ def S2regFunc(image):
 
     # extract the date and time (it is same time for all images in the mosaic)
     sensing_date = S2_sr.get('system:index').getInfo()[0:15]
-    sensing_date_read = sensing_date[0:4] + '-' + sensing_date[4:6] + '-' + sensing_date[6:15]
-
-    # define the filenames
-    fname_10m = 'S2-L2A_mosaic_' + sensing_date_read + '_bands-10m'
+    sensing_date_read = sensing_date[0:4] + '-' + \
+        sensing_date[4:6] + '-' + sensing_date[6:15]
 
     # define the export aoi
     # the full mosaic image geometry covers larger areas outside Switzerland that are not needed
@@ -442,6 +442,8 @@ def S2regFunc(image):
     # SWITCH export
     if export10mBands is True:
         print('Launching export for 10m bands')
+        # define the filenames
+        fname_10m = 'S2-L2A_mosaic_' + sensing_date_read + '_bands-10m'
         band_list = ['B2', 'B3', 'B4', 'B8']
         if exportMasks:
             band_list.extend(['terrainShadowMask', 'cloudAndCloudShadowMask'])
@@ -462,22 +464,24 @@ def S2regFunc(image):
         )
         task.start()
 
-    """"# SWITCH export
+    # SWITCH export
     if export20mBands is True:
-        fname_20m = 'S2-L2A_Mosaic_' + sensing_date_read + '_Bands-20m'
+        print('Launching export for 20m bands')
+        # define the filenames
+        fname_20m = 'S2-L2A_Mosaic_' + sensing_date_read + '_bands-20m'
         # Export COG 20m bands
-        task = ee.batch.Export.image.toDrive(
-            image=S2_sr.select(['B5', 'B6', 'B7', 'B8A', 'B11', 'B12']),
+        task = ee.batch.Export.image.toAsset(
+            image=S2_sr.select(['B8A', 'B11']).clip(aoi_exp),
             scale=20,
-            description=fname_20m,
+            description=task_description,
             crs='EPSG:2056',
             region=aoi_exp,
             maxPixels=1e10,
-            assetId=fname_20m
+            assetId=collection + '/' + fname_20m
         )
         task.start()
 
-    # SWITCH export
+    """"# SWITCH export
     if export60mBands is True:
         fname_60m = 'S2-L2A_Mosaic_' + sensing_date_read + '_Bands-60m'
         task = ee.batch.Export.image.toDrive(

diff --git a/tools/step0_empty_assets.csv b/tools/step0_empty_assets.csv
@@ -35,3 +35,18 @@ COL_S2_HARMONIZED_SWISS,2023-12-02,cloudy
 COL_S2_HARMONIZED_SWISS,2023-12-04,No scene
 COL_S2_HARMONIZED_SWISS,2023-12-06,No scene
 COL_S2_HARMONIZED_SWISS,2023-12-08,No scene
+col_s2_sr,2023-06-09,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-07,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-12,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-17,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-19,cloudy
+COL_S2_HARMONIZED_SWISS,2023-10-20,cloudy
+COL_S2_HARMONIZED_SWISS,2023-10-22,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-23,cloudy
+COL_S2_HARMONIZED_SWISS,2023-10-24,cloudy
+COL_S2_HARMONIZED_SWISS,2023-10-26,cloudy
+COL_S2_HARMONIZED_SWISS,2023-10-27,No candidate scene
+COL_S2_HARMONIZED_SWISS,2023-10-30,cloudy
+COL_S2_HARMONIZED_SWISS,2023-11-01,No candidate scene
+COL_S2_SR_HARMONIZED_SWISS,2023-10-27,No candidate scene
+COL_S2_SR_HARMONIZED_SWISS,2023-11-01,No candidate scene