diff --git a/configuration/dev_config.py b/configuration/dev_config.py index 38fedbb8..51590670 100644 --- a/configuration/dev_config.py +++ b/configuration/dev_config.py @@ -51,27 +51,27 @@ # NDVI product parameters PRODUCT_NDVI_MAX = { "prefix": "Sentinel_NDVI-MAX_SR_CloudFree_crop", - "image_collection": "COPERNICUS/S2_SR_HARMONIZED", + "image_collection": "COPERNICUS/S2_SR_HARMONIZED", # might be obsolete with step0? "temporal_coverage": 8, # Days "spatial_scale_export": 10, # Meters "band_names": [{'NIR': "B8", 'RED': "B4"}], "product_name": "NDVI-MAX", - "step0_collection": "projects/geetest-386915/assets/col_s2_sr" + "step0_collection": "projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS" } PRODUCT_S2_LEVEL_2A = { "prefix": "S2_L2A_SR", - "image_collection": "COPERNICUS/S2_SR_HARMONIZED", + "image_collection": "COPERNICUS/S2_SR_HARMONIZED", # might be obsolete with step0? "temporal_coverage": 1, # Days "spatial_scale_export": 10, # Meters "spatial_scale_export_qa60": 60, # Meters "product_name": "S2_LEVEL_2A", - "step0_collection": "projects/geetest-386915/assets/col_s2_sr" + "step0_collection": "projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS" } PRODUCT_S2_LEVEL_1C = { "prefix": "S2_L1C_TOA", - "image_collection": "COPERNICUS/S2_HARMONIZED", + "image_collection": "COPERNICUS/S2_HARMONIZED", # might be obsolete with step0? "temporal_coverage": 30, # Days "spatial_scale_export": 10, # Meters "spatial_scale_export_mask60": 60, @@ -81,7 +81,7 @@ PRODUCT_NDVI_MAX_TOA = { "prefix": "Sentinel_NDVI-MAX_TOA_CloudFree_crop", - "image_collection": "COPERNICUS/S2_HARMONIZED", + "image_collection": "COPERNICUS/S2_HARMONIZED", # might be obsolete with step0? "temporal_coverage": 1, # Days "spatial_scale_export": 1, # Meters "band_names": [{'NIR': "B8", 'RED': "B4"}], @@ -92,13 +92,13 @@ # dictionary used to manage custom asset, # for example to clear old images not used anymore. - +# Additional personal collections step0 = { - 'projects/satromo-exolabs/assets/col_s2_toa': { - 'step0_function': 'step0_processor_s2_toa.generate_s2_toa_mosaic_for_single_date', - # cleaning_older_than: 2 # entry used to clean assets - }, - 'projects/satromo-exolabs/assets/col_s2_sr': { + # 'projects/satromo-exolabs/assets/col_s2_toa': { + # 'step0_function': 'step0_processor_s2_toa.generate_s2_toa_mosaic_for_single_date', + # # cleaning_older_than: 2 # entry used to clean assets + # }, + 'projects/satromo-int/assets/COL_S2_SR_HARMONIZED_SWISS': { 'step0_function': 'step0_processor_s2_sr.generate_s2_sr_mosaic_for_single_date' # cleaning_older_than: 2 # entry used to clean assets } diff --git a/satromo_processor.py b/satromo_processor.py index c8a10b75..bc42217a 100644 --- a/satromo_processor.py +++ b/satromo_processor.py @@ -177,8 +177,10 @@ def get_collection_info(collection): last_image = sorted_collection.sort('system:time_start', False).first() # Get the dates of the first and last image - first_date = ee.Date(first_image.get('system:time_start')).format('YYYY-MM-dd').getInfo() - last_date = ee.Date(last_image.get('system:time_start')).format('YYYY-MM-dd').getInfo() + first_date = ee.Date(first_image.get('system:time_start') + ).format('YYYY-MM-dd').getInfo() + last_date = ee.Date(last_image.get('system:time_start') + ).format('YYYY-MM-dd').getInfo() # Get the count of images in the filtered collection image_count = collection.size() @@ -544,7 +546,8 @@ def process_NDVI_MAX(roi): timestamp = timestamp.strftime('%Y%m%dT235959') # Generate the filename - filename = config.PRODUCT_NDVI_MAX['prefix'] + '_' + timestamp + '_10m.tif' + filename = config.PRODUCT_NDVI_MAX['prefix'] + \ + '_' + timestamp + '_10m.tif' print(filename) # Check if there is at least 1 scene to be defined (if minimal scene count is required) TODO: is this necessary? @@ -637,7 +640,8 @@ def add_footprint(image, lst): multiband_export, sensor_stats, current_date_str) # Export QA60 band as a separate GeoTIFF with '_QA60' - masks_export = clipped_image.select(['terrainShadowMask', 'cloudAndCloudShadowMask']) + masks_export = clipped_image.select( + ['terrainShadowMask', 'cloudAndCloudShadowMask']) masks_export_name = mosaic_id.replace('_bands-10m', '_masks-10m') prepare_export(clipped_image_bounding_box, mosaic_sensing_timestamp, masks_export_name, config.PRODUCT_S2_LEVEL_2A['product_name'], @@ -726,13 +730,15 @@ def add_footprint(image, lst): multiband_export, sensor_stats, current_date_str) # Export QA60 band as a separate GeoTIFF with '_QA60' - masks_export = clipped_image.select(['terrainShadowMask', 'cloudAndCloudShadowMask']) + masks_export = clipped_image.select( + ['terrainShadowMask', 'cloudAndCloudShadowMask']) masks_export_name = mosaic_id.replace('_bands-10m', '_masks-10m') prepare_export(clipped_image_bounding_box, mosaic_sensing_timestamp, masks_export_name, config.PRODUCT_S2_LEVEL_1C['product_name'], config.PRODUCT_S2_LEVEL_1C['spatial_scale_export_mask'], masks_export, sensor_stats, current_date_str) + def process_NDVI_MAX_TOA(roi): """ Process the NDVI MAX product for TOA. @@ -781,7 +787,8 @@ def process_NDVI_MAX_TOA(roi): timestamp = timestamp.strftime('%Y%m%dT235959') # Generate the filename - filename = config.PRODUCT_NDVI_MAX_TOA['prefix'] + '_' + timestamp + '_10m.tif' + filename = config.PRODUCT_NDVI_MAX_TOA['prefix'] + \ + '_' + timestamp + '_10m.tif' print(filename) # Start the export @@ -801,17 +808,17 @@ def process_NDVI_MAX_TOA(roi): current_date_str = datetime.datetime.today().strftime('%Y-%m-%d') # For debugging - # current_date_str = "2023-06-12" - # print("*****************************\n") - # print("using a manual set Date: "+current_date_str) - # print("*****************************\n") + current_date_str = "2023-11-01" + print("*****************************\n") + print("using a manual set Date: "+current_date_str) + print("*****************************\n") current_date = ee.Date(current_date_str) roi = ee.Geometry.Rectangle(config.ROI_RECTANGLE) step0_product_dict = get_step0_dict() print(step0_product_dict) - + collections_ready_for_processors = step0_main( step0_product_dict, current_date_str) print(collections_ready_for_processors) @@ -825,7 +832,8 @@ def process_NDVI_MAX_TOA(roi): result = process_NDVI_MAX(roi) elif product_to_be_processed == 'PRODUCT_S2_LEVEL_2A': - border = ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017").filter(ee.Filter.eq("country_co", "SZ")) + border = ee.FeatureCollection( + "USDOS/LSIB_SIMPLE/2017").filter(ee.Filter.eq("country_co", "SZ")) roi = border.geometry().buffer(config.ROI_BORDER_BUFFER) # roi = ee.Geometry.Rectangle( [ 7.075402, 46.107098, 7.100894, 46.123639]) result = process_S2_LEVEL_2A(roi) diff --git a/step0_processors/step0_processor_s2_sr.py b/step0_processors/step0_processor_s2_sr.py index aeee3826..98bd95e0 100644 --- a/step0_processors/step0_processor_s2_sr.py +++ b/step0_processors/step0_processor_s2_sr.py @@ -45,8 +45,8 @@ def generate_s2_sr_mosaic_for_single_date(day_to_process: str, collection: str, # Export switches # options': True, 'False - defines if 10-m-bands are exported': 'B2','B3','B4','B8' export10mBands = True - # options': True, 'False - defines if 20-m-bands are exported': 'B5','B6','B7','B8A','B11','B12' - # export20mBands = False # NOTEJS: ununsed, export function commented in the script below + # options': True, 'False - defines if 20-m-bands are exported': select from 'B5','B6','B7','B8A','B11','B12'below + export20mBands = True # options': True, 'False - defines if 60-m-bands are exported': 'B1','B9','B10' # export60mBands = False # NOTEJS: ununsed, export function commented in the script below # options': True, 'False - defines if registration layers are exported': 'reg_dx','reg_dy', 'reg_confidence' @@ -70,7 +70,8 @@ def generate_s2_sr_mosaic_for_single_date(day_to_process: str, collection: str, # processing: reprojected in QGIS to epsg32632 aoi_CH = ee.FeatureCollection( "users/wulf/SATROMO/swissBOUNDARIES3D_1_4_TLM_LANDESGEBIET_epsg32632").geometry() - aoi_CH_simplified = ee.FeatureCollection("users/wulf/SATROMO/CH_boundaries_buffer_5000m_epsg32632").geometry() + aoi_CH_simplified = ee.FeatureCollection( + "users/wulf/SATROMO/CH_boundaries_buffer_5000m_epsg32632").geometry() ############################## # REFERENCE DATA @@ -399,7 +400,8 @@ def S2regFunc(image): reg_dx = reg_dx.multiply(100).round().toInt16() reg_dy = displacement.select('dy').rename('reg_dy') reg_dy = reg_dy.multiply(100).round().toInt16() - reg_confidence = displacement.select('confidence').rename('reg_confidence') + reg_confidence = displacement.select( + 'confidence').rename('reg_confidence') reg_confidence = reg_confidence.multiply(100).round().toUint8() # Compute image offset and direction. @@ -427,10 +429,8 @@ def S2regFunc(image): # extract the date and time (it is same time for all images in the mosaic) sensing_date = S2_sr.get('system:index').getInfo()[0:15] - sensing_date_read = sensing_date[0:4] + '-' + sensing_date[4:6] + '-' + sensing_date[6:15] - - # define the filenames - fname_10m = 'S2-L2A_mosaic_' + sensing_date_read + '_bands-10m' + sensing_date_read = sensing_date[0:4] + '-' + \ + sensing_date[4:6] + '-' + sensing_date[6:15] # define the export aoi # the full mosaic image geometry covers larger areas outside Switzerland that are not needed @@ -442,6 +442,8 @@ def S2regFunc(image): # SWITCH export if export10mBands is True: print('Launching export for 10m bands') + # define the filenames + fname_10m = 'S2-L2A_mosaic_' + sensing_date_read + '_bands-10m' band_list = ['B2', 'B3', 'B4', 'B8'] if exportMasks: band_list.extend(['terrainShadowMask', 'cloudAndCloudShadowMask']) @@ -462,22 +464,24 @@ def S2regFunc(image): ) task.start() - """"# SWITCH export + # SWITCH export if export20mBands is True: - fname_20m = 'S2-L2A_Mosaic_' + sensing_date_read + '_Bands-20m' + print('Launching export for 20m bands') + # define the filenames + fname_20m = 'S2-L2A_Mosaic_' + sensing_date_read + '_bands-20m' # Export COG 20m bands - task = ee.batch.Export.image.toDrive( - image=S2_sr.select(['B5', 'B6', 'B7', 'B8A', 'B11', 'B12']), + task = ee.batch.Export.image.toAsset( + image=S2_sr.select(['B8A', 'B11']).clip(aoi_exp), scale=20, - description=fname_20m, + description=task_description, crs='EPSG:2056', region=aoi_exp, maxPixels=1e10, - assetId=fname_20m + assetId=collection + '/' + fname_20m ) task.start() - # SWITCH export + """"# SWITCH export if export60mBands is True: fname_60m = 'S2-L2A_Mosaic_' + sensing_date_read + '_Bands-60m' task = ee.batch.Export.image.toDrive( diff --git a/tools/step0_empty_assets.csv b/tools/step0_empty_assets.csv index ba4a6a0a..a6d0994b 100644 --- a/tools/step0_empty_assets.csv +++ b/tools/step0_empty_assets.csv @@ -35,3 +35,18 @@ COL_S2_HARMONIZED_SWISS,2023-12-02,cloudy COL_S2_HARMONIZED_SWISS,2023-12-04,No scene COL_S2_HARMONIZED_SWISS,2023-12-06,No scene COL_S2_HARMONIZED_SWISS,2023-12-08,No scene +col_s2_sr,2023-06-09,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-07,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-12,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-17,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-19,cloudy +COL_S2_HARMONIZED_SWISS,2023-10-20,cloudy +COL_S2_HARMONIZED_SWISS,2023-10-22,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-23,cloudy +COL_S2_HARMONIZED_SWISS,2023-10-24,cloudy +COL_S2_HARMONIZED_SWISS,2023-10-26,cloudy +COL_S2_HARMONIZED_SWISS,2023-10-27,No candidate scene +COL_S2_HARMONIZED_SWISS,2023-10-30,cloudy +COL_S2_HARMONIZED_SWISS,2023-11-01,No candidate scene +COL_S2_SR_HARMONIZED_SWISS,2023-10-27,No candidate scene +COL_S2_SR_HARMONIZED_SWISS,2023-11-01,No candidate scene