From 70daaad306272e26eca828855d8d87c26897fc05 Mon Sep 17 00:00:00 2001 From: David Gavin Date: Fri, 30 Jul 2021 15:06:22 +1000 Subject: [PATCH 1/5] Add Preserve User date order to C3 ARD styling Add Preserve User date order to C3 ARD styling for upcoming release --- prod/services/wms/ows_refactored/c3/style_c3_cfg.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prod/services/wms/ows_refactored/c3/style_c3_cfg.py b/prod/services/wms/ows_refactored/c3/style_c3_cfg.py index ccf954c55..61576e695 100644 --- a/prod/services/wms/ows_refactored/c3/style_c3_cfg.py +++ b/prod/services/wms/ows_refactored/c3/style_c3_cfg.py @@ -129,6 +129,8 @@ # The count range for which this handler is to be used - a tuple of two ints, the smallest and # largest date counts for which this handler will be used. Required. "allowed_count_range": [2, 2], + # Preserve user date order + "preserve_user_date_order": True, # A function, expressed in the standard format as described elsewhere in this example file. # The function is assumed to take one arguments, an xarray Dataset. # The function returns an xarray Dataset with a single band, which is the input to the @@ -242,6 +244,8 @@ # The count range for which this handler is to be used - a tuple of two ints, the smallest and # largest date counts for which this handler will be used. Required. "allowed_count_range": [2, 2], + # Preserve user date order + "preserve_user_date_order": True, # A function, expressed in the standard format as described elsewhere in this example file. # The function is assumed to take one arguments, an xarray Dataset. # The function returns an xarray Dataset with a single band, which is the input to the @@ -332,6 +336,8 @@ # The count range for which this handler is to be used - a tuple of two ints, the smallest and # largest date counts for which this handler will be used. Required. "allowed_count_range": [2, 2], + # Preserve user date order + "preserve_user_date_order": True, # A function, expressed in the standard format as described elsewhere in this example file. # The function is assumed to take one arguments, an xarray Dataset. # The function returns an xarray Dataset with a single band, which is the input to the @@ -529,6 +535,8 @@ # The count range for which this handler is to be used - a tuple of two ints, the smallest and # largest date counts for which this handler will be used. Required. "allowed_count_range": [2, 2], + # Preserve user date order + "preserve_user_date_order": True, # A function, expressed in the standard format as described elsewhere in this example file. # The function is assumed to take one arguments, an xarray Dataset. # The function returns an xarray Dataset with a single band, which is the input to the From 68d96dcd77fb0d075a5a6ecd82177ea020d76596 Mon Sep 17 00:00:00 2001 From: David Gavin Date: Fri, 30 Jul 2021 15:29:07 +1000 Subject: [PATCH 2/5] Add Sentinel-2 ARD config --- .../ows_refactored/sentinel2/ows_ard_cfg.py | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/prod/services/wms/ows_refactored/sentinel2/ows_ard_cfg.py b/prod/services/wms/ows_refactored/sentinel2/ows_ard_cfg.py index 65c403a62..c25ddfae9 100644 --- a/prod/services/wms/ows_refactored/sentinel2/ows_ard_cfg.py +++ b/prod/services/wms/ows_refactored/sentinel2/ows_ard_cfg.py @@ -1,4 +1,4 @@ -from ows_refactored.ows_reslim_cfg import reslim_wms_min_zoom_35 +from ows_refactored.ows_reslim_cfg import reslim_wms_min_zoom_15_cache_rules from ows_refactored.sentinel2.style_s2_cfg import styles_s2_list bands_sentinel2_ard_nbart = { @@ -7,7 +7,8 @@ "coastal_aerosol", "nbart_coastal_aerosol", "nbart_narrow_blue", - "nbar_narrow_blue" "narrow_blue", + "nbar_narrow_blue", + "narrow_blue", ], "nbart_blue": ["nbar_blue", "blue", "nbart_blue"], "nbart_green": ["nbar_green", "green", "nbart_green"], @@ -19,6 +20,7 @@ "nbart_nir_2": ["nbar_nir_2", "nir2", "nbart_nir_2"], "nbart_swir_2": ["nbar_swir_2", "swir_2", "nbart_swir_2"], "nbart_swir_3": ["nbar_swir_3", "swir_3", "nbart_swir_3"], + "fmask": ["fmask"], } layers = { @@ -45,13 +47,27 @@ "multi_product": True, "product_names": ["s2a_ard_granule", "s2b_ard_granule"], "bands": bands_sentinel2_ard_nbart, - "resource_limits": reslim_wms_min_zoom_35, + "resource_limits": reslim_wms_min_zoom_15_cache_rules, "dynamic": True, "image_processing": { "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "products": ["s2a_ard_granule", "s2b_ard_granule"], + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "products": ["geodata_coast_100k", "geodata_coast_100k"], + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], @@ -67,13 +83,27 @@ """, "product_name": "s2b_ard_granule", "bands": bands_sentinel2_ard_nbart, - "resource_limits": reslim_wms_min_zoom_35, + "resource_limits": reslim_wms_min_zoom_15_cache_rules, "dynamic": True, "image_processing": { "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "product": "s2b_ard_granule", + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], @@ -89,13 +119,27 @@ """, "product_name": "s2a_ard_granule", "bands": bands_sentinel2_ard_nbart, - "resource_limits": reslim_wms_min_zoom_35, + "resource_limits": reslim_wms_min_zoom_15_cache_rules, "dynamic": True, "image_processing": { "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "product": "s2a_ard_granule", + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], From 17796b82ada113f681d3eb4e5ef7e34985b82ab1 Mon Sep 17 00:00:00 2001 From: David Gavin Date: Fri, 30 Jul 2021 15:29:38 +1000 Subject: [PATCH 3/5] Add S2 NRT Config --- .../ows_refactored/sentinel2/ows_nrt_cfg.py | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/prod/services/wms/ows_refactored/sentinel2/ows_nrt_cfg.py b/prod/services/wms/ows_refactored/sentinel2/ows_nrt_cfg.py index 1603d6a39..ab946ed87 100644 --- a/prod/services/wms/ows_refactored/sentinel2/ows_nrt_cfg.py +++ b/prod/services/wms/ows_refactored/sentinel2/ows_nrt_cfg.py @@ -1,6 +1,5 @@ from ows_refactored.ows_reslim_cfg import reslim_wms_min_zoom_15_cache_rules -from ows_refactored.sentinel2.style_s2_cfg import (style_s2_nbr_list, - styles_s2_list) +from ows_refactored.sentinel2.style_s2_cfg import styles_s2_list bands_sentinel2 = { "nbar_coastal_aerosol": ["nbar_coastal_aerosol", "nbar_narrow_blue"], @@ -30,6 +29,7 @@ "nbart_nir_2": ["nbart_nir_2", "nir_2", "nbart_near_infrared_2"], "nbart_swir_2": ["nbart_swir_2", "swir_2", "nbart_shortwave_infrared_2"], "nbart_swir_3": ["nbart_swir_3", "swir_3", "nbart_shortwave_infrared_3"], + "fmask": ["fmask"], } multi_layers = { @@ -59,6 +59,20 @@ "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "products": ["s2a_nrt_granule", "s2b_nrt_granule"], + "ignore_time": False, + "ignore_info_flags": [], + }, + { + "band": "land", + "products": ["geodata_coast_100k", "geodata_coast_100k"], + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], @@ -68,7 +82,7 @@ "nbart_blue", ], }, - "styling": {"default_style": "simple_rgb", "styles": style_s2_nbr_list}, + "styling": {"default_style": "simple_rgb", "styles": styles_s2_list}, } s2b_layer = { @@ -97,6 +111,20 @@ "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "product": "s2b_nrt_granule", + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], @@ -135,6 +163,20 @@ "always_fetch_bands": [], "manual_merge": False, }, + "flags": [ + { + "band": "fmask", + "product": "s2a_nrt_granule", + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [] + }, + ], "wcs": { "native_crs": "EPSG:3577", "native_resolution": [10.0, 10.0], @@ -144,5 +186,5 @@ "nbart_blue", ], }, - "styling": {"default_style": "simple_rgb", "styles": style_s2_nbr_list}, + "styling": {"default_style": "simple_rgb", "styles": styles_s2_list}, } From 11a05abf5a7f53b3d36e8dd7836053b26acab325 Mon Sep 17 00:00:00 2001 From: David Gavin Date: Fri, 30 Jul 2021 15:30:14 +1000 Subject: [PATCH 4/5] Add update Sentinel-2 Style --- .../ows_refactored/sentinel2/style_s2_cfg.py | 305 +++++++++++++++++- 1 file changed, 288 insertions(+), 17 deletions(-) diff --git a/prod/services/wms/ows_refactored/sentinel2/style_s2_cfg.py b/prod/services/wms/ows_refactored/sentinel2/style_s2_cfg.py index 205b458e4..7ec7273bc 100644 --- a/prod/services/wms/ows_refactored/sentinel2/style_s2_cfg.py +++ b/prod/services/wms/ows_refactored/sentinel2/style_s2_cfg.py @@ -22,7 +22,7 @@ style_s2_ndvi = { "name": "ndvi", - "title": "NDVI - Red, NIR", + "title": "Normalised Difference Vegetation Index - Red, NIR", "abstract": "Normalised Difference Vegetation Index - a derived index that correlates well with the existence of vegetation", "index_function": { "function": "datacube_ows.band_utils.norm_diff", @@ -44,12 +44,77 @@ {"value": 0.9, "color": "#237100"}, {"value": 1.0, "color": "#114D04"}, ], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], "legend": legend_idx_0_1_5ticks, + "multi_date": [ + { + "allowed_count_range": [2, 2], + "preserve_user_date_order": True, + "aggregator_function": { + "function": "datacube_ows.band_utils.multi_date_delta" + }, + "mpl_ramp": "RdYlBu", + "range": [-1.0, 1.0], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], + "legend": { + "begin": "-1.0", + "end": "1.0", + "ticks": [ + "-1.0", + "0.0", + "1.0", + ], + }, + "feature_info_label": "ndvi_delta", + } + ], } style_s2_ndwi = { "name": "ndwi", - "title": "NDWI - Green, NIR", + "title": "Normalised Difference Water Index - Green, NIR", "abstract": "Normalised Difference Water Index - a derived index that correlates well with the existence of water (McFeeters 1996)", "index_function": { "function": "datacube_ows.band_utils.norm_diff", @@ -78,6 +143,28 @@ "color": "#08306b", }, ], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], "legend": { "begin": "0.0", "end": "0.5", @@ -88,13 +175,56 @@ "0.2": {"label": "0.2"}, "0.4": {"label": "0.4"}, "0.5": {"prefix": ">"}, - }, + } }, + "multi_date": [ + { + "allowed_count_range": [2, 2], + "preserve_user_date_order": True, + "aggregator_function": { + "function": "datacube_ows.band_utils.multi_date_delta" + }, + "mpl_ramp": "RdYlBu", + "range": [-1.0, 1.0], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], + "legend": { + "begin": "-1.0", + "end": "1.0", + "ticks": [ + "-1.0", + "-0.0", + "1.0", + ], + }, + "feature_info_label": "ndwi_delta", + } + ], } style_s2_mndwi = { # Cannot reuse landsat as we need swir_2 to landsat's swir_1 "name": "mndwi", - "title": "MNDWI - Green, SWIR", + "title": "Modified Normalised Difference Water Index - Green, SWIR", "abstract": "Modified Normalised Difference Water Index - a derived index that correlates well with the existence of water (Xu 2006)", "index_function": { "function": "datacube_ows.band_utils.norm_diff", @@ -111,12 +241,77 @@ {"value": 0.8, "color": "#1563aa"}, {"value": 1.0, "color": "#08306b"}, ], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], "legend": legend_idx_0_1_5ticks, + "multi_date": [ + { + "allowed_count_range": [2, 2], + "preserve_user_date_order": True, + "aggregator_function": { + "function": "datacube_ows.band_utils.multi_date_delta" + }, + "mpl_ramp": "RdYlBu", + "range": [-1.0, 1.0], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], + "legend": { + "begin": "-1.0", + "end": "1.0", + "ticks": [ + "-1.0", + "0.0", + "1.0", + ], + }, + "feature_info_label": "mndwi_delta", + } + ], } style_s2_ndci = { "name": "ndci", - "title": "NDCI - Red Edge, Red", + "title": "Normalised Difference Chlorophyll Index - Red Edge, Red", "abstract": "Normalised Difference Chlorophyll Index - a derived index that correlates well with the existence of chlorophyll", "index_function": { "function": "datacube_ows.band_utils.sentinel2_ndci", @@ -166,8 +361,8 @@ } style_s2_nbr = { - "name": "NBR", - "title": "Normalised Burn Ratio", + "name": "nbr", + "title": "Normalised Burn Ratio - NIR, SWIR", "abstract": "Normalised Burn Ratio - a derived index that that uses the differences in the way health green vegetation and burned vegetation reflect light to find burned area", "index_function": { "function": "datacube_ows.band_utils.norm_diff", @@ -205,6 +400,28 @@ "color": "#053061", }, ], + "pq_masks": [ + { + "band": "fmask", + "enum": 0, + "invert": True, + }, + { + "band": "fmask", + "enum": 2, + "invert": True, + }, + { + "band": "fmask", + "enum": 3, + "invert": True, + }, + { + "band": "land", + "invert": True, + "enum": 1, + }, + ], "legend": { "show_legend": True, "begin": "-1.0", @@ -213,17 +430,10 @@ "decimal_places": 0, "tick_labels": {"-1.0": {"prefix": "<"}, "1.0": {"suffix": ">"}}, }, - # Define behaviour(s) for multi-date requests. If not declared, style only supports single-date requests. "multi_date": [ - # A multi-date handler. Different handlers can be declared for different numbers of dates in a request. { - # The count range for which this handler is to be used - a tuple of two ints, the smallest and - # largest date counts for which this handler will be used. Required. "allowed_count_range": [2, 2], - # A function, expressed in the standard format as described elsewhere in this example file. - # The function is assumed to take one arguments, an xarray Dataset. - # The function returns an xarray Dataset with a single band, which is the input to the - # colour ramp defined below. + "preserve_user_date_order": True, "aggregator_function": { "function": "datacube_ows.band_utils.multi_date_delta" }, @@ -414,6 +624,67 @@ "scale_range": [0.0, 3000.0], } +style_s2_fmask = { + "name": "fmask", + "title": "Fmask Classification", + "abstract": "Fmask (Function of mask) is used for automated clouds, cloud shadows, snow, and water masking for Landsats 4-8 and Sentinel 2 images.", + "include_in_feature_info": False, + "needed_bands": ["fmask"], + "value_map": { + "fmask": [ + { + "title": "No Data", + "abstract": "", + "values": [ + 0, # nodata + ], + "alpha": 0.0, + "color": "#FFFFFF", + }, + { + "title": "Clear", + "abstract": "", + "values": [ + 1, # clear/valid + ], + "color": "#84A278", + }, + { + "title": "Cloud", + "abstract": "", + "values": [ + 2, # cloud + ], + "color": "#D0CFCE", + }, + { + "title": "Shadow", + "abstract": "", + "values": [ + 3 # shadow + ], + "color": "#464633", + }, + { + "title": "Snow", + "abstract": "", + "values": [ + 4 # snow + ], + "color": "#E0EDFF", + }, + { + "title": "Water", + "abstract": "", + "values": [ + 5 # water + ], + "color": "#475B74", + }, + ] + } +} + bands_sentinel2_ard_nbar = { "nbar_coastal_aerosol": [ "nbar_coastal_aerosol", @@ -472,6 +743,7 @@ style_s2_ndwi, style_s2_mndwi, style_s2_ndci, + style_s2_nbr, style_s2_pure_aerosol, style_s2_pure_blue, style_s2_pure_green, @@ -483,6 +755,5 @@ style_s2_pure_narrow_nir, style_s2_pure_swir1, style_s2_pure_swir2, + style_s2_fmask, ] - -style_s2_nbr_list = styles_s2_list From 7c2c8029156cde9bd64f93a04d6d19c2836be0e9 Mon Sep 17 00:00:00 2001 From: David Gavin Date: Fri, 30 Jul 2021 15:51:48 +1000 Subject: [PATCH 5/5] Add updated Sentinel 2 styles to inventory --- prod/services/wms/inventory.json | 38 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/prod/services/wms/inventory.json b/prod/services/wms/inventory.json index 184feb9ed..3f17e413d 100644 --- a/prod/services/wms/inventory.json +++ b/prod/services/wms/inventory.json @@ -249,7 +249,7 @@ "s2a_nrt_granule", "s2b_nrt_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -267,14 +267,16 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { "product": [ "s2b_nrt_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -292,14 +294,16 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { "product": [ "s2a_nrt_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -317,7 +321,9 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { @@ -325,7 +331,7 @@ "s2a_ard_granule", "s2b_ard_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -343,14 +349,16 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { "product": [ "s2b_ard_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -368,14 +376,16 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { "product": [ "s2a_ard_granule" ], - "styles_count": 17, + "styles_count": 19, "styles_list": [ "simple_rgb", "infrared_green", @@ -393,7 +403,9 @@ "nir", "narrow_nir", "swir1", - "swir2" + "swir2", + "nbr", + "fmask" ] }, { @@ -990,4 +1002,4 @@ } ], "total_layers_count": 79 -} \ No newline at end of file +}