Skip to content

Commit

Permalink
Add extra params to CLIs, fix requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
@robbibt
robbibt committed Jan 10, 2023
1 parent 5e97838 commit 3b08312
Show file tree
Hide file tree
Showing 7 changed files with 250 additions and 1,030 deletions.
21 changes: 17 additions & 4 deletions coastlines/raster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ def tide_cutoffs(ds, tides_lowres, tide_centre=0.0, resampling="bilinear"):
tide_centre : float, optional
The central tide height used to compute the min and max
tide height cutoffs. Tide heights will be masked so all
satellite observations are approximiately centred over this
satellite observations are approximately centred over this
value. The default is 0.0 which represents 0 m Above Mean
Sea Level.
resampling : string, optional
Expand Down Expand Up @@ -451,7 +451,7 @@ def export_annual_gapfill(
ds.sel(time=str(year + 1)),
tide_cutoff_min=tide_cutoff_min,
tide_cutoff_max=tide_cutoff_max,
).drop("tide_m")
).drop_vars("tide_m")

except KeyError:

Expand Down Expand Up @@ -503,7 +503,7 @@ def export_annual_gapfill(


def generate_rasters(
dc, config, study_area, raster_version, start_year, end_year, log=None
dc, config, study_area, raster_version, start_year, end_year, tide_centre, log=None
):
#####################################
# Connect to datacube, Dask cluster #
Expand Down Expand Up @@ -572,7 +572,9 @@ def generate_rasters(
# and min satellite-observed tide height for each pixel, then
# calculate tide cutoffs used to restrict our data to satellite
# observations centred over mid-tide (0 m Above Mean Sea Level).
tide_cutoff_min, tide_cutoff_max = tide_cutoffs(ds, tides_lowres, tide_centre=0.0)
tide_cutoff_min, tide_cutoff_max = tide_cutoffs(
ds, tides_lowres, tide_centre=tide_centre
)
log.info(
f"Study area {study_area}: Calculating low and high tide cutoffs for each pixel"
)
Expand Down Expand Up @@ -653,6 +655,15 @@ def generate_rasters(
"recommend that at least one year of satellite data exists in your "
"datacube after `--end_year`.",
)
@click.option(
"--tide_centre",
type=float,
default=0.0,
help="The central tide height used to compute the min and max tide "
"height cutoffs. Tide heights will be masked so all satellite "
"observations are approximately centred over this value. The "
"default is 0.0 which represents 0 m Above Mean Sea Level.",
)
@click.option(
"--aws_unsigned/--no-aws_unsigned",
type=bool,
Expand All @@ -672,6 +683,7 @@ def generate_rasters_cli(
raster_version,
start_year,
end_year,
tide_centre,
aws_unsigned,
overwrite,
):
Expand Down Expand Up @@ -706,6 +718,7 @@ def generate_rasters_cli(
raster_version,
start_year,
end_year,
tide_centre,
log=log,
)

Expand Down
3 changes: 1 addition & 2 deletions coastlines/vector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,8 +790,7 @@ def _point_interp(points, array, **kwargs):
lambda x: x.geometry.distance(x[f"p_{comp_year}"]), axis=1
)

# Set any value over X m to NaN, and drop any points with
# less than 50% valid observations
# Set any value over X m to NaN
points_gdf[f"dist_{comp_year}"] = distances.where(distances < max_valid_dist)

# Extract comparison array containing water index values for the
Expand Down
190 changes: 130 additions & 60 deletions notebooks/DEACoastlines_generation_CLI.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 3,
"id": "4729d261-6b27-4a61-a937-366c5174ce05",
"metadata": {},
"outputs": [],
Expand All @@ -106,16 +106,14 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 4,
"id": "dc5632c4-f872-46e5-9762-80b8556f3622",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/env/lib/python3.8/site-packages/rasterio/path.py:15: RasterioDeprecationWarning: rasterio.path will be removed in version 1.4.\n",
" warnings.warn(\n",
"Usage: python -m coastlines.raster [OPTIONS]\n",
"\n",
"Options:\n",
Expand Down Expand Up @@ -157,6 +155,12 @@
" purposes). Because of this, we recommend\n",
" that at least one year of satellite data\n",
" exists in your datacube after `--end_year`.\n",
" --tide_centre FLOAT The central tide height used to compute the\n",
" min and max tide height cutoffs. Tide\n",
" heights will be masked so all satellite\n",
" observations are approximately centred over\n",
" this value. The default is 0.0 which\n",
" represents 0 m Above Mean Sea Level.\n",
" --aws_unsigned / --no-aws_unsigned\n",
" Whether to use sign AWS requests for S3\n",
" access\n",
Expand All @@ -180,34 +184,30 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 5,
"id": "47397eb8-ddec-46c6-9f7c-431a50bb7eae",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/env/lib/python3.8/site-packages/rasterio/path.py:15: RasterioDeprecationWarning: rasterio.path will be removed in version 1.4.\n",
" warnings.warn(\n",
"/env/lib/python3.8/site-packages/rasterio/path.py:15: RasterioDeprecationWarning: rasterio.path will be removed in version 1.4.\n",
" warnings.warn(\n",
"<Client: 'tcp://127.0.0.1:46643' processes=1 threads=31, memory=237.21 GiB>\n",
"2022-12-13 04:46:31 INFO Study area 398: Loaded study area grid\n",
"2022-12-13 04:46:33 INFO Study area 398: Loaded virtual product\n",
"<Client: 'tcp://127.0.0.1:46411' processes=1 threads=15, memory=117.21 GiB>\n",
"2023-01-10 00:48:31 INFO Study area 3: Loaded study area grid\n",
"2023-01-10 00:48:40 INFO Study area 3: Loaded virtual product\n",
"Creating reduced resolution tide modelling array\n",
"Modelling tides using FES2014 tide model\n",
"Reprojecting tides into original array\n",
"100%|█████████████████████████████████████████| 258/258 [00:03<00:00, 67.79it/s]\n",
"2022-12-13 04:47:13 INFO Study area 398: Finished modelling tide heights\n",
"2022-12-13 04:47:13 INFO Study area 398: Calculating low and high tide cutoffs for each pixel\n",
"2022-12-13 04:47:13 INFO Study area 398: Started exporting raster data\n",
"2022-12-13 04:49:49 INFO Study area 398: Completed exporting raster data\n"
"100%|██████████████████████████████████████| 1112/1112 [00:04<00:00, 271.95it/s]\n",
"2023-01-10 00:49:05 INFO Study area 3: Finished modelling tide heights\n",
"2023-01-10 00:49:05 INFO Study area 3: Calculating low and high tide cutoffs for each pixel\n",
"2023-01-10 00:49:05 INFO Study area 3: Started exporting raster data\n",
"2023-01-10 00:51:35 INFO Study area 3: Completed exporting raster data\n"
]
}
],
"source": [
"!python -m coastlines.raster --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --start_year 1988 --end_year 2021"
"!python -m coastlines.raster --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --start_year 1988 --end_year 2021 --tide_centre 1.0"
]
},
{
Expand All @@ -220,7 +220,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 6,
"id": "0ed72c21-ba89-4fad-9521-f5493aad9d66",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -294,25 +294,45 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"id": "c23665e0-6d46-4185-b9f0-04a50d856174",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2022-12-20 05:47:56 INFO Study area 3: Starting vector generation\n",
"2022-12-20 05:47:59 INFO Study area 3: Loaded rasters\n",
"2023-01-10 00:51:44 INFO Study area 3: Starting vector generation\n",
"2023-01-10 00:51:48 INFO Study area 3: Loaded rasters\n",
"Operating in single z-value, multiple arrays mode\n",
"2022-12-20 05:48:17 INFO Study area 3: Extracted shorelines from raster data\n",
"2022-12-20 05:48:18 INFO Study area 3: Extracted rates of change points\n",
"2022-12-20 05:49:01 INFO Study area 3: Calculated distances to each annual shoreline\n",
"2022-12-20 05:49:05 INFO Study area 3: Calculated rates of change regressions\n",
"2022-12-20 05:49:12 INFO Study area 3: Calculated all of time statistics\n",
"2022-12-20 05:49:12 INFO Study area 3: Calculated rate of change certainty flags\n",
"2022-12-20 05:49:14 INFO Study area 3: Added region attributes and geohash UIDs\n",
"2022-12-20 05:49:23 INFO Study area 3: Output vector files written to data/interim/vector/development/3_development\n"
"2023-01-10 00:52:08 INFO Study area 3: Extracted shorelines from raster data\n",
"2023-01-10 00:52:09 INFO Study area 3: Extracted rates of change points\n",
"2023-01-10 00:52:10 ERROR Study area 3: Failed to run process with error No matching signature found\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1792, in generate_vectors_cli\n",
" generate_vectors(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1461, in generate_vectors\n",
" points_gdf = annual_movements(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 802, in annual_movements\n",
" points_gdf[\"index_comp_p1\"] = _point_interp(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 765, in _point_interp\n",
" return array.interp(x=x_vals, y=y_vals, **kwargs)\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/dataarray.py\", line 1746, in interp\n",
" ds = self._to_temp_dataset().interp(\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/dataset.py\", line 3196, in interp\n",
" variables[name] = missing.interp(var, var_indexers, method, **kwargs)\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 640, in interp\n",
" interped = interp_func(\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 765, in interp_func\n",
" return _interpnd(var, x, new_x, func, kwargs)\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 789, in _interpnd\n",
" rslt = func(x, var, xi, **kwargs)\n",
" File \"/env/lib/python3.8/site-packages/scipy/interpolate/_rgi.py\", line 654, in interpn\n",
" return interp(xi)\n",
" File \"/env/lib/python3.8/site-packages/scipy/interpolate/_rgi.py\", line 336, in __call__\n",
" result = evaluate_linear_2d(self.values,\n",
" File \"_rgi_cython.pyx\", line 19, in scipy.interpolate._rgi_cython.__pyx_fused_cpdef\n",
"TypeError: No matching signature found\n"
]
}
],
Expand All @@ -330,10 +350,63 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"id": "518bf214-ce70-4baa-a2ad-28302986fe01",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Usage: python -m coastlines.continental [OPTIONS]\n",
"\n",
"Options:\n",
" --vector_version TEXT A unique string proving a name that was used\n",
" for output vector directories and files.\n",
" This is used to identify the tiled annual\n",
" shoreline and rates of change layers that\n",
" will be combined into continental-scale\n",
" layers. [required]\n",
" --continental_version TEXT A unique string proving a name that will be\n",
" used for output continental-scale layers.\n",
" This allows multiple versions of\n",
" continental-scale layers to be generated\n",
" from the same input vector data, e.g. for\n",
" testing different hotspot of coastal change\n",
" summary layers. If not provided, this will\n",
" default to the string provided to \"--\n",
" vector_version\".\n",
" --shorelines BOOLEAN A boolean indicating whether to combine\n",
" tiled annual shorelines layers into a single\n",
" continental-scale annual shorelines layer.\n",
" --ratesofchange BOOLEAN A boolean indicating whether to combine\n",
" tiled rates of change statistics layers into\n",
" a single continental-scale rates of change\n",
" statistics layer.\n",
" --hotspots BOOLEAN A boolean indicating whether to generate a\n",
" continental-scale hotspots of coastal change\n",
" summary layer.\n",
" --hotspots_radius INTEGER The distance (in metres) used to generate\n",
" coastal change hotspots summary layers. This\n",
" controls the spacing of each summary point,\n",
" and the radius used to aggregate rates of\n",
" change statistics around each point. The\n",
" default generates three hotspot layers with\n",
" radii 10000 m, 5000 m and 1000 m. To specify\n",
" multiple custom radii, repeat this argument,\n",
" e.g. `--hotspots_radius 1000\n",
" --hotspots_radius 5000`.\n",
" --baseline_year INTEGER The annual shoreline used to generate the\n",
" hotspot summary points. This is typically\n",
" the most recent annual shoreline in the\n",
" dataset.\n",
" --include-styles / --no-include-styles\n",
" Set this to indicate whether to include\n",
" styles in output GeoPackage file.\n",
" --help Show this message and exit.\n"
]
}
],
"source": [
"!python -m coastlines.continental --help"
]
Expand All @@ -348,10 +421,30 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"id": "22e3dca9-6a5e-4e6a-ac2c-c3f3b1ae904b",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2023-01-10 01:58:41 INFO Writing data to data/processed/development\n",
"Warning 1: A geometry of type MULTILINESTRING is inserted into layer shorelines_annual of geometry type LINESTRING, which is not normally allowed by the GeoPackage specification, but the driver will however do it. To create a conformant GeoPackage, if using ogr2ogr, the -nlt option can be used to override the layer geometry type. This warning will no longer be emitted for this combination of layer and feature geometry type.\n",
"2023-01-10 01:58:42 INFO Merging annual shorelines complete\n",
"2023-01-10 01:58:42 INFO Merging rates of change points complete\n",
"2023-01-10 01:58:42 INFO Generating continental hotspots\n",
"2023-01-10 01:58:42 INFO Generating continental hotspots\n",
"2023-01-10 01:58:42 INFO Calculating 10000 m hotspots\n",
"2023-01-10 01:58:43 INFO Calculating 5000 m hotspots\n",
"2023-01-10 01:58:43 INFO Calculating 1000 m hotspots\n",
"2023-01-10 01:58:43 INFO Writing hotspots complete\n",
"2023-01-10 01:58:46 INFO Writing rates of change points to zipped shapefiles complete\n",
"2023-01-10 01:58:46 INFO Writing annual shorelines to zipped shapefiles complete\n",
"2023-01-10 01:58:46 INFO Writing styles to GeoPackage file complete\n"
]
}
],
"source": [
"!python -m coastlines.continental --vector_version {vector_version} --continental_version {continental_version} --shorelines True --ratesofchange True --hotspots True --baseline_year 2021"
]
Expand All @@ -367,7 +460,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"id": "6b804fec-39f4-48a9-8525-a78fbf06e084",
"metadata": {},
"outputs": [],
Expand All @@ -378,33 +471,10 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"id": "c39e6533-f4fd-4a3c-90a5-44f19825c2f5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"759\n",
"<Client: 'tcp://127.0.0.1:44537' processes=1 threads=7, memory=59.21 GiB>\n",
"2022-12-09 03:30:59 INFO Study area 759: Loaded study area grid\n",
"2022-12-09 03:31:07 INFO Study area 759: Loaded virtual product\n",
"Creating reduced resolution tide modelling array\n",
"Modelling tides using FES2014 tide model\n",
"Reprojecting tides into original array\n",
"100%|███████████████████████████████████████| 1432/1432 [00:52<00:00, 27.45it/s]\n",
"2022-12-09 03:33:21 INFO Study area 759: Finished modelling tide heights\n",
"2022-12-09 03:33:22 INFO Study area 759: Calculating low and high tide cutoffs for each pixel\n",
"2022-12-09 03:33:22 INFO Study area 759: Started exporting raster data\n",
"2022-12-09 04:00:31 INFO Study area 759: Completed exporting raster data\n",
"distributed.nanny - WARNING - Worker process still alive after 3.9999988555908206 seconds, killing\n",
"Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n",
"Traceback (most recent call last):\n",
" File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n"
]
}
],
"outputs": [],
"source": [
"# Run raster and vector generation for each study area\n",
"for study_area in study_areas:\n",
Expand Down
Loading

0 comments on commit 3b08312

Please sign in to comment.