diff --git a/coastlines/continental.py b/coastlines/continental.py index 873f5c6..dacd8a6 100644 --- a/coastlines/continental.py +++ b/coastlines/continental.py @@ -95,9 +95,9 @@ def wms_fields(gdf): "--hotspots", type=bool, default=True, - help="A boolean indicating whether to generate a " - "continental-scale hotspots of coastal change summary " - "layer.", + help="A boolean indicating whether to generate " + "continental-scale coastal change hotspot summary layers " + "(this can be very slow depending on the radii requested).", ) @click.option( "--hotspots_radius", @@ -120,11 +120,20 @@ def wms_fields(gdf): "summary points. This is typically the most recent " "annual shoreline in the dataset.", ) +@click.option( + "--shapefiles", + type=bool, + default=True, + help="A boolean indicating whether to export outputs as a zipped " + "ESRI Shapefile dataset (this can be slow depending on the size " + "of the analysis).", +) @click.option( "--include-styles/--no-include-styles", is_flag=True, default=True, - help="Set this to indicate whether to include styles " "in output GeoPackage file.", + help="Set this to indicate whether to include styles " + "in the output OGC GeoPackage file.", ) def continental_cli( vector_version, @@ -134,14 +143,12 @@ def continental_cli( hotspots, hotspots_radius, baseline_year, + shapefiles, include_styles, ): - - print(hotspots) - - ################# - # Merge vectors # - ################# + ######### + # Setup # + ######### log = configure_logging("Continental layers and hotspots generation") @@ -153,6 +160,10 @@ def continental_cli( output_dir.mkdir(exist_ok=True, parents=True) log.info(f"Writing data to {output_dir}") + ###################### + # Merge tile vectors # + ###################### + # Setup input and output file paths shoreline_paths = ( f"data/interim/vector/{vector_version}/*/" f"annualshorelines*.shp" @@ -161,13 +172,8 @@ def continental_cli( f"data/interim/vector/{vector_version}/*/" f"ratesofchange*.shp" ) - # Output path for geopackage and zipped shapefiles + # Output path for geopackage OUTPUT_GPKG = output_dir / f"coastlines_{continental_version}.gpkg" - OUTPUT_SHPS = output_dir / f"coastlines_{continental_version}.shp.zip" - - # If shapefile zip exists, delete it first - if OUTPUT_SHPS.exists(): - OUTPUT_SHPS.unlink() # Combine annual shorelines into a single continental layer if shorelines: @@ -197,35 +203,54 @@ def continental_cli( else: log.info("Not writing annual rates of change points") - ############################### - # Load DEA CoastLines vectors # - ############################### + # If shapefiles are requested, set up file paths + if shapefiles: - log.info("Generating continental hotspots") + # Output path for zipped shapefiles + OUTPUT_SHPS = output_dir / f"coastlines_{continental_version}.shp.zip" - # Load continental shoreline and rates of change data - try: + # If shapefile zip exists, delete it first + if OUTPUT_SHPS.exists(): + OUTPUT_SHPS.unlink() - # Load continental rates of change data - ratesofchange_gdf = gpd.read_file( - OUTPUT_GPKG, layer="rates_of_change" - ).set_index("uid") + else: + log.info("Not exporting zipped ESRI Shapefile outputs") - # Load continental shorelines data - shorelines_gdf = gpd.read_file( - OUTPUT_GPKG, layer="shorelines_annual" - ).set_index("year") - shorelines_gdf = shorelines_gdf.loc[shorelines_gdf.geometry.is_valid] + ############################ + # Load continental vectors # + ############################ - except (fiona.errors.DriverError, ValueError): + # Load merged continental data into memory if either hotspot + # generation or zipped shapefile exports are required + if hotspots or shapefiles: + + # Load continental shoreline and rates of change data + try: + + # Load continental rates of change data + ratesofchange_gdf = gpd.read_file( + OUTPUT_GPKG, layer="rates_of_change" + ).set_index("uid") + + # Load continental shorelines data + shorelines_gdf = gpd.read_file( + OUTPUT_GPKG, layer="shorelines_annual" + ).set_index("year") + shorelines_gdf = shorelines_gdf.loc[shorelines_gdf.geometry.is_valid] + + log.info( + "Loading continental shoreline and rates of change data into memory" + ) + + except (fiona.errors.DriverError, ValueError): + + raise FileNotFoundError( + "Continental-scale annual shoreline and rates of " + "change layers are required for hotspot generation or " + "shapefile export. Try re-running this analysis with " + "the following settings: `--shorelines True --ratesofchange True`." + ) - raise FileNotFoundError( - "Continental-scale annual shoreline and rates of " - "change layers are required for hotspot generation. " - "Try re-running this analysis with the following " - "settings: `--shorelines True --ratesofchange True`." - ) - ##################### # Generate hotspots # ##################### @@ -234,7 +259,7 @@ def continental_cli( # of hotspots of coastal erosion and growth if hotspots: - log.info("Generating continental hotspots") + log.info("Generating coastal change hotspots") ###################### # Calculate hotspots # @@ -326,66 +351,74 @@ def continental_cli( }, ) - # Add additional WMS fields and add to shapefile - hotspots_gdf = pd.concat( - [hotspots_gdf, wms_fields(gdf=hotspots_gdf)], axis=1 - ) - hotspots_gdf.to_file( - OUTPUT_SHPS, - layer=f"coastlines_{continental_version}_{layer_name}", - schema={ - "properties": vector_schema(hotspots_gdf), - "geometry": "Point", - }, - ) + if shapefiles: + + # Add additional WMS fields and add to shapefile + hotspots_gdf = pd.concat( + [hotspots_gdf, wms_fields(gdf=hotspots_gdf)], axis=1 + ) + hotspots_gdf.to_file( + OUTPUT_SHPS, + layer=f"coastlines_{continental_version}_{layer_name}", + schema={ + "properties": vector_schema(hotspots_gdf), + "geometry": "Point", + }, + ) except ValueError as e: log.exception(f"Failed to generate hotspots with error: {e}") sys.exit(1) - log.info("Writing hotspots complete") + log.info("Writing coastal change hotspots complete") else: - log.info("Not writing hotspots...") + log.info("Not generating coastal change hotspots") ############################ # Export zipped shapefiles # ############################ - if ratesofchange: + if shapefiles: - # Add rates of change points to shapefile zip - # Add additional WMS fields and add to shapefile - ratesofchange_gdf = pd.concat( - [ratesofchange_gdf, wms_fields(gdf=ratesofchange_gdf)], axis=1 - ) + log.info("Started writing outputs as zipped ESRI Shapefiles") - ratesofchange_gdf.to_file( - OUTPUT_SHPS, - layer=f"coastlines_{continental_version}_rates_of_change", - schema={ - "properties": vector_schema(ratesofchange_gdf), - "geometry": "Point", - }, - ) + if ratesofchange: - log.info("Writing rates of change points to zipped shapefiles complete") + # Add rates of change points to shapefile zip + # Add additional WMS fields and add to shapefile + ratesofchange_gdf = pd.concat( + [ratesofchange_gdf, wms_fields(gdf=ratesofchange_gdf)], axis=1 + ) - if shorelines: + ratesofchange_gdf.to_file( + OUTPUT_SHPS, + layer=f"coastlines_{continental_version}_rates_of_change", + schema={ + "properties": vector_schema(ratesofchange_gdf), + "geometry": "Point", + }, + ) - # Add annual shorelines to shapefile zip - shorelines_gdf.to_file( - OUTPUT_SHPS, - layer=f"coastlines_{continental_version}_shorelines_annual", - schema={ - "properties": vector_schema(shorelines_gdf), - "geometry": ["MultiLineString", "LineString"], - }, - ) + log.info( + "Completed writing rates of change points to zipped ESRI Shapefiles" + ) + + if shorelines: + + # Add annual shorelines to shapefile zip + shorelines_gdf.to_file( + OUTPUT_SHPS, + layer=f"coastlines_{continental_version}_shorelines_annual", + schema={ + "properties": vector_schema(shorelines_gdf), + "geometry": ["MultiLineString", "LineString"], + }, + ) - log.info("Writing annual shorelines to zipped shapefiles complete") + log.info("Completed writing annual shorelines to zipped ESRI Shapefiles") ######################### # Add GeoPackage styles # @@ -395,11 +428,11 @@ def continental_cli( styles = gpd.read_file(STYLES_FILE) styles.to_file(OUTPUT_GPKG, layer="layer_styles") - log.info("Writing styles to GeoPackage file complete") + log.info("Writing styles to OGC GeoPackage file complete") else: - log.info("Not writing styles to GeoPackage") + log.info("Not writing styles to OGC GeoPackage") if __name__ == "__main__": diff --git a/notebooks/DEACoastlines_generation_CLI.ipynb b/notebooks/DEACoastlines_generation_CLI.ipynb index 2bd0084..98ee27c 100644 --- a/notebooks/DEACoastlines_generation_CLI.ipynb +++ b/notebooks/DEACoastlines_generation_CLI.ipynb @@ -89,7 +89,7 @@ "outputs": [], "source": [ "config_path = 'configs/dea_coastlines_config_development.yaml'\n", - "study_area = 2\n", + "study_area = 8\n", "raster_version = 'development'\n", "vector_version = 'development'\n", "continental_version = 'development'" @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "id": "dc5632c4-f872-46e5-9762-80b8556f3622", "metadata": {}, "outputs": [ @@ -184,28 +184,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "47397eb8-ddec-46c6-9f7c-431a50bb7eae", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "2023-01-11 23:15:05 INFO Study area 2: Loaded study area grid\n", - "2023-01-11 23:15:16 INFO Study area 2: Loaded virtual product\n", - "Creating reduced resolution tide modelling array\n", - "Modelling tides using FES2014 tide model\n", - "Reprojecting tides into original array\n", - "100%|██████████████████████████████████████| 1610/1610 [00:10<00:00, 157.90it/s]\n", - "2023-01-11 23:15:53 INFO Study area 2: Finished modelling tide heights\n", - "2023-01-11 23:15:53 INFO Study area 2: Calculating low and high tide cutoffs for each pixel\n", - "2023-01-11 23:15:53 INFO Study area 2: Started exporting raster data\n", - "2023-01-11 23:19:42 INFO Study area 2: Completed exporting raster data\n" - ] - } - ], + "outputs": [], "source": [ "!python -m coastlines.raster --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --start_year 1988 --end_year 2021" ] @@ -294,48 +276,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "c23665e0-6d46-4185-b9f0-04a50d856174", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-01-10 00:51:44 INFO Study area 3: Starting vector generation\n", - "2023-01-10 00:51:48 INFO Study area 3: Loaded rasters\n", - "Operating in single z-value, multiple arrays mode\n", - "2023-01-10 00:52:08 INFO Study area 3: Extracted shorelines from raster data\n", - "2023-01-10 00:52:09 INFO Study area 3: Extracted rates of change points\n", - "2023-01-10 00:52:10 ERROR Study area 3: Failed to run process with error No matching signature found\n", - "Traceback (most recent call last):\n", - " File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1792, in generate_vectors_cli\n", - " generate_vectors(\n", - " File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1461, in generate_vectors\n", - " points_gdf = annual_movements(\n", - " File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 802, in annual_movements\n", - " points_gdf[\"index_comp_p1\"] = _point_interp(\n", - " File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 765, in _point_interp\n", - " return array.interp(x=x_vals, y=y_vals, **kwargs)\n", - " File \"/env/lib/python3.8/site-packages/xarray/core/dataarray.py\", line 1746, in interp\n", - " ds = self._to_temp_dataset().interp(\n", - " File \"/env/lib/python3.8/site-packages/xarray/core/dataset.py\", line 3196, in interp\n", - " variables[name] = missing.interp(var, var_indexers, method, **kwargs)\n", - " File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 640, in interp\n", - " interped = interp_func(\n", - " File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 765, in interp_func\n", - " return _interpnd(var, x, new_x, func, kwargs)\n", - " File \"/env/lib/python3.8/site-packages/xarray/core/missing.py\", line 789, in _interpnd\n", - " rslt = func(x, var, xi, **kwargs)\n", - " File \"/env/lib/python3.8/site-packages/scipy/interpolate/_rgi.py\", line 654, in interpn\n", - " return interp(xi)\n", - " File \"/env/lib/python3.8/site-packages/scipy/interpolate/_rgi.py\", line 336, in __call__\n", - " result = evaluate_linear_2d(self.values,\n", - " File \"_rgi_cython.pyx\", line 19, in scipy.interpolate._rgi_cython.__pyx_fused_cpdef\n", - "TypeError: No matching signature found\n" - ] - } - ], + "outputs": [], "source": [ "!python -m coastlines.vector --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --start_year 1988 --end_year 2021 --baseline_year 2021" ] @@ -350,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 11, "id": "518bf214-ce70-4baa-a2ad-28302986fe01", "metadata": {}, "outputs": [ @@ -383,9 +327,10 @@ " tiled rates of change statistics layers into\n", " a single continental-scale rates of change\n", " statistics layer.\n", - " --hotspots BOOLEAN A boolean indicating whether to generate a\n", - " continental-scale hotspots of coastal change\n", - " summary layer.\n", + " --hotspots BOOLEAN A boolean indicating whether to generate\n", + " continental-scale coastal change hotspot\n", + " summary layers (this can be very slow\n", + " depending on the radii requested).\n", " --hotspots_radius INTEGER The distance (in metres) used to generate\n", " coastal change hotspots summary layers. This\n", " controls the spacing of each summary point,\n", @@ -400,6 +345,9 @@ " hotspot summary points. This is typically\n", " the most recent annual shoreline in the\n", " dataset.\n", + " --shapefiles BOOLEAN A boolean indicating whether to export\n", + " outputs as a zipped ESRI Shapefile dataset\n", + " (this can be slow).\n", " --include-styles / --no-include-styles\n", " Set this to indicate whether to include\n", " styles in output GeoPackage file.\n", @@ -421,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 14, "id": "22e3dca9-6a5e-4e6a-ac2c-c3f3b1ae904b", "metadata": {}, "outputs": [ @@ -429,25 +377,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "True\n", - "2023-01-10 03:28:15 INFO Writing data to data/processed/development\n", + "2023-01-16 00:29:06 INFO Writing data to data/processed/development\n", "Warning 1: A geometry of type MULTILINESTRING is inserted into layer shorelines_annual of geometry type LINESTRING, which is not normally allowed by the GeoPackage specification, but the driver will however do it. To create a conformant GeoPackage, if using ogr2ogr, the -nlt option can be used to override the layer geometry type. This warning will no longer be emitted for this combination of layer and feature geometry type.\n", - "2023-01-10 03:28:15 INFO Merging annual shorelines complete\n", - "2023-01-10 03:28:16 INFO Merging rates of change points complete\n", - "2023-01-10 03:28:16 INFO Generating continental hotspots\n", - "2023-01-10 03:28:16 INFO Generating continental hotspots\n", - "2023-01-10 03:28:16 INFO Calculating 10000 m hotspots\n", - "2023-01-10 03:28:16 INFO Calculating 5000 m hotspots\n", - "2023-01-10 03:28:17 INFO Calculating 1000 m hotspots\n", - "2023-01-10 03:28:17 INFO Writing hotspots complete\n", - "^C\n", - "\n", - "Aborted!\n" + "2023-01-16 00:29:07 INFO Merging annual shorelines complete\n", + "2023-01-16 00:29:09 INFO Merging rates of change points complete\n", + "2023-01-16 00:29:10 INFO Loading continental shoreline and rates of change data into memory\n", + "2023-01-16 00:29:10 INFO Generating coastal change hotspots\n", + "2023-01-16 00:29:10 INFO Calculating 10000 m hotspots\n", + "2023-01-16 00:29:10 INFO Calculating 5000 m hotspots\n", + "2023-01-16 00:29:11 INFO Calculating 1000 m hotspots\n", + "2023-01-16 00:29:12 INFO Writing coastal change hotspots complete\n", + "2023-01-16 00:29:12 INFO Started writing outputs as zipped ESRI Shapefiles\n", + "2023-01-16 00:29:20 INFO Completed writing rates of change points to zipped ESRI Shapefiles\n", + "2023-01-16 00:29:22 INFO Completed writing annual shorelines to zipped ESRI Shapefiles\n", + "2023-01-16 00:29:22 INFO Writing styles to OGC GeoPackage file complete\n" ] } ], "source": [ - "!python -m coastlines.continental --vector_version {vector_version} --continental_version {continental_version} --shorelines True --ratesofchange True --hotspots 1 --baseline_year 2021" + "!python -m coastlines.continental --vector_version {vector_version} --continental_version {continental_version} --shorelines True --ratesofchange True --hotspots True --baseline_year 2021" ] }, { @@ -461,27 +409,283 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "6b804fec-39f4-48a9-8525-a78fbf06e084", "metadata": {}, "outputs": [], "source": [ "# Study areas\n", - "study_areas = [759]" + "study_areas = [1, 2, 3, 4, 5, 6, 7, 8]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "c39e6533-f4fd-4a3c-90a5-44f19825c2f5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 36699 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:05:14 INFO Study area 1: Loaded study area grid\n", + "2023-01-13 01:05:21 INFO Study area 1: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1076/1076 [00:04<00:00, 234.61it/s]\n", + "2023-01-13 01:05:49 INFO Study area 1: Finished modelling tide heights\n", + "2023-01-13 01:05:49 INFO Study area 1: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:05:49 INFO Study area 1: Started exporting raster data\n", + "2023-01-13 01:09:49 INFO Study area 1: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "2023-01-13 01:09:56 INFO Study area 1: Starting vector generation\n", + "2023-01-13 01:10:00 INFO Study area 1: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:10:26 INFO Study area 1: Extracted shorelines from raster data\n", + "2023-01-13 01:10:26 INFO Study area 1: Extracted rates of change points\n", + "2023-01-13 01:10:40 INFO Study area 1: Calculated distances to each annual shoreline\n", + "2023-01-13 01:10:41 INFO Study area 1: Calculated rates of change regressions\n", + "2023-01-13 01:10:43 INFO Study area 1: Calculated all of time statistics\n", + "2023-01-13 01:10:43 INFO Study area 1: Calculated rate of change certainty flags\n", + "2023-01-13 01:10:44 INFO Study area 1: Added region attributes and geohash UIDs\n", + "2023-01-13 01:10:47 INFO Study area 1: Output vector files written to data/interim/vector/development/1_development\n", + "2\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 39529 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:10:53 INFO Study area 2: Loaded study area grid\n", + "2023-01-13 01:11:00 INFO Study area 2: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1610/1610 [00:06<00:00, 247.64it/s]\n", + "2023-01-13 01:11:32 INFO Study area 2: Finished modelling tide heights\n", + "2023-01-13 01:11:32 INFO Study area 2: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:11:32 INFO Study area 2: Started exporting raster data\n", + "CPLReleaseMutex: Error = 1 (Operation not permitted)\n", + "2023-01-13 01:14:33 INFO Study area 2: Completed exporting raster data\n", + "2023-01-13 01:14:41 INFO Study area 2: Starting vector generation\n", + "2023-01-13 01:14:45 INFO Study area 2: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:15:08 INFO Study area 2: Extracted shorelines from raster data\n", + "2023-01-13 01:15:08 INFO Study area 2: Extracted rates of change points\n", + "2023-01-13 01:15:27 INFO Study area 2: Calculated distances to each annual shoreline\n", + "2023-01-13 01:15:28 INFO Study area 2: Calculated rates of change regressions\n", + "2023-01-13 01:15:31 INFO Study area 2: Calculated all of time statistics\n", + "2023-01-13 01:15:31 INFO Study area 2: Calculated rate of change certainty flags\n", + "2023-01-13 01:15:32 INFO Study area 2: Added region attributes and geohash UIDs\n", + "2023-01-13 01:15:35 INFO Study area 2: Output vector files written to data/interim/vector/development/2_development\n", + "3\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 43513 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:15:41 INFO Study area 3: Loaded study area grid\n", + "2023-01-13 01:15:46 INFO Study area 3: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1112/1112 [00:04<00:00, 249.85it/s]\n", + "2023-01-13 01:16:13 INFO Study area 3: Finished modelling tide heights\n", + "2023-01-13 01:16:13 INFO Study area 3: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:16:13 INFO Study area 3: Started exporting raster data\n", + "2023-01-13 01:19:30 INFO Study area 3: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n", + "Traceback (most recent call last):\n", + " File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n", + "2023-01-13 01:19:37 INFO Study area 3: Starting vector generation\n", + "2023-01-13 01:19:41 INFO Study area 3: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:20:03 INFO Study area 3: Extracted shorelines from raster data\n", + "2023-01-13 01:20:03 INFO Study area 3: Extracted rates of change points\n", + "2023-01-13 01:20:39 INFO Study area 3: Calculated distances to each annual shoreline\n", + "2023-01-13 01:20:42 INFO Study area 3: Calculated rates of change regressions\n", + "2023-01-13 01:20:49 INFO Study area 3: Calculated all of time statistics\n", + "2023-01-13 01:20:49 INFO Study area 3: Calculated rate of change certainty flags\n", + "2023-01-13 01:20:50 INFO Study area 3: Added region attributes and geohash UIDs\n", + "2023-01-13 01:20:59 INFO Study area 3: Output vector files written to data/interim/vector/development/3_development\n", + "4\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 39401 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:21:05 INFO Study area 4: Loaded study area grid\n", + "2023-01-13 01:21:10 INFO Study area 4: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1385/1385 [00:03<00:00, 396.38it/s]\n", + "2023-01-13 01:21:32 INFO Study area 4: Finished modelling tide heights\n", + "2023-01-13 01:21:32 INFO Study area 4: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:21:32 INFO Study area 4: Started exporting raster data\n", + "CPLReleaseMutex: Error = 1 (Operation not permitted)\n", + "2023-01-13 01:23:39 INFO Study area 4: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "2023-01-13 01:23:46 INFO Study area 4: Starting vector generation\n", + "2023-01-13 01:23:48 INFO Study area 4: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:24:07 INFO Study area 4: Extracted shorelines from raster data\n", + "2023-01-13 01:24:07 INFO Study area 4: Extracted rates of change points\n", + "2023-01-13 01:24:14 INFO Study area 4: Calculated distances to each annual shoreline\n", + "2023-01-13 01:24:14 INFO Study area 4: Calculated rates of change regressions\n", + "2023-01-13 01:24:15 INFO Study area 4: Calculated all of time statistics\n", + "2023-01-13 01:24:15 INFO Study area 4: Calculated rate of change certainty flags\n", + "2023-01-13 01:24:16 INFO Study area 4: Added region attributes and geohash UIDs\n", + "2023-01-13 01:24:18 INFO Study area 4: Output vector files written to data/interim/vector/development/4_development\n", + "5\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 44953 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:24:24 INFO Study area 5: Loaded study area grid\n", + "2023-01-13 01:24:31 INFO Study area 5: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1065/1065 [00:10<00:00, 100.92it/s]\n", + "2023-01-13 01:25:18 INFO Study area 5: Finished modelling tide heights\n", + "2023-01-13 01:25:18 INFO Study area 5: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:25:18 INFO Study area 5: Started exporting raster data\n", + "2023-01-13 01:30:55 INFO Study area 5: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n", + "Traceback (most recent call last):\n", + " File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n", + "2023-01-13 01:31:02 INFO Study area 5: Starting vector generation\n", + "2023-01-13 01:31:11 INFO Study area 5: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:31:35 INFO Study area 5: Extracted shorelines from raster data\n", + "2023-01-13 01:31:36 INFO Study area 5: Extracted rates of change points\n", + "2023-01-13 01:32:19 INFO Study area 5: Calculated distances to each annual shoreline\n", + "2023-01-13 01:32:23 INFO Study area 5: Calculated rates of change regressions\n", + "2023-01-13 01:32:29 INFO Study area 5: Calculated all of time statistics\n", + "2023-01-13 01:32:29 INFO Study area 5: Calculated rate of change certainty flags\n", + "2023-01-13 01:32:30 INFO Study area 5: Added region attributes and geohash UIDs\n", + "2023-01-13 01:32:40 INFO Study area 5: Output vector files written to data/interim/vector/development/5_development\n", + "6\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 39225 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:32:46 INFO Study area 6: Loaded study area grid\n", + "2023-01-13 01:32:53 INFO Study area 6: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1814/1814 [00:07<00:00, 250.34it/s]\n", + "2023-01-13 01:33:28 INFO Study area 6: Finished modelling tide heights\n", + "2023-01-13 01:33:28 INFO Study area 6: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:33:28 INFO Study area 6: Started exporting raster data\n", + "/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: divide by zero encountered in true_divide\n", + " return func(*(_execute_task(a, cache) for a in args))\n", + "/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: invalid value encountered in true_divide\n", + " return func(*(_execute_task(a, cache) for a in args))\n", + "2023-01-13 01:37:46 INFO Study area 6: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n", + "Traceback (most recent call last):\n", + " File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n", + "2023-01-13 01:37:54 INFO Study area 6: Starting vector generation\n", + "2023-01-13 01:37:57 INFO Study area 6: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:38:14 INFO Study area 6: Extracted shorelines from raster data\n", + "2023-01-13 01:38:14 INFO Study area 6: Extracted rates of change points\n", + "2023-01-13 01:38:37 INFO Study area 6: Calculated distances to each annual shoreline\n", + "2023-01-13 01:38:39 INFO Study area 6: Calculated rates of change regressions\n", + "2023-01-13 01:38:43 INFO Study area 6: Calculated all of time statistics\n", + "2023-01-13 01:38:43 INFO Study area 6: Calculated rate of change certainty flags\n", + "2023-01-13 01:38:43 INFO Study area 6: Added region attributes and geohash UIDs\n", + "2023-01-13 01:38:50 INFO Study area 6: Output vector files written to data/interim/vector/development/6_development\n", + "7\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 42111 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:38:56 INFO Study area 7: Loaded study area grid\n", + "2023-01-13 01:39:04 INFO Study area 7: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1978/1978 [00:04<00:00, 400.21it/s]\n", + "2023-01-13 01:39:31 INFO Study area 7: Finished modelling tide heights\n", + "2023-01-13 01:39:31 INFO Study area 7: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:39:31 INFO Study area 7: Started exporting raster data\n", + "CPLReleaseMutex: Error = 1 (Operation not permitted)\n", + "2023-01-13 01:42:26 INFO Study area 7: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999988555908206 seconds, killing\n", + "Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n", + "Traceback (most recent call last):\n", + " File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n", + "2023-01-13 01:42:33 INFO Study area 7: Starting vector generation\n", + "2023-01-13 01:42:35 INFO Study area 7: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:42:51 INFO Study area 7: Extracted shorelines from raster data\n", + "2023-01-13 01:42:51 INFO Study area 7: Extracted rates of change points\n", + "2023-01-13 01:42:57 INFO Study area 7: Calculated distances to each annual shoreline\n", + "2023-01-13 01:42:58 INFO Study area 7: Calculated rates of change regressions\n", + "2023-01-13 01:42:58 INFO Study area 7: Calculated all of time statistics\n", + "2023-01-13 01:42:59 INFO Study area 7: Calculated rate of change certainty flags\n", + "2023-01-13 01:42:59 INFO Study area 7: Added region attributes and geohash UIDs\n", + "2023-01-13 01:43:01 INFO Study area 7: Output vector files written to data/interim/vector/development/7_development\n", + "8\n", + "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 35771 instead\n", + " warnings.warn(\n", + "\n", + "2023-01-13 01:43:07 INFO Study area 8: Loaded study area grid\n", + "2023-01-13 01:43:16 INFO Study area 8: Loaded virtual product\n", + "Creating reduced resolution tide modelling array\n", + "Modelling tides using FES2014 tide model\n", + "Reprojecting tides into original array\n", + "100%|██████████████████████████████████████| 1764/1764 [00:03<00:00, 455.20it/s]\n", + "2023-01-13 01:43:37 INFO Study area 8: Finished modelling tide heights\n", + "2023-01-13 01:43:37 INFO Study area 8: Calculating low and high tide cutoffs for each pixel\n", + "2023-01-13 01:43:37 INFO Study area 8: Started exporting raster data\n", + "/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: divide by zero encountered in true_divide\n", + " return func(*(_execute_task(a, cache) for a in args))\n", + "/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: invalid value encountered in true_divide\n", + " return func(*(_execute_task(a, cache) for a in args))\n", + "2023-01-13 01:46:52 INFO Study area 8: Completed exporting raster data\n", + "distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n", + "Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n", + "Traceback (most recent call last):\n", + " File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n", + "2023-01-13 01:46:59 INFO Study area 8: Starting vector generation\n", + "2023-01-13 01:47:01 INFO Study area 8: Loaded rasters\n", + "Operating in single z-value, multiple arrays mode\n", + "2023-01-13 01:47:15 INFO Study area 8: Extracted shorelines from raster data\n", + "2023-01-13 01:47:15 INFO Study area 8: Extracted rates of change points\n", + "2023-01-13 01:47:25 INFO Study area 8: Calculated distances to each annual shoreline\n", + "2023-01-13 01:47:26 INFO Study area 8: Calculated rates of change regressions\n", + "2023-01-13 01:47:27 INFO Study area 8: Calculated all of time statistics\n", + "2023-01-13 01:47:27 INFO Study area 8: Calculated rate of change certainty flags\n", + "2023-01-13 01:47:28 INFO Study area 8: Added region attributes and geohash UIDs\n", + "2023-01-13 01:47:30 INFO Study area 8: Output vector files written to data/interim/vector/development/8_development\n" + ] + } + ], "source": [ "# Run raster and vector generation for each study area\n", "for study_area in study_areas:\n", " print(study_area)\n", " !python -m coastlines.raster --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --start_year 1988 --end_year 2021\n", - "# !python -m coastlines.vector --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --vector_version {vector_version} --start_year 1988 --end_year 2021 --baseline_year 2021\n", + " !python -m coastlines.vector --config_path {config_path} --study_area {study_area} --raster_version {raster_version} --vector_version {vector_version} --start_year 1988 --end_year 2021 --baseline_year 2021\n", " \n", "# When complete, combine into single continental outputs\n", "# !python -m coastlines.continental --vector_version {vector_version} --continental_version {continental_version} --shorelines True --ratesofchange True --hotspots True --baseline_year 2021"