Skip to content

Commit

Permalink
pin environment pkgs, update for new package APIs, save 2 extra vecto…
Browse files Browse the repository at this point in the history
…r datasets (#10)

* save two additional vector datasets

* pin packages in env

* update upload_pkgs for new distributed API

* update spatial for new (geo)pandas API

* update HISTORY
bolliger32 authored Oct 15, 2023
1 parent d009d85 commit 234bc08
Showing 6 changed files with 98 additions and 152 deletions.
6 changes: 6 additions & 0 deletions HISTORY.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
History
=======

v1.1.2
------
* Add specific package version pins to environment.yml
* Update `spatial.py` and `notebooks/data-processing/2-present-day-exposure/03-create-segment-regions` to account for some package API changes
* Save two additional vector datasets: Region linestrings and region Voronoi polygons

v1.1.1
------
* Update settings file for local development
62 changes: 31 additions & 31 deletions environment/environment.yml
Original file line number Diff line number Diff line change
@@ -2,34 +2,34 @@ name: sliiders
channels:
- conda-forge
dependencies:
- beautifulsoup4
- cartopy
- contextily
- dask-geopandas
- distributed
- fsspec
- jupyterlab
- matplotlib
- networkx
- numba
- numpy
- numpy_groupies
- pandas
- pandas-datareader
- pint
- pygeos
- pyinterp
- python
- pypdf2
- pyreadr
- regionmask
- requests
- rioxarray
- scikit-learn
- scipy
- shapely
- statsmodels
- tqdm
- xarray
- xesmf
- zarr
- beautifulsoup4=4.12.2
- cartopy=0.22.0
- contextily=1.4.0
- dask-geopandas=0.3.1
- distributed=2023.8.1
- fsspec=2023.9.0
- jupyterlab=4.0.5
- matplotlib=3.8.0
- networkx=3.1
- numba=0.57.1
- numpy=1.24.4
- numpy_groupies=0.9.22
- pandas=2.1.0
- pandas-datareader=0.10.0
- pint=0.22
- pygeos=0.14
- pyinterp=2023.10.1
- pypdf2=2.11.1
- pyreadr=0.4.9
- python=3.10.12
- regionmask=0.11.0
- requests=2.31.0
- rioxarray=0.15.0
- scikit-learn=1.3.0
- scipy=1.11.2
- shapely=2.0.1
- statsmodels=0.14.0
- tqdm=4.66.1
- xarray=2023.8.0
- xesmf=0.8.1
- zarr=2.16.1

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion sliiders/dask.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,10 @@

from sliiders import __file__ as sliiders_path

import uuid
import sys


def upload_sliiders(client, restart_client=True):
"""Upload a local package to Dask Workers. After calling this function, the package
contained at ``pkg_dir`` will be available on all workers in your Dask cluster,
@@ -20,7 +23,7 @@ def upload_sliiders(client, restart_client=True):
**kwargs
Passed directly to :py:class:`distributed.diagnostics.plugin.UploadDirectory`
"""
client.register_worker_plugin(
client.register_plugin(
UploadDirectory(
Path(sliiders_path).parents[1],
update_path=True,
2 changes: 2 additions & 0 deletions sliiders/settings.py
Original file line number Diff line number Diff line change
@@ -103,6 +103,7 @@
PATH_SEG_PTS_MANUAL = DIR_GEOG_INT / "ciam_segment_pts_manual_adds.parquet"

PATH_SEG_CENTROIDS = DIR_GEOG_INT / "gtsm_stations_thinned_ciam.parquet"
PATH_SEGS = DIR_GEOG_INT / "coastal_segments.parquet"

PATH_GEOG_COASTLINES = DIR_GEOG_INT / "ne_coastline_lines_CIAM_wexp_or_gtsm_10m.parquet"

@@ -118,6 +119,7 @@
)

DIR_CIAM_VORONOI = DIR_GEOG_INT / "seg_region_intersections" / EXPOSURE_BINNED_VERS
PATH_SEG_VORONOI = DIR_CIAM_VORONOI / "seg.parquet"
PATH_SEG_REGION_VORONOI_INTERSECTIONS = (
DIR_CIAM_VORONOI / "seg_region_intersections.parquet"
)
17 changes: 7 additions & 10 deletions sliiders/spatial.py
Original file line number Diff line number Diff line change
@@ -1741,12 +1741,7 @@ def get_voronoi_regions(full_regions):

out_cols = [c for c in full_regions.columns if c != "geometry"]

# avoiding GeoDataFrame.explode until geopandas v0.10.3 b/c of
# https://github.com/geopandas/geopandas/issues/2271
# region_polys = full_regions.explode(index_parts=False)
region_polys = full_regions.drop(columns="geometry").join(
full_regions.geometry.explode(index_parts=False)
)
region_polys = full_regions.explode(index_parts=False)

print("...Subdividing region grid to ease computation")
gridded_gdf, all_oc = grid_gdf(region_polys)
@@ -1758,7 +1753,7 @@ def get_voronoi_regions(full_regions):
vor_gdf = get_spherical_voronoi_gser(pts).to_frame()

vor_shapes = pygeos.from_shapely(vor_gdf["geometry"])
all_gridded = pygeos.from_shapely(gridded_gdf["geometry"])
all_gridded = gridded_gdf["geometry"].values

tree = pygeos.STRtree(all_gridded)

@@ -2285,8 +2280,9 @@ def get_coastlines_by_iso(coastlines, regions_voronoi, plot=True):

# Check output
if plot:
coastlines.reset_index(drop=False).plot(
color=add_rand_color(coastlines, col="ISO"), figsize=(20, 20)
tmp = out.reset_index(drop=False)
tmp.plot(
color=add_rand_color(tmp, col="ISO"), figsize=(20, 20)
)

return out
@@ -2388,6 +2384,7 @@ def create_overlay_voronois(
# Generate global Voronoi shapes for regions
print("Generating global Voronoi shapes for regions...")
reg_vor = get_voronoi_regions(regions)

adm0 = reg_vor.dissolve("ISO")

# Assign ISO to seg centroids based on country Voronoi
@@ -2438,7 +2435,7 @@ def create_overlay_voronois(
out = _drop_tiny(out, min_sq_degrees, overlay_name)
adm0 = _drop_tiny(adm0, min_sq_degrees, "ISO")

return out, adm0
return out, adm0, coastal_segs


def _drop_tiny(df, min_sq_degrees, colname):

0 comments on commit 234bc08

Please sign in to comment.