From c7e9dacdfe25566004ca38417fff13103fbcb320 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Fri, 23 Feb 2024 14:00:39 -0800 Subject: [PATCH] fix #156 (#157) --- extras/channel_width/Makefile | 27 ++++++++----------- .../sql/channel_width_measured.sql | 4 +-- .../sql/fiss_stream_sample_sites_events.sql | 8 +++--- extras/discharge/Makefile | 4 +-- extras/precipitation/mean_annual_precip.sh | 3 +-- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/extras/channel_width/Makefile b/extras/channel_width/Makefile index ceef2cf..b585525 100644 --- a/extras/channel_width/Makefile +++ b/extras/channel_width/Makefile @@ -9,12 +9,11 @@ PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1 WSG = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly ORDER BY watershed_group_code") -GENERATED_FILES = .make/bcdata \ - .make/channel_width_measured \ +GENERATED_FILES = .make/channel_width_measured \ .make/channel_width_mapped \ channel_width_analysis.csv \ .make/channel_width_modelled \ - fwa_stream_networks_channel_width.zip + fwa_stream_networks_channel_width.gz GENERATED_TABLES = whse_fish.fiss_stream_sample_sites_sp\ fwapg.channel_width_measured \ @@ -25,13 +24,13 @@ GENERATED_TABLES = whse_fish.fiss_stream_sample_sites_sp\ all: $(GENERATED_FILES) # -------- -# download BC data containing channel width measurements, plus bcfishpass pscis table linking assessments to streams +# MEASURED # -------- -.make/bcdata: +.make/channel_width_measured: sql/channel_width_measured.sql sql/fiss_stream_sample_sites_events.sql mkdir -p .make - bcdata bc2pg WHSE_FISH.FISS_STREAM_SAMPLE_SITES_SP - $(PSQL) -c "CREATE INDEX ON whse_fish.fiss_stream_sample_sites_sp (new_watershed_code)" - bcdata bc2pg WHSE_FISH.PSCIS_ASSESSMENT_SVW + # load to fwapg schema, these are just used temporarily + bcdata bc2pg --schema fwapg WHSE_FISH.FISS_STREAM_SAMPLE_SITES_SP + bcdata bc2pg --schema fwapg WHSE_FISH.PSCIS_ASSESSMENT_SVW # bcfishpass crossings should be separated from streams to make this much more efficient curl -o bcfishpass.gpkg.zip https://bcfishpass.s3.us-west-2.amazonaws.com/freshwater_fish_habitat_accessibility_MODEL.gpkg.zip ogr2ogr \ @@ -43,17 +42,13 @@ all: $(GENERATED_FILES) /vsizip/bcfishpass.gpkg.zip \ crossings rm bcfishpass.gpkg.zip - touch $@ - -# -------- -# MEASURED -# -------- -.make/channel_width_measured: .make/bcdata sql/channel_width_measured.sql sql/fiss_stream_sample_sites_events.sql # match stream sample sites to streams $(PSQL) -f sql/fiss_stream_sample_sites_events.sql # Now load the measured channel widths where we have them, averaging measurements on the same stream - # NOTE - this presumes PSCIS data is already loaded $(PSQL) -f sql/channel_width_measured.sql + # drop the source tables + $(PSQL) -c "drop table fwapg.fiss_stream_sample_sites_sp" + $(PSQL) -c "drop table fwapg.pscis_assessment_svw" touch $@ # -------- @@ -91,7 +86,7 @@ channel_width_analysis.csv: .make/channel_width_mapped .make/channel_width_measu # OUTPUT TABLE # -------- # combine the measured/mapped/modelled data into a single table for easy relating to streams table -fwa_stream_networks_channel_width.zip: .make/channel_width_modelled sql/channel_width.sql +fwa_stream_networks_channel_width.gz: .make/channel_width_modelled sql/channel_width.sql $(PSQL) -c "truncate whse_basemapping.fwa_stream_networks_channel_width;" parallel $(PSQL) -f sql/channel_width.sql -v wsg={1} ::: $(WSG) # dump output to file diff --git a/extras/channel_width/sql/channel_width_measured.sql b/extras/channel_width/sql/channel_width_measured.sql index f75e5f4..4579f69 100644 --- a/extras/channel_width/sql/channel_width_measured.sql +++ b/extras/channel_width/sql/channel_width_measured.sql @@ -26,7 +26,7 @@ WITH fiss_measurements AS w.watershed_group_code, p.channel_width as channel_width_fiss FROM fwapg.fiss_stream_sample_sites_events_sp e -INNER JOIN whse_fish.fiss_stream_sample_sites_sp p +INNER JOIN fwapg.fiss_stream_sample_sites_sp p ON e.stream_sample_site_id = p.stream_sample_site_id LEFT OUTER JOIN whse_basemapping.fwa_watersheds_poly w ON ST_Intersects(p.geom, w.geom) @@ -48,7 +48,7 @@ SELECT s.watershed_group_code, a.downstream_channel_width as channel_width_pscis FROM fwapg.pscis_crossings e -LEFT OUTER JOIN whse_fish.pscis_assessment_svw a +LEFT OUTER JOIN fwapg.pscis_assessment_svw a ON e.stream_crossing_id = a.stream_crossing_id LEFT OUTER JOIN whse_basemapping.fwa_watersheds_poly w ON ST_Intersects(e.geom, w.geom) diff --git a/extras/channel_width/sql/fiss_stream_sample_sites_events.sql b/extras/channel_width/sql/fiss_stream_sample_sites_events.sql index c0dce92..3ad42e1 100644 --- a/extras/channel_width/sql/fiss_stream_sample_sites_events.sql +++ b/extras/channel_width/sql/fiss_stream_sample_sites_events.sql @@ -8,7 +8,7 @@ CREATE TABLE fwapg.fiss_stream_sample_sites_events_sp AS WITH pts AS ( SELECT pt.stream_sample_site_id, pt.geom - FROM whse_fish.fiss_stream_sample_sites_sp as pt + FROM fwapg.fiss_stream_sample_sites_sp as pt INNER JOIN whse_basemapping.fwa_watershed_groups_poly wsd ON ST_Intersects(pt.geom, wsd.geom) ), @@ -85,7 +85,7 @@ FROM bluelines INNER JOIN candidates ON bluelines.stream_sample_site_id = candidates.stream_sample_site_id AND bluelines.blue_line_key = candidates.blue_line_key AND bluelines.distance_to_stream = candidates.distance_to_stream -INNER JOIN whse_fish.fiss_stream_sample_sites_sp pts +INNER JOIN fwapg.fiss_stream_sample_sites_sp pts ON bluelines.stream_sample_site_id = pts.stream_sample_site_id INNER JOIN whse_basemapping.fwa_stream_networks_sp s ON candidates.linear_feature_id = s.linear_feature_id @@ -107,7 +107,7 @@ WITH pts AS SELECT pt.stream_sample_site_id, pt.geom - FROM whse_fish.fiss_stream_sample_sites_sp as pt + FROM fwapg.fiss_stream_sample_sites_sp as pt INNER JOIN whse_basemapping.fwa_watershed_groups_poly wsd ON ST_Intersects(pt.geom, wsd.geom) LEFT OUTER JOIN fwapg.fiss_stream_sample_sites_events_sp e @@ -171,7 +171,7 @@ indexed AS ) )).geom::geometry(PointZM, 3005) as geom FROM candidates c -INNER JOIN whse_fish.fiss_stream_sample_sites_sp pts +INNER JOIN fwapg.fiss_stream_sample_sites_sp pts ON c.stream_sample_site_id = pts.stream_sample_site_id INNER JOIN whse_basemapping.fwa_stream_networks_sp s ON c.linear_feature_id = s.linear_feature_id diff --git a/extras/discharge/Makefile b/extras/discharge/Makefile index c47b862..66f1b93 100644 --- a/extras/discharge/Makefile +++ b/extras/discharge/Makefile @@ -10,7 +10,7 @@ TARGETS = data/baseflow.nc \ .make/discharge01_raster \ .make/discharge02_load \ .make/discharge03_wsd \ - .make/discharge + fwa_stream_networks_discharge.gz TABLES = fwapg.discharge01_raster \ fwapg.discharge02_load \ @@ -96,7 +96,7 @@ data/discharge.nc: data/runoff_yearsum_mean.nc data/baseflow_yearsum_mean.nc touch $@ ## For habitat modelling, it is easier if discharge is per-stream -fwa_stream_networks_discharge.zip: .make/discharge03_wsd +fwa_stream_networks_discharge.gz: .make/discharge03_wsd # Transfer per-watershed discharge data to per-stream $(PSQL) -c "truncate whse_basemapping.fwa_stream_networks_discharge" for wsg in $(WSGS) ; do \ diff --git a/extras/precipitation/mean_annual_precip.sh b/extras/precipitation/mean_annual_precip.sh index 9e699fd..0756d96 100755 --- a/extras/precipitation/mean_annual_precip.sh +++ b/extras/precipitation/mean_annual_precip.sh @@ -139,5 +139,4 @@ rm data/MAP.tif* # dump output to file $PSQL -c "\copy whse_basemapping.fwa_stream_networks_mean_annual_precip TO 'fwa_stream_networks_mean_annual_precip.csv' DELIMITER ',' CSV HEADER;" -zip -r fwa_stream_networks_mean_annual_precip.zip fwa_stream_networks_mean_annual_precip.csv -rm fwa_stream_networks_mean_annual_precip.csv \ No newline at end of file +gzip fwa_stream_networks_mean_annual_precip.csv \ No newline at end of file