This repository has been archived by the owner on Jun 1, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 7
/
2_nwis_pull.yml
157 lines (124 loc) · 5.11 KB
/
2_nwis_pull.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
target_default: 2_nwis_pull
packages:
- dataRetrieval
- yaml
- maps
- ggplot2
- ggthemes
- scipiper
- dplyr
- qs
sources:
- 2_nwis_pull/src/inventory_nwis.R
- lib/src/require_local.R
- 2_nwis_pull/src/nwis_pull.R
- 2_nwis_pull/src/compare_nwis_services.R
- 2_nwis_pull/src/nwis_partition.R
- 2_nwis_pull/src/nwis_misc.R
targets:
2_nwis_pull:
depends:
- 2_nwis_pull/inout/nwis_dv_inventory_summary.csv
- 2_nwis_pull/inout/nwis_uv_inventory_summary.csv
- 2_nwis_pull/out/nwis_dv_summary.csv
- 2_nwis_pull/out/nwis_uv_summary.csv
# -- get a CONUS inventory of available data for download -- #
# !! for a new pull, modify end date in '2_nwis_pull/cfg/nwis_pull_params.yml'
nwis_pull_parameters:
command: yaml.load_file('2_nwis_pull/cfg/nwis_pull_params.yml')
nwis_pull_size:
command: I(250000)
# dummy pull dates
# !! update these to rebuild inventory and NWIS pulls
nwis_dv_pull_date:
command: c(I('20220330'))
nwis_uv_pull_date:
command: c(I('20220330'))
# -- get inventory of observations available to download -- #
# dv data
# force rebuilt of the inventories if you want to start over
2_nwis_pull/inout/nwis_dv_inventory.feather.ind:
command: inventory_nwis(inv_ind = target_name,
nwis_pull_params = nwis_pull_parameters,
service = I('dv'))
depends: nwis_dv_pull_date
# write summary table of inventory for quick PR comparison
2_nwis_pull/inout/nwis_dv_inventory_summary.csv:
command: summarize_nwis_inventory(inv_ind = '2_nwis_pull/inout/nwis_dv_inventory.feather.ind',
out_file = target_name)
# uv data
2_nwis_pull/inout/nwis_uv_inventory.feather.ind:
command: inventory_nwis(inv_ind = target_name,
nwis_pull_params = nwis_pull_parameters,
service = I('uv'))
depends: nwis_uv_pull_date
# create a table that compares services
2_nwis_pull/inout/nwis_compare_services.feather.ind:
command: compare_services(dv_ind = '2_nwis_pull/inout/nwis_dv_inventory.feather.ind',
uv_ind = '2_nwis_pull/inout/nwis_uv_inventory.feather.ind', compare_ind = target_name)
# filter the UV inventory to only those sites that are not in DV
# this isn't perfect - as some sites overlap but don't have the same data.
2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind:
command: reduce_redundancy(uv_inv_ind = '2_nwis_pull/inout/nwis_uv_inventory.feather.ind',
compare_ind = '2_nwis_pull/inout/nwis_compare_services.feather.ind',
out_ind = target_name)
# write summary table of inventory for quick PR comparison
2_nwis_pull/inout/nwis_uv_inventory_summary.csv:
command: summarize_nwis_inventory(inv_ind = '2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind',
out_file = target_name)
# -- partition data -- #
# dv
site_type_keep:
command: c(I(c('ST', 'ST-CA', 'ST-DCH', 'ST-TS', 'SP')))
2_nwis_pull/inout/nwis_dv_partition.feather.ind:
command: partition_inventory(inventory_ind = '2_nwis_pull/inout/nwis_dv_inventory.feather.ind',
nwis_pull_size,
site_types = site_type_keep,
partitions_ind = target_name,
pull_date = nwis_dv_pull_date)
#uv
2_nwis_pull/inout/nwis_uv_partition.feather.ind:
command: partition_inventory(inventory_ind = '2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind',
nwis_pull_size,
site_types = site_type_keep,
partitions_ind = target_name,
pull_date = nwis_uv_pull_date)
# -- create and execute tasks from pull table -- #
# prepare a remake-style plan for running each partition as a separate
# remake target in a separate remake file
#dv
nwis_dv_pull_plan:
command: plan_nwis_pull(partitions_ind='2_nwis_pull/inout/nwis_dv_partition.feather.ind',
service = I('dv'))
2_nwis_dv_pull_tasks.yml:
command: create_nwis_pull_makefile(makefile=target_name, task_plan=nwis_dv_pull_plan,
final_targets = I('2_nwis_pull/out/nwis_dv_data.rds.ind'))
2_nwis_pull/out/nwis_dv_data.rds.ind:
command: loop_tasks(task_plan=nwis_dv_pull_plan, task_makefile='2_nwis_dv_pull_tasks.yml',
num_tries=I(30), sleep_on_error=I(20))
depends:
- 2_nwis_pull/src/nwis_pull.R
# add summary and push to github
2_nwis_pull/out/nwis_dv_summary.csv:
command: summarize_nwis_data(
data_ind = '2_nwis_pull/out/nwis_dv_data.rds.ind',
out_file = target_name)
#uv
nwis_uv_pull_plan:
command: plan_nwis_pull(partitions_ind='2_nwis_pull/inout/nwis_uv_partition.feather.ind',
service = I('uv'))
2_nwis_uv_pull_tasks.yml:
command: create_nwis_pull_makefile(makefile=target_name, task_plan=nwis_uv_pull_plan,
final_targets = I('2_nwis_pull/out/nwis_uv_data.rds.ind'))
temp_tasks:
command: create_task_subset(nwis_uv_pull_plan)
2_nwis_pull/out/nwis_uv_data.rds.ind:
command: loop_tasks(task_plan=nwis_uv_pull_plan, task_makefile='2_nwis_uv_pull_tasks.yml',
num_tries=I(30), sleep_on_error=I(20))
depends:
- 2_nwis_pull/src/nwis_pull.R
# add summary and push to github
2_nwis_pull/out/nwis_uv_summary.csv:
command: summarize_nwis_data(
data_ind = '2_nwis_pull/out/nwis_uv_data.rds.ind',
out_file = target_name)