forked from Parallel-NetCDF/E3SM-IO
-
Notifications
You must be signed in to change notification settings - Fork 0
319 lines (315 loc) · 12.5 KB
/
ubuntu_ompi.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
name: OpenMPI
on:
push:
branches:
- master
- github_action
paths-ignore:
- '**/*.md'
- '**/*.txt'
- '**/*.jpg'
- '**/*.png'
- 'tests/*'
- 'datasets/*'
pull_request:
branches: master
paths-ignore:
- '**/*.md'
- '**/*.txt'
- '**/*.jpg'
- '**/*.png'
- 'tests/*'
- 'datasets/*'
env:
PNETCDF_VERSION: 1.12.3
HDF5_VERSION: 1.14.2
NETCDF4_VERSION: 4.9.2
ADIOS_VERSION: 2.8.3
LOG_VOL_VERSION: 1.4.0
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- name: Set up dependencies
run: |
set -x
sudo apt-get update
sudo apt-get install automake autoconf libtool libtool-bin m4
# zlib
sudo apt-get install zlib1g-dev
# OpenMPI
sudo apt-get install openmpi-bin openmpi-common libopenmpi-dev libgtk2.0-dev
echo "---- location of OpenMPI C compiler ----"
which mpicc
- name: Install PnetCDF
run: |
set -x
cd ${GITHUB_WORKSPACE}
echo "Install PnetCDF on ${GITHUB_WORKSPACE}/PnetCDF"
rm -rf PnetCDF ; mkdir PnetCDF ; cd PnetCDF
curl -LO https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz
tar -zxf pnetcdf-${PNETCDF_VERSION}.tar.gz
cd pnetcdf-${PNETCDF_VERSION}
./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \
--silent \
--disable-fortran \
--disable-cxx
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s distclean >> qout 2>&1
- name: Dump PnetCDF log file
if: ${{ failure() }}
run: |
set -x
cat ${GITHUB_WORKSPACE}/PnetCDF/pnetcdf-${PNETCDF_VERSION}/qout
cat ${GITHUB_WORKSPACE}/PnetCDF/pnetcdf-${PNETCDF_VERSION}/config.log
- name: Install HDF5
if: ${{ success() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
echo "Install HDF5 on ${GITHUB_WORKSPACE}/HDF5"
rm -rf HDF5 ; mkdir HDF5 ; cd HDF5
VER_MAJOR=${HDF5_VERSION%.*}
VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
tar -zxf hdf5-${HDF5_VERSION}.tar.gz
cd hdf5-${HDF5_VERSION}
./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \
--silent \
--enable-hl \
--enable-parallel \
--enable-build-mode=production \
--disable-doxygen-doc \
--disable-doxygen-man \
--disable-doxygen-html \
--disable-tools \
--disable-tests \
--disable-fortran \
--disable-cxx \
CC=mpicc
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s distclean >> qout 2>&1
- name: Dump HDF5 log file
if: ${{ failure() }}
run: |
set -x
cat ${GITHUB_WORKSPACE}/HDF5/hdf5-${HDF5_VERSION}/qout
cat ${GITHUB_WORKSPACE}/HDF5/hdf5-${HDF5_VERSION}/config.log
- name: Install NetCDF-C
if: ${{ success() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
echo "Install NetCDF-C on ${GITHUB_WORKSPACE}/NetCDF"
rm -rf NetCDF ; mkdir NetCDF ; cd NetCDF
curl -LO https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF4_VERSION}.tar.gz
tar -zxf v${NETCDF4_VERSION}.tar.gz
cd netcdf-c-${NETCDF4_VERSION}
# TODO: Always use an official release. E3SM-IO requires NetCDF-C 4.9.0 or later.
# rm -rf netcdf-c
# git clone -q https://github.com/Unidata/netcdf-c.git
# cd netcdf-c
# autoreconf -i
./configure --prefix=${GITHUB_WORKSPACE}/NetCDF \
--silent \
--disable-dap \
--disable-nczarr \
--disable-nczarr-filters \
--disable-filter-testing \
--disable-byterange \
CC=mpicc \
CPPFLAGS="-I${GITHUB_WORKSPACE}/HDF5/include" \
LDFLAGS="-L${GITHUB_WORKSPACE}/HDF5/lib" \
LIBS="-lhdf5"
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s distclean >> qout 2>&1
- name: Dump NetCDF-C log file
if: ${{ failure() }}
run: |
set -x
cat ${GITHUB_WORKSPACE}/NetCDF/netcdf-c-${NETCDF4_VERSION}/qout
cat ${GITHUB_WORKSPACE}/NetCDF/netcdf-c-${NETCDF4_VERSION}/config.log
- name: Install ADIOS2
if: ${{ success() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
echo "Install ADIOS2 on ${GITHUB_WORKSPACE}/ADIOS"
rm -rf ADIOS ; mkdir ADIOS ; cd ADIOS
curl -LO https://github.com/ornladios/ADIOS2/archive/refs/tags/v${ADIOS_VERSION}.tar.gz
tar -zxf v${ADIOS_VERSION}.tar.gz
mkdir adios2_build
cd adios2_build
cmake -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/ADIOS \
-DADIOS2_USE_MPI=ON \
-DADIOS2_USE_Fortran=OFF \
-DADIOS2_USE_Python=OFF \
-DADIOS2_USE_ZeroMQ=OFF \
-DADIOS2_USE_HDF5=OFF \
-DADIOS2_USE_IME=OFF \
-DADIOS2_USE_BZip2=OFF \
-DADIOS2_USE_ZFP=OFF \
-DADIOS2_USE_SZ=OFF \
-DADIOS2_USE_MGARD=OFF \
-DADIOS2_USE_PNG=OFF \
-DADIOS2_USE_Blosc=OFF \
-DADIOS2_BUILD_EXAMPLES=OFF \
-DBUILD_TESTING=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DADIOS2_USE_SST=ON \
-DADIOS2_USE_BP5=ON \
../ADIOS2-${ADIOS_VERSION}
make -j 8 install > qout 2>&1
make clean >> qout 2>&1
- name: Dump ADIOS log file
if: ${{ failure() }}
run: |
set -x
cat ${GITHUB_WORKSPACE}/ADIOS/adios2_build/qout
- name: Install Log VOL
if: ${{ success() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
echo "Install Log VOL connector on ${GITHUB_WORKSPACE}/LOGVOL"
rm -rf LOGVOL ; mkdir LOGVOL ; cd LOGVOL
curl -LO https://github.com/DataLib-ECP/vol-log-based/archive/refs/tags/logvol.${LOG_VOL_VERSION}.tar.gz
tar -zxf logvol.${LOG_VOL_VERSION}.tar.gz
cd vol-log-based-logvol.${LOG_VOL_VERSION}
autoreconf -i
./configure --prefix=${GITHUB_WORKSPACE}/LOGVOL \
--silent \
--with-hdf5=${GITHUB_WORKSPACE}/HDF5
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s distclean >> qout 2>&1
- name: Dump log file
if: ${{ failure() }}
run: |
set -x
cat ${GITHUB_WORKSPACE}/LOGVOL/vol-log-based-logvol.${LOG_VOL_VERSION}/qout
cat ${GITHUB_WORKSPACE}/LOGVOL/vol-log-based-logvol.${LOG_VOL_VERSION}/config.log
- name: Build and check E3SM_IO with PnetCDF only
if: ${{ success() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
rm -rf ./test_output
autoreconf -i
./configure --with-pnetcdf=${GITHUB_WORKSPACE}/PnetCDF \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments
make -j 8
make check
- name: Dump log files if E3SM_IO with PnetCDF-only test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
- name: Build and check E3SM_IO with HDF5 only
if: ${{ success() }}
run: |
set -x
make -s distclean
./configure --with-hdf5=${GITHUB_WORKSPACE}/HDF5 \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments
make -j 8
make check
- name: Dump log files if E3SM_IO with HDF5-only test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
- name: Build and check E3SM_IO with HDF5 Log VOL connector only
if: ${{ success() }}
run: |
set -x
make -s distclean
./configure --with-hdf5=${GITHUB_WORKSPACE}/HDF5 \
--with-logvol=${GITHUB_WORKSPACE}/LOGVOL \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments
make -j 8
make check
- name: Dump log files if E3SM_IO with HDF5 Log VOL connector test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
- name: Build and check E3SM_IO with NetCDF4 only
if: ${{ success() }}
run: |
set -x
make -s distclean
./configure --with-netcdf4=${GITHUB_WORKSPACE}/NetCDF \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments
make -j 8
make check
- name: Dump log files if E3SM_IO with NetCDF4-only test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
- name: Build and check E3SM_IO with ADIOS only
if: ${{ success() }}
run: |
set -x
make -s distclean
./configure --with-adios2=${GITHUB_WORKSPACE}/ADIOS \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments
make -j 8
make check
- name: Dump log files if E3SM_IO with ADIOS-only test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
- name: Build and check E3SM_IO with all APIs
if: ${{ success() }}
run: |
set -x
make -s distclean
./configure --with-pnetcdf=${GITHUB_WORKSPACE}/PnetCDF \
--with-hdf5=${GITHUB_WORKSPACE}/HDF5 \
--with-netcdf4=${GITHUB_WORKSPACE}/NetCDF \
--with-logvol=${GITHUB_WORKSPACE}/LOGVOL \
--with-adios2=${GITHUB_WORKSPACE}/ADIOS \
CC=mpicc CXX=mpicxx \
CFLAGS=-fno-var-tracking-assignments \
CXXFLAGS=-fno-var-tracking-assignments \
TESTMPIRUN="mpiexec --oversubscribe"
make -j 8
make check
- name: Dump log files if E3SM_IO test failed
if: ${{ failure() }}
run: |
set -x
cd ${GITHUB_WORKSPACE}
cat test.sh.log utils/*.log
cat config.log
ls -r ./test_output
- name: Test E3SM_IO all APIs -- parallel runs
if: ${{ success() }}
run: |
set -x
make ptest
- name: Test make distcheck
if: ${{ success() }}
run: |
set -x
make distcheck DISTCHECK_CONFIGURE_FLAGS="--with-pnetcdf=${GITHUB_WORKSPACE}/PnetCDF --with-hdf5=${GITHUB_WORKSPACE}/HDF5 --with-netcdf4=${GITHUB_WORKSPACE}/NetCDF --with-logvol=${GITHUB_WORKSPACE}/LOGVOL --with-adios2=${GITHUB_WORKSPACE}/ADIOS CC=mpicc CXX=mpicxx CFLAGS=-fno-var-tracking-assignments CXXFLAGS=-fno-var-tracking-assignments"
make distclean