From 78513bbb7b4e95b3f5d90a623a38472dc06dabb8 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 14:28:36 -0600 Subject: [PATCH 01/13] Add Python script to build binary package. Fix 'make dist' errors. --- Makefile.am | 16 +- configure.ac | 9 +- docker/build_binary.py | 401 ++++++++++++++++++++++++ docker/geomodelgrids-userenv | 5 +- docs/Makefile.am | 44 +-- libsrc/geomodelgrids/serial/Makefile.am | 3 + setup.cfg | 2 +- tests/pytests/Makefile.am | 2 +- 8 files changed, 455 insertions(+), 27 deletions(-) create mode 100755 docker/build_binary.py diff --git a/Makefile.am b/Makefile.am index 995d463..5e0e3e9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,8 +29,20 @@ EXTRA_DIST = \ code.json \ developer/autopep8.cfg \ developer/uncrustify.cfg \ - docker/debian-stable-gcc8 \ - docker/geomodelgrids-testenv + docker/README.md \ + docker/build_binary.py \ + docker/builder.py \ + docker/update_testenv.py \ + docker/debian-stable \ + docker/debian-testing \ + docker/ubuntu-20.04 \ + docker/ubuntu-20.10 \ + docker/fedora-32 \ + docker/fedora-33 \ + docker/centos-8 \ + docker/geomodelgrids-testenv \ + docker/geomodelgrids-devenv \ + docker/geomodelgrids-userenv .PHONY: coverage-libtests coverage-html clean-coverage diff --git a/configure.ac b/configure.ac index 0fe8f1f..6e908e9 100644 --- a/configure.ac +++ b/configure.ac @@ -127,6 +127,10 @@ AC_PROG_CC AC_PROG_LIBTOOL AC_PROG_INSTALL +# Require C++-11 +AX_CXX_COMPILE_STDCXX(11) + + AC_PROG_LIBTOOL if test "$allow_undefined_flag" = unsupported; then # See issue119. @@ -197,9 +201,6 @@ fi # CPPUNIT if test "$enable_testing" = "yes" ; then - # Require C++-11 - AX_CXX_COMPILE_STDCXX(11) - if test "$with_cppunit_incdir" != no; then CPPUNIT_INCLUDES="-I$with_cppunit_incdir" fi @@ -294,7 +295,7 @@ AC_CONFIG_FILES([Makefile tests/libtests/utils/Makefile tests/libtests/serial/Makefile tests/libtests/apps/Makefile - tests/pytests/Makefile + tests/pytests/Makefile docs/Makefile models/Makefile ]) diff --git a/docker/build_binary.py b/docker/build_binary.py new file mode 100755 index 0000000..96eef05 --- /dev/null +++ b/docker/build_binary.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Create binary package. + +1. Install dependencies. + * gcc (Linux only) + * sqlite3 + * libtiff + * proj + * hdf5 +2. Install geomodelgrids +3. Create setup.sh +4. Create tarball. +""" + +import argparse +import multiprocessing +import os +import pathlib +import platform +import requests +import shutil +import subprocess +import tarfile +import pkg_resources + + +# -------------------------------------------------------------------------------------------------- +class Package(object): + + def __init__(self, install_dir, env, show_progress=True): + self.install_dir = install_dir + self.env = env + self.nthreads = multiprocessing.cpu_count() + self.show_progress = show_progress + + def install(self): + self.download() + self.extract_tarball() + self.configure() + self.build() + + def download(self, url=None, tarball=None): + if not url: + url = self.URL + if not tarball: + tarball = self.TARBALL + if not os.path.isfile(tarball): + self._status(f"Downloading {url} ...") + response = requests.get(url, stream=True) + if response.status_code == 200: + with open(tarball, 'wb') as f: + f.write(response.raw.read()) + assert tarfile.is_tarfile(tarball) + else: + raise IOError(f"Could not download {url}. Response status={response.status_code}.") + + def extract_tarball(self, tarball=None, path="."): + if not tarball: + tarball = self.TARBALL + self._status(f"Extracting {tarball} ...") + tfile = tarfile.open(tarball, mode="r:*") + tfile.extractall(path=path) + + def configure(self, args): + if not os.path.exists(self.BUILD_DIR): + os.mkdir(self.BUILD_DIR) + cwd = os.path.abspath(os.curdir) + os.chdir(self.BUILD_DIR) + configure = os.path.join("..", self.SRC_DIR, "configure") + self._run_cmd([configure] + args) + os.chdir(cwd) + return + + def build(self): + cwd = os.path.abspath(os.curdir) + print(f"CWD={cwd}") + os.chdir(self.BUILD_DIR) + self._run_cmd(["make", f"-j{self.nthreads}"]) + self._run_cmd(["make", "install"]) + os.chdir(cwd) + return + + def _run_cmd(self, cmd): + self._status("Running {} ...".format(" ".join(cmd))) + result = subprocess.run(cmd, env=self.env, check=True) + + def _status(self, msg): + if self.show_progress: + print(msg) + + +# -------------------------------------------------------------------------------------------------- +class GCC(Package): + VERSION = "10.3.0" + TARBALL = f"gcc-10.3.0.tar.gz" + URL = f"https://mirrors.kernel.org/gnu/gcc/gcc-{VERSION}/{TARBALL}" + BUILD_DIR = "gcc-build" + SRC_DIR = f"gcc-{VERSION}" + + EXTRAS = ( + "gmp": { + "version": "6.2.1", + "tarball": "gmp-{version}.tar.bz2", + "url": "https://mirrors.kernel.org/gnu/gmp/{tarball}", + }, + "mpc": { + "version": "1.2.1", + "tarball": "mpi-{version}.tar.gz", + "url": "https://mirrors.kernel.org/gnu/mpc/{tarball}", + }, + "mpfr": { + "version": "4.1.0", + "tarball": "mpfr-{version}.tar.gz", + "url": "https://mirrors.kernel.org/gnu/mpfr/{tarball}", + } + ) + + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-languages=c,c++", + "--disable-multilib", + ] + super().configure(ARGS) + + + def install(self): + self.download() + self.extract_tarball() + + gcc_src = f"gcc-{VERSION}" + for label,pkg in self.EXTRAS.items(): + tarball = pkg["tarball"].format(version=pkg["version"]) + url = pkg["url"].format(version=pkg["version"], tarball=tarball) + + self.download(url=url, tarball=tarball) + self.extract_tarball(path=gcc_src, tarball=tarball) + + link_src = os.path.join(gcc_sc, f"{label}-{pkg['version']}") + link_dest = os.path.join(gcc_sc, label) + os.symlink(link_src, link_dest) + + self.configure() + self.build() + +# -------------------------------------------------------------------------------------------------- +class Sqlite(Package): + VERSION = "3350500" + TARBALL = f"sqlite-autoconf-{VERSION}.tar.gz" + URL = f"https://www.sqlite.org/2021/{TARBALL}" + BUILD_DIR = "sqlite-build" + SRC_DIR = f"sqlite-autoconf-{VERSION}" + + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + ] + super().configure(ARGS) + + +# -------------------------------------------------------------------------------------------------- +class Tiff(Package): + VERSION = "4.3.0" + TARBALL = f"tiff-{VERSION}.tar.gz" + URL = f"http://download.osgeo.org/libtiff/{TARBALL}" + BUILD_DIR = "tiff-build" + SRC_DIR = f"tiff-{VERSION}" + + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + ] + super().configure(ARGS) + + +# -------------------------------------------------------------------------------------------------- +class Proj(Package): + VERSION = "7.2.1" + TARBALL = f"proj-{VERSION}.tar.gz" + URL = f"https://download.osgeo.org/proj/{TARBALL}" + BUILD_DIR = "proj-build" + SRC_DIR = f"proj-{VERSION}" + + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + f"SQLITE3_CFLAGS=-I{self.install_dir}/include", + f"SQLITE3_LIBS=-L{self.install_dir}/lib -lsqlite3", + f"TIFF_CFLAGS=-I{self.install_dir}/include", + f"TIFF_LIBS=-L{self.install_dir}/lib -ltiff", + ] + super().configure(ARGS) + + + def install(self): + super().install() + + cmd = ["projsync", "--system-directory", "--bbox -128.0,34.0,-118.0,42.0"] + self._run_cmd(cmd) + + +# -------------------------------------------------------------------------------------------------- +class HDF5(Package): + VERSION = "1.12.0" + TARBALL = f"hdf5-{VERSION}.tar.gz" + URL = f"https://www.sqlite.org/2021/{TARBALL}" + BUILD_DIR = "hdf5-build" + SRC_DIR = f"hdf5-{VERSION}" + + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + "--disable-static", + ] + super().configure(ARGS) + + +# -------------------------------------------------------------------------------------------------- +class GeoModelGrids(Package): + BUILD_DIR = "geomodelgrids-build" + + def install(self): + self.VERSION = pkg_resources.get_distribution("geomodelgrids").version + self.SRC_DIR = f"geomodelgrids-{self.VERSION}" + self.TARBALL = f"geomodelgrids-{self.VERSION}.tar.gz" + + self.extract_tarball() + self.configure() + self.build() + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + "--disable-python", + "--disable-gdal", + "--disable-testing", + ] + super().configure(ARGS) + + +# -------------------------------------------------------------------------------------------------- +class Darwin(object): + + @staticmethod + def update_linking(install_dir): + path = pathlib.Path(install_dir) + for proj in path.glob("bin/*"): + Darwin.update_deplibs(proj) + + for lib in path.glob("lib/*.dylib"): + Darwin.update_deplibs(lib) + + @staticmethod + def update_deplibs(filename): + if filename.is_symlink() or filename.is_dir(): + return + + proc = subprocess.run(["otool", "-L", filename], stdout=subprocess.PIPE, check=True) + output = proc.stdout.decode("utf-8") + deplibs = [] + for line in output.split("\t")[1:]: + deplibs.append(line.split()[0]) + for libPathAbs in deplibs: + if libPathAbs.startswith("/usr") or libPathAbs.startswith("/System"): + continue + libName = os.path.split(libPathAbs)[1] + libPathNew = f"@executable_path/../lib/{libName}" + cmd = ["install_name_tool", "-change", libPathAbs, libPathNew, str(filename)] + subprocess.run(cmd, check=True) + + +# -------------------------------------------------------------------------------------------------- +class App(object): + + def __init__(self): + self.show_progress = True + + def main(self, **kwargs): + args = argparse.Namespace(**kwargs) if kwargs else self._parse_command_line() + self.show_progress = args.show_progress + self.install_dir = os.path.abspath(args.install_dir) + self._set_env() + + if (args.install_gcc or args.all) and platform.system() == "Linux": + self._install(GCC) + if args.install_sqlite or args.all: + self._install(Sqlite) + if args.install_tiff or args.all: + self._install(Tiff) + if args.install_proj or args.all: + self._install(Proj) + if args.install_hdf5 or args.all: + self._install(HDF5) + if args.install_gmg or args.all: + self._install(GeoModelGrids) + if (args.update_linking or args.all) and platform.system() == "Darwin": + Darwin.update_linking(self.install_dir) + if args.create_setup or args.all: + self._create_setup() + if args.create_tarball or args.all: + self._create_tarball() + + def _parse_command_line(self): + """Parse command line arguments. + """ + DESCRIPTION = ( + "Application for creating GeoModelGrids binary packages." + ) + + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument("--install-dir", action="store", dest="install_dir", help="Install directory.") + + parser.add_argument("--gcc", action="store_true", dest="install_gcc", help="Install gcc.") + parser.add_argument("--sqlite", action="store_true", dest="install_sqlite", help="Install sqlite.") + parser.add_argument("--tiff", action="store_true", dest="install_tiff", help="Install libtiff.") + parser.add_argument("--proj", action="store_true", dest="install_proj", help="Install proj.") + parser.add_argument("--hdf5", action="store_true", dest="install_hdf5", help="Install HDF5.") + parser.add_argument("--geomodelgrids", action="store_true", dest="install_gmg", help="Install GeoModelGrids.") + parser.add_argument("--create-setup", action="store_true", dest="create_setup", help="Create setup.sh.") + parser.add_argument("--update-linking", action="store_true", dest="update_linking", help="Update Darwin linking.") + parser.add_argument("--create-tarball", action="store_true", dest="create_tarball", help="Create tarball.") + + parser.add_argument("--all", action="store_true", dest="all", help="Run all steps.") + parser.add_argument("--quiet", action="store_false", dest="show_progress", default=True) + args = parser.parse_args() + + return args + + def _install(self, cls): + dep = cls(self.install_dir, self.env, show_progress=self.show_progress) + dep.install() + + def _set_env(self): + self.env = { + "PATH": f"{self.install_dir}/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "CPPFLAGS": f"-I{self.install_dir}/include -DNDEBUG", + "LDFLAGS": f"-L{self.install_dir}/lib", + "CFLAGS": "-O3", + "CXXFLAGS": "-O3", + } + if platform.system() == "Darwin": + self.env.update({ + "CC": "clang", + "CXX": "clang++", + }) + elif platform.system() == "Linux": + self.env.update({ + "CC": "gcc", + "CXX": "g++", + }) + else: + raise ValueError(f"Unknown platform '{system.platform()}'.") + + def _create_setup(self): + filename = os.path.join(self.install_dir, "setup.sh") + with open(filename, "w") as fout: + lines = [ + 'gmg=`pwd`', + '', + 'if test ! -f bin/geomodelgrids_query; then', + ' echo "ERROR: Source this script from the top-level geomodelgrids directory."', + ' echo " cd [directory constaining setup.sh]"', + ' echo " source setup.sh"', + '', + 'else', + ' export PATH="$gmg:bin:/bin:/usr/bin:/sbin:/usr/sbin:$PATH"', + ' echo "Ready to run GeoModelGrids!"', + 'fi', + ] + fout.write("\n".join(lines)) + + def _create_tarball(self): + if platform.system() == "Darwin": + arch = "Darwin-{}".format(platform.mac_ver()[0]) + elif platform.system() == "Linux": + arch = "Linux-{}".format(platform.machine()) + + version = pkg_resources.get_distribution("geomodelgrids").version + tarball = f"geomodelgrids-{arch}-{version}.tar.gz" + + shutil.copytree(f"geomodelgrids-{version}", os.path.join(self.install_dir, "src")) + with tarfile.open(tarball, mode="w:gz") as tfile: + tfile.add(self.install_dir, arcname=f"geomodelgrids-{arch}-{version}") + + +# -------------------------------------------------------------------------------------------------- +if __name__ == "__main__": + App().main() + + +# End of file diff --git a/docker/geomodelgrids-userenv b/docker/geomodelgrids-userenv index 33fdc9f..4cfc123 100644 --- a/docker/geomodelgrids-userenv +++ b/docker/geomodelgrids-userenv @@ -22,6 +22,7 @@ RUN apt-get update \ git \ ca-certificates \ libcppunit-dev \ + hdf5-tools \ libhdf5-dev \ sqlite3 \ libsqlite3-0 \ @@ -31,7 +32,9 @@ RUN apt-get update \ libproj-dev \ vim-common \ vim-runtime \ - vim-nox + vim-nox \ + less + COPY docker/certs/ /usr/local/share/ca-certificates RUN update-ca-certificates diff --git a/docs/Makefile.am b/docs/Makefile.am index ac6840b..7713fe8 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -4,52 +4,60 @@ EXTRA_DIST = \ README.md \ _config.yml \ _data/navbar.yml \ - _includes/footer.html \ - _includes/head.html \ - _includes/nav.html \ - _includes/page-banner.html \ - _layouts/default.html \ - _sass/ - _site/ index.md \ + apps/borehole.md \ + apps/create.md \ apps/info.md \ + apps/isosurface.md \ apps/query.md \ - assets/ + apps/queryelev.md \ + c-api/example-query.md \ c-api/serial.md \ + c-api/serial-query.md \ + c-api/utils-errorhandler.md \ c-api/parallel.md \ + cxx-api/example-query.md \ cxx-api/serial.md \ cxx-api/serial-block.md \ cxx-api/serial-hdf5.md \ + cxx-api/serial-hyperslab.md \ cxx-api/serial-model.md \ cxx-api/serial-modelinfo.md \ cxx-api/serial-query.md \ - cxx-api/serial-topography.md \ + cxx-api/serial-surface.md \ cxx-api/parallel.md \ + cxx-api/utils.md \ + cxx-api/utils-crstransformer.md \ + cxx-api/utils-errorhandler.md \ + cxx-api/utils-testdriver.md \ developer/code-layout.md \ - figs/hdf5layout.tex \ + developer/docker-devenv.md \ figs/palettes/palette_general.tex \ figs/palettes/palette_mmi.tex \ figs/palettes/palette_usgs.tex \ getting-started/install.md \ getting-started/running.md \ images/USGSsimple.png \ - images/cc0-screen.png \ models/seismic_graymer_delta.md \ overview/features.md \ overview/storage-layout.md \ + overview/figs/coordsys.png \ + overview/figs/coordsys.tex \ overview/figs/gridmapping.png \ overview/figs/hdf5layout.png \ overview/figs/hdf5layout.tex overview/plot_gridmapping.py* scripting/scripting.md \ scripting/data-structures.md \ - scripting/data-structures/block.md \ - scripting/data-structures/earthvision-api.md \ - scripting/data-structures/earthvision-model.md \ - scripting/data-structures/io-hdf5.md \ - scripting/data-structures/model.md \ - scripting/data-structures/rasterize.md \ - scripting/data-structures/topography.md + scripting/create.md \ + scripting/create/apps-create-model.md \ + scripting/create/core-block.md \ + scripting/create/core-datasrc.md \ + scripting/create/core-model.md \ + scripting/create/core-surface.md \ + scripting/create/earthvision-api.md \ + scripting/create/earthvision-datasrc.md \ + scripting/create/io-hdf5.md # End of file diff --git a/libsrc/geomodelgrids/serial/Makefile.am b/libsrc/geomodelgrids/serial/Makefile.am index d4f4611..f2fb5d7 100644 --- a/libsrc/geomodelgrids/serial/Makefile.am +++ b/libsrc/geomodelgrids/serial/Makefile.am @@ -2,6 +2,9 @@ subpackage = serial include $(top_srcdir)/subpackage.am subpkginclude_HEADERS = \ + Block.hh \ + Surface.hh \ + Hyperslab.hh \ ModelInfo.hh \ Model.hh \ Query.hh \ diff --git a/setup.cfg b/setup.cfg index 4e6533c..da196e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ tag_date = 0 [metadata] name = geomodelgrids -version = 0.1 +version = 0.2.0 author = Brad Aagaard author_email = baagaard@usgs.gov description = USGS Georeferenced Gridded Models diff --git a/tests/pytests/Makefile.am b/tests/pytests/Makefile.am index 1840094..cee065f 100644 --- a/tests/pytests/Makefile.am +++ b/tests/pytests/Makefile.am @@ -7,7 +7,7 @@ dist_check_SCRIPTS = test_geomodelgrids.py dist_noinst_PYTHON = \ test_driver.py \ test_units.py \ - test_batch py \ + test_batch.py \ test_createapp.py From f3983fdac8bf83e958849b68247971bc91a25eeb Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 20:04:57 -0600 Subject: [PATCH 02/13] Update dockerignore (don't ignore script to build binary). --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 3a1b6e4..d4ecce6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,8 @@ autom4te.cache aux-config docker !docker/certs +!docker/build_binary.py +!docker/pip.conf build models/plotting/data models/plotting/plots From 426e4869d12954aa3f8679b06ed42c47d811dd62 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 20:05:23 -0600 Subject: [PATCH 03/13] Fix errors for gcc build. --- docker/build_binary.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/build_binary.py b/docker/build_binary.py index 96eef05..29fae5f 100755 --- a/docker/build_binary.py +++ b/docker/build_binary.py @@ -97,7 +97,7 @@ class GCC(Package): BUILD_DIR = "gcc-build" SRC_DIR = f"gcc-{VERSION}" - EXTRAS = ( + EXTRAS = { "gmp": { "version": "6.2.1", "tarball": "gmp-{version}.tar.bz2", @@ -105,7 +105,7 @@ class GCC(Package): }, "mpc": { "version": "1.2.1", - "tarball": "mpi-{version}.tar.gz", + "tarball": "mpc-{version}.tar.gz", "url": "https://mirrors.kernel.org/gnu/mpc/{tarball}", }, "mpfr": { @@ -113,7 +113,7 @@ class GCC(Package): "tarball": "mpfr-{version}.tar.gz", "url": "https://mirrors.kernel.org/gnu/mpfr/{tarball}", } - ) + } def configure(self): @@ -129,7 +129,7 @@ def install(self): self.download() self.extract_tarball() - gcc_src = f"gcc-{VERSION}" + gcc_src = f"gcc-{self.VERSION}" for label,pkg in self.EXTRAS.items(): tarball = pkg["tarball"].format(version=pkg["version"]) url = pkg["url"].format(version=pkg["version"], tarball=tarball) @@ -137,8 +137,8 @@ def install(self): self.download(url=url, tarball=tarball) self.extract_tarball(path=gcc_src, tarball=tarball) - link_src = os.path.join(gcc_sc, f"{label}-{pkg['version']}") - link_dest = os.path.join(gcc_sc, label) + link_src = f"{label}-{pkg['version']}" + link_dest = os.path.join(gcc_src, label) os.symlink(link_src, link_dest) self.configure() @@ -318,7 +318,7 @@ def _parse_command_line(self): ) parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument("--install-dir", action="store", dest="install_dir", help="Install directory.") + parser.add_argument("--install-dir", action="store", dest="install_dir", required=True, help="Install directory.") parser.add_argument("--gcc", action="store_true", dest="install_gcc", help="Install gcc.") parser.add_argument("--sqlite", action="store_true", dest="install_sqlite", help="Install sqlite.") From de681e6947669c1a6c10e4e7b97f253abaf0691c Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 22:49:21 -0600 Subject: [PATCH 04/13] Add OpenSSL and curl to binary package. --- docker/build_binary.py | 166 +++++++++++++++++++++++++++-------------- 1 file changed, 110 insertions(+), 56 deletions(-) diff --git a/docker/build_binary.py b/docker/build_binary.py index 29fae5f..c172b33 100755 --- a/docker/build_binary.py +++ b/docker/build_binary.py @@ -21,22 +21,26 @@ import shutil import subprocess import tarfile -import pkg_resources # -------------------------------------------------------------------------------------------------- class Package(object): + VERSION = None + TARBALL = None + URL = None + SRC_DIR = None + BUILD_DIR = None def __init__(self, install_dir, env, show_progress=True): self.install_dir = install_dir self.env = env self.nthreads = multiprocessing.cpu_count() self.show_progress = show_progress - + def install(self): self.download() self.extract_tarball() - self.configure() + self.configure(args=None) self.build() def download(self, url=None, tarball=None): @@ -74,7 +78,7 @@ def configure(self, args): def build(self): cwd = os.path.abspath(os.curdir) print(f"CWD={cwd}") - os.chdir(self.BUILD_DIR) + os.chdir(self.BUILD_DIR) self._run_cmd(["make", f"-j{self.nthreads}"]) self._run_cmd(["make", "install"]) os.chdir(cwd) @@ -82,13 +86,13 @@ def build(self): def _run_cmd(self, cmd): self._status("Running {} ...".format(" ".join(cmd))) - result = subprocess.run(cmd, env=self.env, check=True) + subprocess.run(cmd, env=self.env, check=True) def _status(self, msg): if self.show_progress: print(msg) - - + + # -------------------------------------------------------------------------------------------------- class GCC(Package): VERSION = "10.3.0" @@ -102,35 +106,33 @@ class GCC(Package): "version": "6.2.1", "tarball": "gmp-{version}.tar.bz2", "url": "https://mirrors.kernel.org/gnu/gmp/{tarball}", - }, - "mpc": { - "version": "1.2.1", - "tarball": "mpc-{version}.tar.gz", - "url": "https://mirrors.kernel.org/gnu/mpc/{tarball}", - }, - "mpfr": { - "version": "4.1.0", - "tarball": "mpfr-{version}.tar.gz", - "url": "https://mirrors.kernel.org/gnu/mpfr/{tarball}", - } - } - + }, + "mpc": { + "version": "1.2.1", + "tarball": "mpc-{version}.tar.gz", + "url": "https://mirrors.kernel.org/gnu/mpc/{tarball}", + }, + "mpfr": { + "version": "4.1.0", + "tarball": "mpfr-{version}.tar.gz", + "url": "https://mirrors.kernel.org/gnu/mpfr/{tarball}", + } + } def configure(self): ARGS = [ f"--prefix={self.install_dir}", "--enable-languages=c,c++", "--disable-multilib", - ] + ] super().configure(ARGS) - def install(self): self.download() self.extract_tarball() gcc_src = f"gcc-{self.VERSION}" - for label,pkg in self.EXTRAS.items(): + for label, pkg in self.EXTRAS.items(): tarball = pkg["tarball"].format(version=pkg["version"]) url = pkg["url"].format(version=pkg["version"], tarball=tarball) @@ -143,21 +145,22 @@ def install(self): self.configure() self.build() - + # -------------------------------------------------------------------------------------------------- + + class Sqlite(Package): VERSION = "3350500" TARBALL = f"sqlite-autoconf-{VERSION}.tar.gz" URL = f"https://www.sqlite.org/2021/{TARBALL}" BUILD_DIR = "sqlite-build" SRC_DIR = f"sqlite-autoconf-{VERSION}" - def configure(self): ARGS = [ f"--prefix={self.install_dir}", "--enable-shared", - ] + ] super().configure(ARGS) @@ -168,13 +171,49 @@ class Tiff(Package): URL = f"http://download.osgeo.org/libtiff/{TARBALL}" BUILD_DIR = "tiff-build" SRC_DIR = f"tiff-{VERSION}" - def configure(self): ARGS = [ f"--prefix={self.install_dir}", "--enable-shared", - ] + ] + super().configure(ARGS) + + +# -------------------------------------------------------------------------------------------------- +class OpenSSL(Package): + VERSION = "1.1.1k" + TARBALL = f"openssl-{VERSION}.tar.gz" + URL = f"https://www.openssl.org/source/{TARBALL}" + SRC_DIR = f"openssl-{VERSION}" + BUILD_DIR = SRC_DIR + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + f"--openssldir={self.install_dir}/share/openssl", + "shared", + ] + cwd = os.path.abspath(os.curdir) + os.chdir(self.SRC_DIR) + configure = os.path.join(".", "config") + self._run_cmd([configure] + ARGS) + os.chdir(cwd) + + +# -------------------------------------------------------------------------------------------------- +class Curl(Package): + VERSION = "7.76.1" + TARBALL = f"curl-{VERSION}.tar.gz" + URL = f"https://curl.se/download/{TARBALL}" + BUILD_DIR = "curl-build" + SRC_DIR = f"curl-{VERSION}" + + def configure(self): + ARGS = [ + f"--prefix={self.install_dir}", + "--enable-shared", + ] super().configure(ARGS) @@ -186,7 +225,6 @@ class Proj(Package): BUILD_DIR = "proj-build" SRC_DIR = f"proj-{VERSION}" - def configure(self): ARGS = [ f"--prefix={self.install_dir}", @@ -195,16 +233,15 @@ def configure(self): f"SQLITE3_LIBS=-L{self.install_dir}/lib -lsqlite3", f"TIFF_CFLAGS=-I{self.install_dir}/include", f"TIFF_LIBS=-L{self.install_dir}/lib -ltiff", - ] + ] super().configure(ARGS) - def install(self): super().install() - cmd = ["projsync", "--system-directory", "--bbox -128.0,34.0,-118.0,42.0"] + cmd = ["projsync", "--system-directory", "--bbox", "-128.0,34.0,-118.0,42.0"] self._run_cmd(cmd) - + # -------------------------------------------------------------------------------------------------- class HDF5(Package): @@ -213,30 +250,28 @@ class HDF5(Package): URL = f"https://www.sqlite.org/2021/{TARBALL}" BUILD_DIR = "hdf5-build" SRC_DIR = f"hdf5-{VERSION}" - def configure(self): ARGS = [ f"--prefix={self.install_dir}", "--enable-shared", "--disable-static", - ] + ] super().configure(ARGS) # -------------------------------------------------------------------------------------------------- class GeoModelGrids(Package): BUILD_DIR = "geomodelgrids-build" - + def install(self): - self.VERSION = pkg_resources.get_distribution("geomodelgrids").version self.SRC_DIR = f"geomodelgrids-{self.VERSION}" self.TARBALL = f"geomodelgrids-{self.VERSION}.tar.gz" self.extract_tarball() self.configure() self.build() - + def configure(self): ARGS = [ f"--prefix={self.install_dir}", @@ -244,7 +279,7 @@ def configure(self): "--disable-python", "--disable-gdal", "--disable-testing", - ] + ] super().configure(ARGS) @@ -284,17 +319,22 @@ class App(object): def __init__(self): self.show_progress = True - + def main(self, **kwargs): args = argparse.Namespace(**kwargs) if kwargs else self._parse_command_line() self.show_progress = args.show_progress self.install_dir = os.path.abspath(args.install_dir) + GeoModelGrids.VERSION = args.version self._set_env() if (args.install_gcc or args.all) and platform.system() == "Linux": self._install(GCC) if args.install_sqlite or args.all: self._install(Sqlite) + if args.install_openssl or args.all: + self._install(OpenSSL) + if args.install_curl or args.all: + self._install(Curl) if args.install_tiff or args.all: self._install(Tiff) if args.install_proj or args.all: @@ -309,7 +349,7 @@ def main(self, **kwargs): self._create_setup() if args.create_tarball or args.all: self._create_tarball() - + def _parse_command_line(self): """Parse command line arguments. """ @@ -318,18 +358,24 @@ def _parse_command_line(self): ) parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument("--install-dir", action="store", dest="install_dir", required=True, help="Install directory.") + parser.add_argument("--install-dir", action="store", dest="install_dir", + required=True, help="Install directory.") + parser.add_argument("--version", action="store", dest="version", + required=True, help="Set version tag.") parser.add_argument("--gcc", action="store_true", dest="install_gcc", help="Install gcc.") parser.add_argument("--sqlite", action="store_true", dest="install_sqlite", help="Install sqlite.") parser.add_argument("--tiff", action="store_true", dest="install_tiff", help="Install libtiff.") + parser.add_argument("--openssl", action="store_true", dest="install_openssl", help="Install openssl.") + parser.add_argument("--curl", action="store_true", dest="install_curl", help="Install libcurl.") parser.add_argument("--proj", action="store_true", dest="install_proj", help="Install proj.") parser.add_argument("--hdf5", action="store_true", dest="install_hdf5", help="Install HDF5.") parser.add_argument("--geomodelgrids", action="store_true", dest="install_gmg", help="Install GeoModelGrids.") parser.add_argument("--create-setup", action="store_true", dest="create_setup", help="Create setup.sh.") - parser.add_argument("--update-linking", action="store_true", dest="update_linking", help="Update Darwin linking.") + parser.add_argument("--update-linking", action="store_true", + dest="update_linking", help="Update Darwin linking.") parser.add_argument("--create-tarball", action="store_true", dest="create_tarball", help="Create tarball.") - + parser.add_argument("--all", action="store_true", dest="all", help="Run all steps.") parser.add_argument("--quiet", action="store_false", dest="show_progress", default=True) args = parser.parse_args() @@ -347,19 +393,21 @@ def _set_env(self): "LDFLAGS": f"-L{self.install_dir}/lib", "CFLAGS": "-O3", "CXXFLAGS": "-O3", - } + } if platform.system() == "Darwin": self.env.update({ "CC": "clang", "CXX": "clang++", - }) + }) elif platform.system() == "Linux": self.env.update({ "CC": "gcc", "CXX": "g++", - }) + "LDFLAGS": f"-L{self.install_dir}/lib -L{self.install_dir}/lib64", + "LD_LIBRARY_PATH": f"{self.install_dir}/lib:{self.install_dir}/lib64", + }) else: - raise ValueError(f"Unknown platform '{system.platform()}'.") + raise ValueError(f"Unknown platform '{platform.system()}'.") def _create_setup(self): filename = os.path.join(self.install_dir, "setup.sh") @@ -373,26 +421,32 @@ def _create_setup(self): ' echo " source setup.sh"', '', 'else', - ' export PATH="$gmg:bin:/bin:/usr/bin:/sbin:/usr/sbin:$PATH"', + ' export PATH="$gmg/bin:/bin:/usr/bin:/sbin:/usr/sbin:$PATH"', + ] + if platform.system() == "Linux": + lines += [ + ' export LD_LIBRARY_PATH="$gmg/lib:$gmg/lib64:${LD_LIBRARY_PATH}"', + ] + lines += [ ' echo "Ready to run GeoModelGrids!"', 'fi', - ] + ] fout.write("\n".join(lines)) def _create_tarball(self): + version = GeoModelGrids.VERSION + if platform.system() == "Darwin": arch = "Darwin-{}".format(platform.mac_ver()[0]) elif platform.system() == "Linux": - arch = "Linux-{}".format(platform.machine()) - - version = pkg_resources.get_distribution("geomodelgrids").version - tarball = f"geomodelgrids-{arch}-{version}.tar.gz" + arch = "Linux_{}".format(platform.machine()) + tarball = f"geomodelgrids-{version}-{arch}.tar.gz" shutil.copytree(f"geomodelgrids-{version}", os.path.join(self.install_dir, "src")) with tarfile.open(tarball, mode="w:gz") as tfile: tfile.add(self.install_dir, arcname=f"geomodelgrids-{arch}-{version}") - - + + # -------------------------------------------------------------------------------------------------- if __name__ == "__main__": App().main() From 6371018f15f9c24c5dc2b208a9b132c2c6e7f571 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 22:50:03 -0600 Subject: [PATCH 05/13] Add release notes and installation instructions. --- CHANGES.md | 8 ++- configure.ac | 18 +++---- docs/_data/navbar.yml | 2 + docs/apps/create.md | 2 + docs/getting-started/install.md | 70 ++++++++++++++++++++++----- docs/getting-started/release-notes.md | 7 +++ docs/index.md | 2 +- 7 files changed, 87 insertions(+), 22 deletions(-) create mode 100644 docs/getting-started/release-notes.md diff --git a/CHANGES.md b/CHANGES.md index afa8674..174cfd0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1 +1,7 @@ -# Version X.X +# Version 1.0.0 + +## Known issues + +* The interpolation algorithm does not respect fault block and zone boundaries and the boundary between water and solid materials. This means fault block and zone ids will be interpolated across block and zone boundaries, and Vp and density will be interpolated Vp and density across the boundary between water and solid materials. Vs is not interpolated across the boundary between water and solid material, because it is not defined in water. +* GeoModelGrids does not support unit conversions. All values returned in queries are in the units of the underlying models. We do check that all models have consistent units for values contained in multiple models. Queries for the elevations of the top surface or topography/bathymetry will be returned in the units of the input Coordinate Reference System. +* The model storage has been optimized for faster successive queries in the vertical direction. That is, querying for points on a vertical slice will generally be faster than querying the same number of points on a horizontal slice. diff --git a/configure.ac b/configure.ac index 6e908e9..b24091b 100644 --- a/configure.ac +++ b/configure.ac @@ -41,7 +41,7 @@ AC_ARG_ENABLE([test-coverage], [enable_test_coverage=no]) AC_ARG_WITH([python-coverage], [AC_HELP_STRING([--with-python-coverage], - [set executable for python-coverage @<:@default=python3-coverage@:>@])], + [Specify executable for python-coverage @<:@default=python3-coverage@:>@])], [python_coverage=$withval], [python_coverage="python3-coverage"]) AC_SUBST(python_coverage) @@ -66,13 +66,13 @@ AC_ARG_ENABLE([spatialdata], # CPPUNIT AC_ARG_WITH([cppunit-incdir], [AC_HELP_STRING([--with-cppunit-incdir], - [location of cppunit header files @<:@default=no@:>@])], + [Specify location of cppunit header files @<:@default=no@:>@])], [with_cppunit_incdir=$withval], [with_cppunit_incdir=no]) AC_SUBST(with_cppunit_incdir) AC_ARG_WITH([cppunit-libdir], [AC_HELP_STRING([--with-cppunit-libdir], - [location of cppunit library @<:@default=no@:>@])], + [Specify location of cppunit library @<:@default=no@:>@])], [with_cppunit_libdir=$withval], [with_cppunit_libdir=no]) AC_SUBST(with_cppunit_libdir) @@ -80,13 +80,13 @@ AC_SUBST(with_cppunit_libdir) # PROJ AC_ARG_WITH([proj-incdir], [AC_HELP_STRING([--with-proj-incdir], - [location of proj header files @<:@default=no@:>@])], + [Specify location of proj header files @<:@default=no@:>@])], [with_proj_incdir=$withval], [with_proj_incdir=no]) AC_SUBST(with_proj_incdir) AC_ARG_WITH([proj-libdir], [AC_HELP_STRING([--with-proj-libdir], - [location of proj library @<:@default=no@:>@])], + [Specify location of proj library @<:@default=no@:>@])], [with_proj_libdir=$withval], [with_proj_libdir=no]) AC_SUBST(with_proj_libdir) @@ -94,13 +94,13 @@ AC_SUBST(with_proj_libdir) # HDF5 AC_ARG_WITH([hdf5-incdir], [AC_HELP_STRING([--with-hdf5-incdir], - [location of hdf5 header files @<:@default=no@:>@])], + [Specify location of hdf5 header files @<:@default=no@:>@])], [with_hdf5_incdir=$withval], [with_hdf5_incdir=no]) AC_SUBST(with_hdf5_incdir) AC_ARG_WITH([hdf5-libdir], [AC_HELP_STRING([--with-hdf5-libdir], - [location of hdf5 library @<:@default=no@:>@])], + [Specify location of hdf5 library @<:@default=no@:>@])], [with_hdf5_libdir=$withval], [with_hdf5_libdir=no]) AC_SUBST(with_hdf5_libdir) @@ -108,13 +108,13 @@ AC_SUBST(with_hdf5_libdir) # GDAL AC_ARG_WITH([gdal-incdir], [AC_HELP_STRING([--with-gdal-incdir], - [location of gdal header files @<:@default=no@:>@])], + [Specify location of gdal header files @<:@default=no@:>@])], [with_gdal_incdir=$withval], [with_gdal_incdir=no]) AC_SUBST(with_gdal_incdir) AC_ARG_WITH([gdal-libdir], [AC_HELP_STRING([--with-gdal-libdir], - [location of gdal library @<:@default=no@:>@])], + [Specify location of gdal library @<:@default=no@:>@])], [with_gdal_libdir=$withval], [with_gdal_libdir=no]) AC_SUBST(with_gdal_libdir) diff --git a/docs/_data/navbar.yml b/docs/_data/navbar.yml index 9fdb1b8..29df55a 100644 --- a/docs/_data/navbar.yml +++ b/docs/_data/navbar.yml @@ -9,6 +9,8 @@ toc: url: /overview/storage-layout.html - title: Getting Started subitems: + - page: Release notes + url: /getting-started/release-notes.html - page: Installation url: /getting-started/install.html - page: Running the code diff --git a/docs/apps/create.md b/docs/apps/create.md index 4192b70..e449dd2 100644 --- a/docs/apps/create.md +++ b/docs/apps/create.md @@ -1,5 +1,7 @@ # geomodelgrids_create_model +Note: This application requires Python 3.7 or later and is not included in the binary package. + The `geomodelgrids_create_model` command line program is used to generate a model from a data source. Currently, only the EarthVision data source is implemented. diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index 95c185b..3987593 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -1,30 +1,78 @@ # Installation -**:TODO:**: Finish instructions after building binary. +**TODO** Add intro here. ## Installing via binary package +**Note**: The binary package contains the query library and command line programs for accessing GeoModelGrids files. It does not contain the Python code to create GeoModelGrids files. + +Binary packages are provided for Linux (GLIBC X.X and later) and MacOS (10.14 and later). +The binary packages are 200MB+ in size; the large size is a result of the inclusion of geographic coordinate datum information used by Proj.4. + ## Installing via source -### Dependencies +### Prerequisites -* C/C++ compiler +Most Linux distributions can provide all of the prerequisites via the package manager. +On MacOS systems the operating system supplies with XCode command line tools installed supply the compiler, Sqlite, libtiff, openssl, and libcurl. You can use a package manager to install Proj.4, HDF5, and CppUnit (if desired) or build them from source. -* autotools (automake, autoconf, libtool) +* C/C++ compiler supporting C++11 +* HDF5 (version 1.10.0 or later) +* Sqlite (version 3 or later; required by Proj.4) +* Proj.4 (version 6.3.0 or later). Proj 7.0.0 and later also require: + * libtiff + * openssl + * libcurl +* Python 3 (version 3.7 or later; if generating models) + * h5py + * numpy +* CppUnit (version 1.14.0 or later; if running test suite) +### Downloading the source code -* HDF5 library (version 1.10.0 or later) +**TODO** Add how to download. -* Proj.4 (version 6.3.0 or later) +We highly recommend building in a separate directory from the source tree. +In the following configuration, we will unpack the source code in `$HOME/src`, build the source code in `$HOME/build/geomodelgrids`, and install the code to `$HOME/geomodelgrids`. -* Python 3 (if generating models) +### Running configure - + h5py - + numpy +Useful configure options (run `configure --help` to see all options): -### Downloading the source code +* `--prefix=DIR` Install GeoModelGrids in directory `DIR`. +* `--enable-python` Enable building Python modules [default=no] +* `--enable-gdal` Enable GDAL support for writing GeoTiff files [default=no] +* `--enable-testing` Enable Python and C++ (requires cppunit) unit testing [default=no] +* `--with-cppunit-incdir` Specify location of cppunit header files [default=no] +* `--with-cppunit-libdir` Specify location of cppunit library [default=no] +* `--with-proj-incdir` Specify location of proj header files [default=no] +* `--with-proj-libdir` Specify location of proj library [default=no] +* `--with-hdf5-incdir` Specify location of hdf5 header files [default=no] +* `--with-hdf5-libdir` Specify location of hdf5 library [default=no] +* `--with-gdal-incdir` Specify location of gdal header files [default=no] +* `--with-gdal-libdir` Specify location of gdal library [default=no] -### Running configure +Creating GeoModelGrids files using `geomodelgrids_create_model` requires Python and configuring with `--enable-python`. Generating horizontal isosurfaces using `geomodelgrids_isosurface` requires the GDAL library and configuring with `--enable-gdal`. + +```bash +mkdir -p $HOME/build/geomodelgrids +cd $HOME/build/geomodelgrids + +# Typical configure on a Linux system with all dependencies installed by a package manager. +$HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids + +# Typical configure on a MacOS system with HDF5 installed in $HOME/hdf5 +# and Proj.4 installed in $HOME/proj. +$HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids \ +--with-hdf5-incdir=$HOME/hdf5/include --with-hdf5-libdir=$HOME/hdf5/lib \ +--with-proj-incdir=$HOME/proj/include --with-proj-libdir=$HOME/proj/lib +``` ### Building and installing +```bash +make && make install +``` + ### Running tests + +If GeoModelGrids is configured with `--enable-testing` (requires CppUnit), then the test suite can be run via `make check`. If GeoModelGrids was configured with Python enabled, then `make check` will also run the unit tests for the Python code. diff --git a/docs/getting-started/release-notes.md b/docs/getting-started/release-notes.md new file mode 100644 index 0000000..174cfd0 --- /dev/null +++ b/docs/getting-started/release-notes.md @@ -0,0 +1,7 @@ +# Version 1.0.0 + +## Known issues + +* The interpolation algorithm does not respect fault block and zone boundaries and the boundary between water and solid materials. This means fault block and zone ids will be interpolated across block and zone boundaries, and Vp and density will be interpolated Vp and density across the boundary between water and solid materials. Vs is not interpolated across the boundary between water and solid material, because it is not defined in water. +* GeoModelGrids does not support unit conversions. All values returned in queries are in the units of the underlying models. We do check that all models have consistent units for values contained in multiple models. Queries for the elevations of the top surface or topography/bathymetry will be returned in the units of the input Coordinate Reference System. +* The model storage has been optimized for faster successive queries in the vertical direction. That is, querying for points on a vertical slice will generally be faster than querying the same number of points on a horizontal slice. diff --git a/docs/index.md b/docs/index.md index b6d1070..218d268 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ The GeoModelGrids project focuses on storing and querying georeferenced raster-based models. The models are composed of blocks, -where each block is a uniform grid with a potentially different +in which each block is a uniform grid with a potentially different resolution. # Motivation From a41e5c26179a74b937253966930acc45d210bdb4 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Mon, 3 May 2021 22:50:18 -0600 Subject: [PATCH 06/13] Add Dockerfile for Linux binary. --- docker/linux-binary | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 docker/linux-binary diff --git a/docker/linux-binary b/docker/linux-binary new file mode 100644 index 0000000..e5bb10d --- /dev/null +++ b/docker/linux-binary @@ -0,0 +1,75 @@ +FROM centos:7 as os-update +LABEL maintainer="Brad Aagaard " + +RUN yum install -y \ + gcc \ + gcc-c++ \ + make \ + file \ + which \ + diffutils \ + gettext \ + git \ + uncrustify \ + gnupg2 \ + automake \ + autoconf \ + libtool \ + openssh \ + openssl \ + zlib-devel \ + libffi-devel \ + ca-certificates \ + vim-common \ + vim-runtime \ + centos-release-scl +RUN yum install -y rh-python38 +RUN scl enable rh-python38 -- python3 -m pip install requests + +COPY docker/certs/ /usr/share/pki/ca-trust-source/anchors +RUN update-ca-trust enable + +# ------------------------------------------------------------------------------ +from os-update as build-setup + +# Create 'gmg-user' user +ENV GMG_USER gmg-user +RUN useradd --create-home --shell /bin/bash $GMG_USER +ENV STORAGE_DIR=/opt/geomodelgrids +ENV INSTALL_DIR=$STORAGE_DIR/dest HOME=/home/$GMG_USER + +RUN mkdir ${STORAGE_DIR} \ + && chown ${GMG_USER} ${STORAGE_DIR} \ + && chgrp ${GMG_USER} ${STORAGE_DIR} +USER ${GMG_USER} + +RUN mkdir -p ~/.config/pip \ + && echo "[global]" > $HOME/.config/pip/pip.conf \ + && echo "trusted-host = pypi.org files.pythonhosted.org" >> $HOME/.config/pip/pip.conf \ + && scl enable rh-python38 -- python3 -m pip install requests --user + +WORKDIR $STORAGE_DIR + +# To build up to here: +# docker build -t geomodelgrids-binaryenv -f docker/linux-binary --target build-setup . +# +# To build binary manually making use of persistent storage +# docker run -ti --rm --name geomodelgrids-binary-workspace -v geomodelgrids-binary:/opt/geomodelgrids geomodelgrids-binaryenv +# docker cp TARBALL geomodelgrids-binary-workspace:/opt/geomodelgrids/build +# docker cp docker/build_binary.py geomodelgrids-binary-workspace:/opt/geomodelgrids/build +# svn enable rh-python38 -- /opt/geomodelgrids/src/docker/build_binary.py --install-dir=$INSTALL_DIR --version=X.X.X --all + +# ------------------------------------------------------------------------------ +from build-setup as build-binary + +COPY --chown=gmg-user:gmg-user docker/build_binary.py ${BUILD_DIR} + +WORKDIR $BUILD_DIR + +RUN mv ${SRC_DIR}/geomodelgrids-*.tar.gz ${BUILD_DIR} +RUN scl enable rh-python38 -- ${SRC_DIR}/docker/build_binary.py --install-dir=${INSTALL_DIR} --version=1.0.0rc1 --gcc +RUN scl enable rh-python38 -- ${SRC_DIR}/docker/build_binary.py --install-dir=${INSTALL_DIR} --version=1.0.0rc1 --tiff --sqlite --openssl --curl --proj --hdf5 +RUN scl enable rh-python38 -- ${SRC_DIR}/docker/build_binary.py --install-dir=${INSTALL_DIR} --version=1.0.0rc1 --geomodelgrids --create-setup --create-tarball + + +CMD /bin/bash From a2f8ff27eaf6799df8cc6c51b9456f24710d5c65 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Tue, 4 May 2021 10:05:12 -0600 Subject: [PATCH 07/13] Finish installation instructions. --- docker/build_binary.py | 4 +- docs/getting-started/install.md | 87 +++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/docker/build_binary.py b/docker/build_binary.py index c172b33..a6f1d88 100755 --- a/docker/build_binary.py +++ b/docker/build_binary.py @@ -437,14 +437,14 @@ def _create_tarball(self): version = GeoModelGrids.VERSION if platform.system() == "Darwin": - arch = "Darwin-{}".format(platform.mac_ver()[0]) + arch = "Darwin_{}".format(platform.machine()) elif platform.system() == "Linux": arch = "Linux_{}".format(platform.machine()) tarball = f"geomodelgrids-{version}-{arch}.tar.gz" shutil.copytree(f"geomodelgrids-{version}", os.path.join(self.install_dir, "src")) with tarfile.open(tarball, mode="w:gz") as tfile: - tfile.add(self.install_dir, arcname=f"geomodelgrids-{arch}-{version}") + tfile.add(self.install_dir, arcname=f"geomodelgrids-{version}-{arch}") # -------------------------------------------------------------------------------------------------- diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index 3987593..9de806a 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -1,20 +1,68 @@ # Installation -**TODO** Add intro here. +There are two basic means of installing the GeoModelGrids software. -## Installing via binary package +1. Use the Linux or macOS binary package. +2. Build from source. + +The binary packages provide the command line programs. If you want to use the C/C++ API in your own code, then we recommend building from source to insure compatibility of the compiled code. +For anyone wanting to contribute to GeoModelGrids software development, we offer a Docker development environment. +See [Developer Environment]({{"/developer/docker-devenv.html" | relative_url}}) for more information. + +## Installing the binary package **Note**: The binary package contains the query library and command line programs for accessing GeoModelGrids files. It does not contain the Python code to create GeoModelGrids files. -Binary packages are provided for Linux (GLIBC X.X and later) and MacOS (10.14 and later). +Binary packages are provided for Linux (GLIBC 2.17 and later; use `ldd --version` to check your version of GLIBC) and macOS (10.14 and later). +The macOS binary package has not been tested on machines with Apple silicon processors. +Windows users can use the Linux binary package within the Windows Subsystem for Linux. The binary packages are 200MB+ in size; the large size is a result of the inclusion of geographic coordinate datum information used by Proj.4. -## Installing via source +Open a terminal window and \item Open a terminal window and change to the directory where you want to place the binary distribution. + +```bash +cd $HOME +mkdir geomodelgrids +cd geomodelgrids +``` + +Download the Linux or macOS tarball from GeoModelGrids GitHub [releases](https://github.com/baagaard-usgs/geomodelgrids/releases) and save it to the desired location, e.g., `$HOME/geomodelgrids`. + +Unpack the tarball. + +```bash +# Linux 64-bit +$ tar -xf geomodelgrids-1.0.0-Linux_x86_64.tar.gz + +# macOS +$ tar -xf geomodelgrids-1.0.0-Darwin_x86_64.tar.gz +``` + +Set environment variables. The provided `setup.sh` script in the top-level directory script only works if you are using bash shell. If you are using adifferent shell, you will need to alter how the environment variables are set in `setup.sh`. + +```bash +# Linux 64-bit +cd geomodelgrids-1.0.0-Linux_x86_64 + +# macOS +cd geomodelgrids-1.0.0-Darwin_x86_64 + +source setup.sh +``` + +**IMPORTANT**: You will need to either source the `setup.sh` script each time you open a new bash shell (terminal) window and want to use GeoModelGrids or add the environment variables to your shell setup script (for example, `.bashrc`). + +**WARNING**: The binary distribution contains the GeoModelGrids software and all of its dependencies (Proj4, HDF5, OpenSSL, sqlite, curl, and tiff). +If you have any of this software already installed on your system, you need to be careful in setting up your environment so that preexisting software does not conflict with the GeoModelGrids binary. +By default the `setup.sh` script will prepend to the `PATH` (for Linux and macOS) and `LD_LIBRARY_PATH` (for Linux) environment variables. +This will prevent most conflicts. + +## Installing from source ### Prerequisites Most Linux distributions can provide all of the prerequisites via the package manager. -On MacOS systems the operating system supplies with XCode command line tools installed supply the compiler, Sqlite, libtiff, openssl, and libcurl. You can use a package manager to install Proj.4, HDF5, and CppUnit (if desired) or build them from source. +On macOS systems the operating system supplies with XCode command line tools installed supply the compiler, Sqlite, libtiff, openssl, and libcurl. You can use a package manager to install Proj.4, HDF5, and CppUnit (if desired) or build them from source. You can also use the `build_binary.py` Python script in the `docker` directory of the GeoModelGrids source code to install the software and any prerequisites that you do not have. * C/C++ compiler supporting C++11 * HDF5 (version 1.10.0 or later) @@ -27,13 +75,21 @@ On MacOS systems the operating system supplies with XCode command line tools ins * h5py * numpy * CppUnit (version 1.14.0 or later; if running test suite) + ### Downloading the source code -**TODO** Add how to download. - We highly recommend building in a separate directory from the source tree. In the following configuration, we will unpack the source code in `$HOME/src`, build the source code in `$HOME/build/geomodelgrids`, and install the code to `$HOME/geomodelgrids`. + Download the source code for the latest release from GeoModelGrids GitHub [release]([https://github.com/baagaard-usgs/geomodelgrids/releases) and unpack the tarball. + + ```bash + mkdir $HOME/src + mv geomodelgrids-1.0.0.tar.gz $HOME/src + cd $HOME/src + tar -xf geomodelgrids-1.0.0.tar.gz + ``` + ### Running configure Useful configure options (run `configure --help` to see all options): @@ -58,9 +114,11 @@ mkdir -p $HOME/build/geomodelgrids cd $HOME/build/geomodelgrids # Typical configure on a Linux system with all dependencies installed by a package manager. -$HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids +# We usually have to specify the location of HDF5 because multiple variations can be present. +$HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids \ +--with-hdf5-incdir=/usr/include/hdf5/serial --with-hdf5-libdir=/usr/lib/x86_64-linux-gnu/hdf5/serial -# Typical configure on a MacOS system with HDF5 installed in $HOME/hdf5 +# Typical configure on a macOS system with HDF5 installed in $HOME/hdf5 # and Proj.4 installed in $HOME/proj. $HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids \ --with-hdf5-incdir=$HOME/hdf5/include --with-hdf5-libdir=$HOME/hdf5/lib \ @@ -73,6 +131,15 @@ $HOME/src/geomodelgrids-1.0.0/configure --prefix=$HOME/geomodelgrids \ make && make install ``` -### Running tests +### Running tests (optional) If GeoModelGrids is configured with `--enable-testing` (requires CppUnit), then the test suite can be run via `make check`. If GeoModelGrids was configured with Python enabled, then `make check` will also run the unit tests for the Python code. + +### Setting environment variables + +Set environment variables for use of the bash shell: + +```bash +PATH=$HOME/geomodelgrids/bin:$PATH +export LD_LIBRARY_PATH=$HOME/geomodelgrids/lib:$LD_LIBRARY_PATH +``` From 051ebc81774b451d038b332d970be1c6e6516b32 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Tue, 4 May 2021 10:05:38 -0600 Subject: [PATCH 08/13] Finish developer environment documentation. --- docs/developer/docker-devenv.md | 64 +++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/developer/docker-devenv.md b/docs/developer/docker-devenv.md index 94e745b..91e53dc 100644 --- a/docs/developer/docker-devenv.md +++ b/docs/developer/docker-devenv.md @@ -1,6 +1,14 @@ # GeoModelGrids Docker Development Environment -The `geomodelgrids-devenv` Docker image (https://registry.gitlab.com/baagaard-usgs/geomodelgrids/geomodelgrids-devenv) provides all of the dependencies and defines the environment for GeoModelGrids development. It is intended to be read only with a separate Docker volume for persistent storage and the GeoModelGrids development workspace. +The `geomodelgrids-devenv` Docker image provides all of the dependencies and defines the environment for GeoModelGrids development. +It is built using the Ubuntu 20.04 Linux distribution. +It is intended to be read only with a separate Docker volume for persistent storage and the GeoModelGrids development workspace. +We separate the development "environment" from the "workspace" so that we can update the development environment without affecting the workspace and easily maintain a persistent workspace while starting and stopping the Docker container that holds the development environment. + +## Prerequisites + +1. You need to have [Docker](https://www.docker.com/products/docker-desktop) installed and running on your computer. +2. You need to have a [GitHub](https://github.com) account. ## Setup @@ -11,23 +19,31 @@ You only need to run these setup steps once. This creates a copy of the repository in your GitHub account. 1. Log in to your [GitHub](https://github.com) account, creating an account if you do not already have one. -2. Fork the GeoModelGrids repository: https://github.com/baagaard-usgs/geomodelgrids +2. Fork the GeoModelGrids repository: . ### Create Docker volume for persistent storage +On your local machine, create a Docker volume for persistent storage. + ``` docker volume create geomodelgrids-dev ``` ### Start GeoModelGrids development Docker container +Running the command below will: + +1. Start (run) the Docker container using the `geomodelgrids-devenv` Docker image and assign it the name `geomodelgrids-dev-workspace`. +2. Mount the docker volume with persistent storage at `/opt/geomodelgrids`. +3. The `geomodelgrids-devenv` Docker image will be downloaded from the GitLab registry . + ``` # Without EarthVision -docker run --name geomodelgrids-dev-workspace --rm -t \ +docker run --name geomodelgrids-dev-workspace --rm -it \ -v geomodelgrids-dev:/opt/geomodelgrids \ registry.gitlab.com/baagaard-usgs/geomodelgrids/geomodelgrids-devenv # -# With EarthVision +# With EarthVision in $TOOLS_DIR/earthvision-11 docker run --name geomodelgrids-dev-workspace --rm -it \ -v geomodelgrids-dev:/opt/geomodelgrids \ -v $TOOLS_DIR/earthvision-11:/opt/earthvision \ @@ -59,7 +75,8 @@ We will use the directory following directory structure for the persistent stora └── share ``` -This directory structure is setup for both a debugging version for development (debug directory) and an optimized version for performance testing (opt directory). For now, we will only setup the debugging version. +This directory structure is setup for both a debugging version for development (debug directory) and an optimized version for performance testing (opt directory). +For now, we will only setup the debugging version. ``` cd /opt/geomodelgrids @@ -81,11 +98,16 @@ git clone --recursive https://github.com/GITHUB_USERNAME/geomodelgrids.git src ``` cd ${TOP_BUILDDIR} pushd ${TOP_SRCDIR} && autoreconf -if && popd -${TOP_SRCDIR}/configure --prefix=${INSTALL_DIR} --enable-python --enable-testing \ - CPPFLAGS="-${HDF5_INCDIR}" LDFLAGS="-L${HDF5_LIBDIR}" \ - CC=gcc CXX=g++ CFLAGS="-g -Wall" CXXFLAGS="-std=c++11 -g -Wall" -make install -j${nproc} -make check -j${nproc} +${TOP_SRCDIR}/configure \ + --prefix=${INSTALL_DIR} \ + --enable-python \ + --enable-gdal \ + --enable-testing \ + CPPFLAGS="-${HDF5_INCDIR}" \ + LDFLAGS="-L${HDF5_LIBDIR}" \ + CC=gcc CXX=g++ CFLAGS="-g -Wall" CXXFLAGS="-g -Wall" +make install -j$(nproc) +make check -j$(nproc) ``` @@ -105,3 +127,25 @@ Additionally, we recommend also installing the following extensions: * GitHub Pull Requests and Issues * GitLens -- Git supercharged * Material Icon Theme + +## Running + +### Start the GeoModelGrids development Docker container + +Whenever you need to restart the `geomodelgrids-dev-workspace` Docker container, simply run + +```bash +# Without EarthVision +docker run --name geomodelgrids-dev-workspace --rm -it \ + -v geomodelgrids-dev:/opt/geomodelgrids \ + registry.gitlab.com/baagaard-usgs/geomodelgrids/geomodelgrids-devenv +``` + +**Tip**: Make sure Docker is running before you start the container. + +### Attach VS Code to the Docker container + +1. Start VS Code. +2. Click on the Docker extension in the Activity Bar on the far left hand side. +3. Find the `geomodelgrids-dev-workspace` container. Verify that it is running. +4. Right-click on the container and select `Attach Visual Studio Code`. This will open a new window. You should see `Container registry.gitlab.com/baagaard-usgs...` at the left side of the status bar at the bottom of the window. From e82f58d5ab5a2d97787dc30b98ea30e3aec7cdd1 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Tue, 4 May 2021 10:05:58 -0600 Subject: [PATCH 09/13] Minor documentation edits. --- docs/getting-started/running.md | 6 ++++-- docs/index.md | 6 +++--- docs/scripting/data-structures.md | 2 +- docs/scripting/scripting.md | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/getting-started/running.md b/docs/getting-started/running.md index 4054864..a5a3f85 100644 --- a/docs/getting-started/running.md +++ b/docs/getting-started/running.md @@ -5,6 +5,8 @@ There are two main ways to run the code: * Use the `geomodelgrids` suite of command line programs, or * Call the API from your own code. -The `geomodelgrids` suite of command line programs targets end users who simply want to query for values in a model at a set of locations. For additional details on how this program works, see the "Using GeoModelGrids" section below. +The `geomodelgrids` suite of command line programs targets end users who simply want to query for values in a model at a set of locations. +For additional details on how this program works, see the "Command line programs" section. -We also provide an API for calling the query routines from external code. See the "C++ API" and "C API" sections for more information. +We also provide an API for calling the query routines from external code. +See the [C++ API]({{ "/cxx-api/serial.html" | relative_url }}) and [C API]({{ "/c-api/serial.html" | relative_url }}) sections for more information. diff --git a/docs/index.md b/docs/index.md index 218d268..026ac78 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,8 +13,8 @@ resolution. desktops to large clusters. * Support variable resolution models that include topography. -# Target Use Cases +# Use Cases -* Storing and querying 3D seismic velocity models for computer -simulations of earthquake ground motions. +There are many possible use cases. +Our first target use case is storing and querying 3D seismic velocity models for simulation and analysis of earthquake ground motions. diff --git a/docs/scripting/data-structures.md b/docs/scripting/data-structures.md index 216e7a6..88199aa 100644 --- a/docs/scripting/data-structures.md +++ b/docs/scripting/data-structures.md @@ -3,4 +3,4 @@ The `geomodelgrids` Python API uses a number of data structures to store and manage a model. -* **:TODO:** WAITING FOR IMPLEMENTATION OF QUERY API +**Warning** A Python API for querying has not yet been implemented yet. See issue [#57](https://github.com/baagaard-usgs/geomodelgrids/issues/57). diff --git a/docs/scripting/scripting.md b/docs/scripting/scripting.md index 7404fbb..60bb8c1 100644 --- a/docs/scripting/scripting.md +++ b/docs/scripting/scripting.md @@ -1,3 +1,3 @@ # Python Scripting -* **:TODO:** ADD MORE HERE +**Warning** A Python API for querying has not yet been implemented yet. See issue [#57](https://github.com/baagaard-usgs/geomodelgrids/issues/57). From e0484b1189ef8da6607fb2681f24dbffd8419dc8 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Tue, 4 May 2021 10:07:32 -0600 Subject: [PATCH 10/13] Set version number to 1.0.0rc1. --- CHANGES.md | 3 ++- configure.ac | 2 +- docs/_config.yml | 2 +- docs/getting-started/release-notes.md | 2 +- setup.cfg | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 174cfd0..1bfcf81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,8 @@ -# Version 1.0.0 +# Version 1.0.0rc1 ## Known issues * The interpolation algorithm does not respect fault block and zone boundaries and the boundary between water and solid materials. This means fault block and zone ids will be interpolated across block and zone boundaries, and Vp and density will be interpolated Vp and density across the boundary between water and solid materials. Vs is not interpolated across the boundary between water and solid material, because it is not defined in water. * GeoModelGrids does not support unit conversions. All values returned in queries are in the units of the underlying models. We do check that all models have consistent units for values contained in multiple models. Queries for the elevations of the top surface or topography/bathymetry will be returned in the units of the input Coordinate Reference System. * The model storage has been optimized for faster successive queries in the vertical direction. That is, querying for points on a vertical slice will generally be faster than querying the same number of points on a horizontal slice. + diff --git a/configure.ac b/configure.ac index b24091b..9a6462f 100644 --- a/configure.ac +++ b/configure.ac @@ -6,7 +6,7 @@ # ---------------------------------------------------------------------- AC_PREREQ(2.59) -AC_INIT([geomodelgrids], [0.2.0], [baagaard@usgs.gov]) +AC_INIT([geomodelgrids], [1.0.0rc1], [baagaard@usgs.gov]) AC_CONFIG_HEADER([portinfo]) AC_CONFIG_AUX_DIR([./aux-config]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/docs/_config.yml b/docs/_config.yml index cc4b9db..006d49b 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -10,4 +10,4 @@ heading_anchors: false search_enabled: true title_image: images/USGSsimple.png -software_version: 0.2.0 +software_version: 1.0.0rc1 diff --git a/docs/getting-started/release-notes.md b/docs/getting-started/release-notes.md index 174cfd0..9db3696 100644 --- a/docs/getting-started/release-notes.md +++ b/docs/getting-started/release-notes.md @@ -1,4 +1,4 @@ -# Version 1.0.0 +# Version 1.0.0rc1 ## Known issues diff --git a/setup.cfg b/setup.cfg index da196e6..8ccae4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ tag_date = 0 [metadata] name = geomodelgrids -version = 0.2.0 +version = 1.0.0rc1 author = Brad Aagaard author_email = baagaard@usgs.gov description = USGS Georeferenced Gridded Models From 14733d4febd3363483155b6119d50920d6f96ce7 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Tue, 4 May 2021 20:51:42 -0600 Subject: [PATCH 11/13] Cleanup build script for binary. Trim Linux tarball. --- docker/build_binary.py | 109 +++++++++++++++++++++++++++-------------- docker/linux-binary | 1 - docker/pip.conf | 2 + 3 files changed, 75 insertions(+), 37 deletions(-) create mode 100644 docker/pip.conf diff --git a/docker/build_binary.py b/docker/build_binary.py index a6f1d88..4748583 100755 --- a/docker/build_binary.py +++ b/docker/build_binary.py @@ -5,10 +5,13 @@ * gcc (Linux only) * sqlite3 * libtiff + * openssl + * libcurl * proj * hdf5 2. Install geomodelgrids 3. Create setup.sh +4. Update linking (Darwin only) 4. Create tarball. """ @@ -40,7 +43,7 @@ def __init__(self, install_dir, env, show_progress=True): def install(self): self.download() self.extract_tarball() - self.configure(args=None) + self.configure() self.build() def download(self, url=None, tarball=None): @@ -65,7 +68,7 @@ def extract_tarball(self, tarball=None, path="."): tfile = tarfile.open(tarball, mode="r:*") tfile.extractall(path=path) - def configure(self, args): + def configure(self, args=[]): if not os.path.exists(self.BUILD_DIR): os.mkdir(self.BUILD_DIR) cwd = os.path.abspath(os.curdir) @@ -247,7 +250,7 @@ def install(self): class HDF5(Package): VERSION = "1.12.0" TARBALL = f"hdf5-{VERSION}.tar.gz" - URL = f"https://www.sqlite.org/2021/{TARBALL}" + URL = f"https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_0/source/{TARBALL}" BUILD_DIR = "hdf5-build" SRC_DIR = f"hdf5-{VERSION}" @@ -350,38 +353,6 @@ def main(self, **kwargs): if args.create_tarball or args.all: self._create_tarball() - def _parse_command_line(self): - """Parse command line arguments. - """ - DESCRIPTION = ( - "Application for creating GeoModelGrids binary packages." - ) - - parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument("--install-dir", action="store", dest="install_dir", - required=True, help="Install directory.") - parser.add_argument("--version", action="store", dest="version", - required=True, help="Set version tag.") - - parser.add_argument("--gcc", action="store_true", dest="install_gcc", help="Install gcc.") - parser.add_argument("--sqlite", action="store_true", dest="install_sqlite", help="Install sqlite.") - parser.add_argument("--tiff", action="store_true", dest="install_tiff", help="Install libtiff.") - parser.add_argument("--openssl", action="store_true", dest="install_openssl", help="Install openssl.") - parser.add_argument("--curl", action="store_true", dest="install_curl", help="Install libcurl.") - parser.add_argument("--proj", action="store_true", dest="install_proj", help="Install proj.") - parser.add_argument("--hdf5", action="store_true", dest="install_hdf5", help="Install HDF5.") - parser.add_argument("--geomodelgrids", action="store_true", dest="install_gmg", help="Install GeoModelGrids.") - parser.add_argument("--create-setup", action="store_true", dest="create_setup", help="Create setup.sh.") - parser.add_argument("--update-linking", action="store_true", - dest="update_linking", help="Update Darwin linking.") - parser.add_argument("--create-tarball", action="store_true", dest="create_tarball", help="Create tarball.") - - parser.add_argument("--all", action="store_true", dest="all", help="Run all steps.") - parser.add_argument("--quiet", action="store_false", dest="show_progress", default=True) - args = parser.parse_args() - - return args - def _install(self, cls): dep = cls(self.install_dir, self.env, show_progress=self.show_progress) dep.install() @@ -410,6 +381,7 @@ def _set_env(self): raise ValueError(f"Unknown platform '{platform.system()}'.") def _create_setup(self): + self._status("Creating setup.sh ...") filename = os.path.join(self.install_dir, "setup.sh") with open(filename, "w") as fout: lines = [ @@ -434,6 +406,35 @@ def _create_setup(self): fout.write("\n".join(lines)) def _create_tarball(self): + def exclude(tarinfo): + EXCLUDE = ( + "gcc", + "g++", + "gcov", + "gcov-dump", + "gcov-tool", + "cpp", + "cc", + "c++", + "lto-dump", + ) + filepath = tarinfo.name + if os.path.splitext(filepath)[1] == ".a": + return None + filename = os.path.split(filepath)[1] + if filename in EXCLUDE: + return None + if filename.startswith("x86_64-pc-linux-gnu"): + return None + if filename.startswith("libasan") or \ + filename.startswith("libtsan") or \ + filename.startswith("libubsan") or \ + filename.startswith("liblsan"): + return None + if os.path.split(filepath)[0].endswith("libexec"): + return None + return tarinfo + self._status("Creating tarball ...") version = GeoModelGrids.VERSION if platform.system() == "Darwin": @@ -444,7 +445,43 @@ def _create_tarball(self): shutil.copytree(f"geomodelgrids-{version}", os.path.join(self.install_dir, "src")) with tarfile.open(tarball, mode="w:gz") as tfile: - tfile.add(self.install_dir, arcname=f"geomodelgrids-{version}-{arch}") + tfile.add(self.install_dir, arcname=f"geomodelgrids-{version}-{arch}", filter=exclude) + + def _parse_command_line(self): + """Parse command line arguments. + """ + DESCRIPTION = ( + "Application for creating GeoModelGrids binary packages." + ) + + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument("--install-dir", action="store", dest="install_dir", + required=True, help="Install directory.") + parser.add_argument("--version", action="store", dest="version", + required=True, help="Set version tag.") + + parser.add_argument("--gcc", action="store_true", dest="install_gcc", help="Install gcc.") + parser.add_argument("--sqlite", action="store_true", dest="install_sqlite", help="Install sqlite.") + parser.add_argument("--tiff", action="store_true", dest="install_tiff", help="Install libtiff.") + parser.add_argument("--openssl", action="store_true", dest="install_openssl", help="Install openssl.") + parser.add_argument("--curl", action="store_true", dest="install_curl", help="Install libcurl.") + parser.add_argument("--proj", action="store_true", dest="install_proj", help="Install proj.") + parser.add_argument("--hdf5", action="store_true", dest="install_hdf5", help="Install HDF5.") + parser.add_argument("--geomodelgrids", action="store_true", dest="install_gmg", help="Install GeoModelGrids.") + parser.add_argument("--create-setup", action="store_true", dest="create_setup", help="Create setup.sh.") + parser.add_argument("--update-linking", action="store_true", + dest="update_linking", help="Update Darwin linking.") + parser.add_argument("--create-tarball", action="store_true", dest="create_tarball", help="Create tarball.") + + parser.add_argument("--all", action="store_true", dest="all", help="Run all steps.") + parser.add_argument("--quiet", action="store_false", dest="show_progress", default=True) + args = parser.parse_args() + + return args + + def _status(self, msg): + if self.show_progress: + print(msg) # -------------------------------------------------------------------------------------------------- diff --git a/docker/linux-binary b/docker/linux-binary index e5bb10d..7d9e5dc 100644 --- a/docker/linux-binary +++ b/docker/linux-binary @@ -24,7 +24,6 @@ RUN yum install -y \ vim-runtime \ centos-release-scl RUN yum install -y rh-python38 -RUN scl enable rh-python38 -- python3 -m pip install requests COPY docker/certs/ /usr/share/pki/ca-trust-source/anchors RUN update-ca-trust enable diff --git a/docker/pip.conf b/docker/pip.conf new file mode 100644 index 0000000..93cdf2d --- /dev/null +++ b/docker/pip.conf @@ -0,0 +1,2 @@ +[global] +trusted-host = pypi.org files.pythonhosted.org From 7469b89c42f81db73f62c1b4a45c643584471301 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Wed, 5 May 2021 20:19:39 -0600 Subject: [PATCH 12/13] Fix typo in Info app help. --- libsrc/geomodelgrids/apps/Info.cc | 2 +- tests/libtests/apps/TestInfo.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libsrc/geomodelgrids/apps/Info.cc b/libsrc/geomodelgrids/apps/Info.cc index 05d3ddf..29f8c27 100644 --- a/libsrc/geomodelgrids/apps/Info.cc +++ b/libsrc/geomodelgrids/apps/Info.cc @@ -195,7 +195,7 @@ geomodelgrids::apps::Info::_printHelp(void) { << " --coordsys Display model coordinate system.\n" << " --values Display names and units of values stored in the model.\n" << " --blocks Display description of blocks.\n" - << " --all Display description, coordinate system, values, and blocks" + << " --all Display description, coordinate system, values, and blocks\n" << " --verify Verify model conforms to GeoModelGrids specifications." << std::endl; } // _printHelp diff --git a/tests/libtests/apps/TestInfo.cc b/tests/libtests/apps/TestInfo.cc index 01bf8d8..180ef40 100644 --- a/tests/libtests/apps/TestInfo.cc +++ b/tests/libtests/apps/TestInfo.cc @@ -327,7 +327,7 @@ geomodelgrids::apps::TestInfo::testPrintHelp(void) { Info info; info._printHelp(); std::cout.rdbuf(coutOrig); - CPPUNIT_ASSERT_EQUAL(size_t(727), coutHelp.str().length()); + CPPUNIT_ASSERT_EQUAL(size_t(728), coutHelp.str().length()); } // testPrintHelp @@ -348,7 +348,7 @@ geomodelgrids::apps::TestInfo::testRunHelp(void) { info.run(nargs, const_cast(args)); std::cout.rdbuf(coutOrig); - CPPUNIT_ASSERT_EQUAL(size_t(727), coutHelp.str().length()); + CPPUNIT_ASSERT_EQUAL(size_t(728), coutHelp.str().length()); } // testRunHelp From 32f3e6239f2e8a5e624d8113e17536ebebfc39a0 Mon Sep 17 00:00:00 2001 From: Brad Aagaard Date: Wed, 5 May 2021 21:10:03 -0600 Subject: [PATCH 13/13] Update docs/Makefile (files were missing). Update config so repo is found when using tarball. --- docs/Makefile.am | 5 +++-- docs/_config.yml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/Makefile.am b/docs/Makefile.am index 7713fe8..363c5eb 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -37,6 +37,7 @@ EXTRA_DIST = \ figs/palettes/palette_usgs.tex \ getting-started/install.md \ getting-started/running.md \ + getting-started/release-notes.md \ images/USGSsimple.png \ models/seismic_graymer_delta.md \ overview/features.md \ @@ -45,8 +46,8 @@ EXTRA_DIST = \ overview/figs/coordsys.tex \ overview/figs/gridmapping.png \ overview/figs/hdf5layout.png \ - overview/figs/hdf5layout.tex - overview/plot_gridmapping.py* + overview/figs/hdf5layout.tex \ + overview/figs/plot_gridmapping.py \ scripting/scripting.md \ scripting/data-structures.md \ scripting/create.md \ diff --git a/docs/_config.yml b/docs/_config.yml index 006d49b..4e42ada 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -4,6 +4,7 @@ baseurl: /geomodelgrids aux_links: "GitHub repository": - "//github.com/baagaard-usgs/geomodelgrids" +repository: baagaard-usgs/geomodelgrids color_scheme: usgs_lightbg heading_anchors: false