diff --git a/.gitattributes b/.gitattributes index 596615322..792b550bd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ .gitattributes export-ignore .gitignore export-ignore +.gitmodules export-ignore diff --git a/Makefile b/Makefile index ce273b3a3..b0d3bd3d3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL=/bin/bash # Add some good stuff to CFLAGS. -export CFLAGS += -std=c11 -Wall +export CFLAGS += -std=c11 -Wall -g .PHONY: all all: VERSION.full bin/version.h bin/version.sh @@ -16,14 +16,7 @@ clean: cd test && $(MAKE) clean cd examples/syscalls && $(MAKE) clean -# VERSION.full contains the version string reported by the executables. -# -# * If VERSION is an unadorned release (e.g. 0.2.3 not 0.2.3~pre), or there's -# no Git information available, VERSION.full is simply a copy of VERSION. -# -# * Otherwise, we add the Git branch if the current branch is not master, the -# Git commit, and a note if the working directory -# contains uncommitted changes, e.g. "0.2.3~pre+experimental.ae24a4e.dirty". +# VERSION.full contains the version string reported by executables; see FAQ. ifeq ($(shell test -d .git && fgrep -q \~ VERSION && echo true),true) .PHONY: VERSION.full # depends on git metadata, not a simple file VERSION.full: VERSION @@ -31,7 +24,10 @@ VERSION.full: VERSION (echo "This is a Git working directory but no git found." && false) printf '%s+%s%s%s\n' \ $$(cat $<) \ - $$(git rev-parse --abbrev-ref HEAD | sed 's/.*/&./g' | sed 's/master.//g') \ + $$( git rev-parse --abbrev-ref HEAD \ + | sed 's/[^A-Za-z0-9]//g' \ + | sed 's/$$/./g' \ + | sed 's/master.//g') \ $$(git rev-parse --short HEAD) \ $$(git diff-index --quiet HEAD || echo '.dirty') \ > $@ @@ -44,25 +40,37 @@ bin/version.h: VERSION.full bin/version.sh: VERSION.full echo "version () { echo 1>&2 '$$(cat $<)'; }" > $@ -# Yes, this is bonkers. We keep it around even though normal "git archive" or -# the zip files on Github work, because it provides an easy way to create a -# self-contained tarball with embedded Bats and man pages. +# These targets provide tarballs of HEAD (not the Git working directory) that +# are self-contained, including the source code as well as the man pages +# (both) and Bats (export-bats). To use them in an unclean working directory, +# set $CH_UNCLEAN_EXPORT_OK to non-empty. # -# You must "cd doc-src && make" before this will work. -.PHONY: export -export: VERSION.full man/charliecloud.1 - test -d .git -a -f test/bats/.git # need recursive Git checkout -# git diff-index --quiet HEAD # need clean working directory +# You must "cd doc-src && make" before they will work. The targets depend on +# the man pages but don't know how to build them. +# +# They are phony because I haven't figured out their real dependencies. +.PHONY: main.tar +main.tar: VERSION.full man/charliecloud.1 + git diff-index --quiet HEAD || [ -n "$$CH_MAKE_EXPORT_UNCLEAN_OK" ] git archive HEAD --prefix=charliecloud-$$(cat VERSION.full)/ \ -o main.tar + tar --xform=s,^,charliecloud-$$(cat VERSION.full)/, \ + -rf main.tar man/*.1 VERSION.full + +.PHONY: export +export: main.tar + gzip -9 main.tar + mv main.tar.gz charliecloud-$$(cat VERSION.full).tar.gz + ls -lh charliecloud-$$(cat VERSION.full).tar.gz + +.PHONY: export-bats +export-bats: main.tar + test -d .git -a -f test/bats/.git # need recursive Git checkout cd test/bats && \ git archive HEAD \ --prefix=charliecloud-$$(cat ../../VERSION.full)/test/bats/ \ -o ../../bats.tar tar Af main.tar bats.tar - tar --xform=s,^,charliecloud-$$(cat VERSION.full)/, \ - -rf main.tar \ - man/*.1 VERSION.full gzip -9 main.tar mv main.tar.gz charliecloud-$$(cat VERSION.full).tar.gz rm bats.tar @@ -95,7 +103,6 @@ endif INSTALL_PREFIX := $(if $(DESTDIR),$(DESTDIR)/$(PREFIX),$(PREFIX)) BIN := $(INSTALL_PREFIX)/bin DOC := $(INSTALL_PREFIX)/share/doc/charliecloud -TEST := $(DOC)/test # LIBEXEC_DIR is modeled after FHS 3.0 and # https://www.gnu.org/prep/standards/html_node/Directory-Variables.html. It # contains any executable helpers that are not needed in PATH. Default is @@ -103,6 +110,7 @@ TEST := $(DOC)/test LIBEXEC_DIR ?= libexec/charliecloud LIBEXEC_INST := $(INSTALL_PREFIX)/$(LIBEXEC_DIR) LIBEXEC_RUN := $(PREFIX)/$(LIBEXEC_DIR) +TEST := $(LIBEXEC_INST)/test .PHONY: install install: all @test -n "$(PREFIX)" || \ @@ -129,14 +137,14 @@ install: all install -pm 644 -t $(DOC) LICENSE README.rst # examples for i in examples/syscalls examples/{serial,mpi,other}/*; do \ - install -d $(DOC)/$$i; \ - install -pm 644 -t $(DOC)/$$i $$i/*; \ + install -d $(LIBEXEC_INST)/$$i; \ + install -pm 644 -t $(LIBEXEC_INST)/$$i $$i/*; \ done - chmod 755 $(DOC)/examples/serial/hello/hello.sh \ - $(DOC)/examples/syscalls/pivot_root \ - $(DOC)/examples/syscalls/userns \ - $(DOC)/examples/*/*/*.sh - find $(DOC)/examples -name Build -exec chmod 755 {} \; + chmod 755 $(LIBEXEC_INST)/examples/serial/hello/hello.sh \ + $(LIBEXEC_INST)/examples/syscalls/pivot_root \ + $(LIBEXEC_INST)/examples/syscalls/userns \ + $(LIBEXEC_INST)/examples/*/*/*.sh + find $(LIBEXEC_INST)/examples -name Build -exec chmod 755 {} \; # tests install -d $(TEST) $(TEST)/run install -pm 644 -t $(TEST) test/*.bats test/common.bash test/Makefile @@ -148,7 +156,7 @@ install: all install -d $(TEST)/chtest install -pm 644 -t $(TEST)/chtest test/chtest/* chmod 755 $(TEST)/chtest/{Build,*.py,printns} - ln -sf ../../../../bin $(TEST)/bin + ln -sf ../../../bin $(TEST)/bin # shared library tests install -d $(TEST)/sotest $(TEST)/sotest/bin $(TEST)/sotest/lib install -pm 755 -t $(TEST)/sotest test/sotest/libsotest.so.1.0 \ diff --git a/bin/ch-run.c b/bin/ch-run.c index a47319461..abd3edb9a 100644 --- a/bin/ch-run.c +++ b/bin/ch-run.c @@ -44,6 +44,7 @@ const struct argp_option options[] = { { "bind", 'b', "SRC[:DST]", 0, "mount SRC at guest DST (default /mnt/0, /mnt/1, etc.)"}, { "cd", 'c', "DIR", 0, "initial working directory in container"}, + { "ch-ssh", -8, 0, 0, "bind ch-ssh into image"}, { "gid", 'g', "GID", 0, "run as GID within container" }, { "join", 'j', 0, 0, "use same container as peer ch-run" }, { "join-pid", -5, "PID", 0, "join a namespace using a PID" }, @@ -106,6 +107,7 @@ int main(int argc, char *argv[]) privs_verify_invoking(); T_ (args.c.binds = calloc(1, sizeof(struct bind))); + args.c.ch_ssh = false; args.c.container_gid = getegid(); args.c.container_uid = geteuid(); args.c.join = false; @@ -384,6 +386,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) Te (strlen(arg) > 0, "--unset-env: GLOB must have non-zero length"); env_delta_append(&(args->env_deltas), UNSET_GLOB, arg); break;; + case -8: // --ch-ssh + args->c.ch_ssh = true; + break; case 'c': args->initial_dir = arg; break; diff --git a/bin/charliecloud.c b/bin/charliecloud.c index 1fc1b63f9..4b746c910 100644 --- a/bin/charliecloud.c +++ b/bin/charliecloud.c @@ -203,11 +203,13 @@ void enter_udss(struct container *c) Z_ (mkdir(cat(c->newroot, newhome), 0755)); bind_mount(c->old_home, newhome, c->newroot, BD_REQUIRED, 0); } - // Bind-mount /usr/bin/ch-ssh if it exists. - if (path_exists(cat(c->newroot, "/usr/bin/ch-ssh"))) { + // Container /usr/bin/ch-ssh. + if (c->ch_ssh) { char chrun_file[PATH_CHARS]; int len = readlink("/proc/self/exe", chrun_file, PATH_CHARS); T_ (len >= 0); + Te (path_exists(cat(c->newroot, "/usr/bin/ch-ssh")), + "--ch-ssh: /usr/bin/ch-ssh not in image"); chrun_file[ lennewroot, BD_REQUIRED, 0); diff --git a/bin/charliecloud.h b/bin/charliecloud.h index d48543a0a..7629c7018 100644 --- a/bin/charliecloud.h +++ b/bin/charliecloud.h @@ -65,6 +65,7 @@ enum bind_dep { struct container { struct bind *binds; + bool ch_ssh; // bind /usr/bin/ch-ssh? gid_t container_gid; uid_t container_uid; char *newroot; diff --git a/doc-src/ch-run_desc.rst b/doc-src/ch-run_desc.rst index 228ffcf76..5e4d8c020 100644 --- a/doc-src/ch-run_desc.rst +++ b/doc-src/ch-run_desc.rst @@ -18,6 +18,9 @@ unpacked image directory located at :code:`NEWROOT`. :code:`-c`, :code:`--cd=DIR` initial working directory in container + :code:`--ch-ssh` + bind :code:`ch-ssh(1)` into container at :code:`/usr/bin/ch-ssh` + :code:`-g`, :code:`--gid=GID` run as group :code:`GID` within container diff --git a/doc-src/dev.rst b/doc-src/dev.rst index c210856a0..5513cb068 100644 --- a/doc-src/dev.rst +++ b/doc-src/dev.rst @@ -411,6 +411,110 @@ suite. * else: An error occurred. +Building RPMs +============= + +We maintain :code:`.spec` files and infrastructure for building RPMs in the +Charliecloud source code. This is for two purposes: + + 1. We maintain our own Fedora RPMs. + 2. We want to be able to build an RPM of any commit. + +Item 2 is tested; i.e., if you break the RPM build, the test suite will fail. + +This section describes how to build the RPMs and the pain we've hopefully +abstracted away. + +Prerequisites +------------- + + * Python 2.7 + * Either: + + * RPM-based system of roughly RHEL/CentOS 7 vintage or newer, with RPM + build tools installed + * System that can run Charliecloud containers + +:code:`rpmbuild` wrapper script +------------------------------- + +While building the Charliecloud RPMs is not too weird, we provide a script to +streamline it. The purpose is to (a) make it easy to build versions not +matching the working directory, (b) use an arbitrary :code:`rpmbuild` +directory, and (c) build in a Charliecloud container for non-RPM-based +environments. + +The script must be run from the root of a Charliecloud Git working directory. + +Usage:: + + $ packaging/fedora/build [OPTIONS] VERSION + +Options: + + * :code:`--image=DIR` : Build in Charliecloud image directory :code:`DIR`. + + * :code:`--install` : Install the RPMs after building into the build + environment. + + * :code:`--rpmbuild=DIR` : Use RPM build directory root :code:`DIR` + (default: :code:`~/rpmbuild`). + +For example, to build a version 0.9.7 RPM, on an RPM system, and leave the +results in :code:`~/rpmbuild/RPMS`:: + + $ packaging/fedora/build 0.9.7-1 + +To build a pre-release RPM of Git HEAD using the CentOS 7 image provided with +the test suite (note that the test suite would also build the necessary image +directory):: + + $ bin/ch-build -t centos7 -f test/Dockerfile.centos7 test + $ bin/ch-docker2tar centos7 $CH_TEST_TARDIR + $ bin/ch-tar2dir $CH_TEST_TARDIR/centos7.tar.gz $CH_TEST_IMGDIR + $ packaging/fedora/build --image $CH_TEST_IMGDIR/centos7 HEAD + +Gotchas and quirks +------------------ + +RPM versions and releases +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If :code:`VERSION` is :code:`HEAD`, then the RPM version will be the content +of :code:`VERSION.full` for that commit, including Git gobbledygook, and the +RPM release will be :code:`0`. Note that such RPMs cannot be reliably upgraded +because their version numbers are unordered. + +Otherwise, :code:`VERSION` should be a released Charliecloud version followed +by a hyphen and the desired RPM release, e.g. :code:`0.9.7-3`. + +Other values of :code:`VERSION` (e.g., a branch name) may work but are not +supported. + +Packaged source code and RPM build config come from different commits +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The spec file, :code:`build` script, :code:`.rpmlintrc`, etc. come from the +working directory, but the package source is from the specified commit. This +is what enables us to make additional RPM releases for a given Charliecloud +release (e.g. 0.9.7-2). + +Corollaries of this policy are that RPM build configuration can be any or no +commit, and it's not possible to create an RPM of uncommitted source code. + +Changelog maintenance +~~~~~~~~~~~~~~~~~~~~~ + +The spec file changelog contains manually maintained release notes for all +Charliecloud released versions and corresponding RPM releases. For released +versions, :code:`build` verifies that the most recent changelog entry matches +the given :code:`VERSION` argument. The timestamp is not automatically +verified. + +For other Charliecloud versions, :code:`build` adds a generic changelog entry +with the appropriate version stating that it's a pre-release RPM. + + Coding style ============ diff --git a/doc-src/faq.rst b/doc-src/faq.rst index 49b9ed863..ca5968ee8 100644 --- a/doc-src/faq.rst +++ b/doc-src/faq.rst @@ -116,6 +116,46 @@ two solutions: Unexpected behavior =================== +What do the version numbers mean? +--------------------------------- + +Released versions of Charliecloud have a pretty standard version number, e.g. +0.9.7. + +Work leading up to a released version also has version numbers, to satisfy +tools that require them and to give the executables something useful to report +on :code:`--version`, but these can be quite messy. We refer to such versions +informally as *pre-releases*, but Charliecloud does not have formal +pre-releases such as alpha, beta, or release candidate. + +*Pre-release version numbers are not in order*, because this work is in a DAG +rather than linear, except they precede the version we are working towards. If +you're dealing with these versions, use Git. + +Pre-release version numbers are the version we are working towards, followed +by: :code:`~pre`, the branch name if not :code:`master` with non-alphanumerics +removed, the commit hash, and finally :code:`dirty` if the working directory +had uncommitted changes. + +Examples: + + * :code:`0.2.0` : Version 0.2.0. Released versions don't include Git + information, even if built in a Git working directory. + + * :code:`0.2.1~pre` : Some snapshot of work leading up to 0.2.1, built from + source code where the Git information has been lost, e.g. the tarballs + Github provides. This should make you wary because you don't have any + provenance. It might even be uncommitted work or an abandoned branch. + + * :code:`0.2.1~pre.1a99f42` : Master branch commit 1a99f42, built from a + clean working directory (i.e., no changes since that commit). + + * :code:`0.2.1~pre.foo1.0729a78` : Commit 0729a78 on branch :code:`foo-1`, + :code:`foo_1`, etc. built from clean working directory. + + * :code:`0.2.1~pre.foo1.0729a78.dirty` : Commit 0729a78 on one of those + branches, plus un-committed changes. + :code:`--uid 0` lets me read files I can’t otherwise! ----------------------------------------------------- diff --git a/packaging/fedora/build b/packaging/fedora/build new file mode 100755 index 000000000..836c984a5 --- /dev/null +++ b/packaging/fedora/build @@ -0,0 +1,200 @@ +#!/usr/bin/env python2.7 + +# See contributors' guide for documentation of this script. + +from __future__ import print_function + +import argparse +import errno +import os +import pipes +import pwd +import re +import shutil +import socket +import subprocess +import time + +CH_BASE = os.path.abspath(os.path.dirname(__file__) + "/../..") +CH_RUN = [CH_BASE + "/bin/ch-run"] +PACKAGES = ["charliecloud", "charliecloud-debuginfo", "charliecloud-doc"] + +def main(): + + # Parse arguments. + ap = argparse.ArgumentParser() + ap.add_argument("version") + ap.add_argument("--image", metavar="DIR") + ap.add_argument("--install", action="store_true") + ap.add_argument("--rpmbuild", metavar="DIR", + default="%s/rpmbuild" % os.getenv("HOME")) + args = ap.parse_args() + print("# Charliecloud root: %s" % CH_BASE) + print("""\ +# version: %(version)s +# image: %(image)s +# install: %(install)s +# rpmbuild root: %(rpmbuild)s""" % args.__dict__) + + # What's the real Git version? + if (args.version == "HEAD"): + try: + # If we're on a branch, we want to build on that branch so the branch + # name shows up in the version name. + commit = subprocess.check_output(["git", "symbolic-ref", "-q", + "--short", "HEAD"])[:-1] + except subprocess.CalledProcessError as x: + if (x.returncode != 1): raise + # Detached HEAD (e.g. Travis) is also fine; use commit hash. + commit = subprocess.check_output(["git", "rev-parse", + "--verify", "HEAD"])[:-1] + rpm_release = "0" + else: + m = re.search(r"([0-9.]+)-([0-9]+)", args.version) + branch = "v" + m.group(1) + rpm_release = m.group(2) + + # Create rpmbuild root + rpm_sources = args.rpmbuild + '/SOURCES' + rpm_specs = args.rpmbuild + '/SPECS' + for d in (rpm_sources, rpm_specs): + print("# mkdir -p %s" % d) + try: + os.makedirs(d) + except OSError as x: + if (x.errno != errno.EEXIST): raise + + # Get a clean Git checkout of the desired version. We do this by making a + # temporary clone so as not to mess up the WD. + git_tmp = rpm_sources + '/charliecloud' + print("# cloning into %s and checking out commit %s" % (git_tmp, commit)) + cmd("git", "clone", '.', git_tmp) + cmd("git", "checkout", commit, cwd=git_tmp) + + # Build tarball. + print("# building docs") + cmd("make", "-j2", cwd=git_tmp+"/doc-src") + print("# building source tarball") + cmd("make", "export", cwd=git_tmp) + ch_version = open(git_tmp + "/VERSION.full").read()[:-1] + ch_tarball = "charliecloud-%s.tar.gz" % ch_version + print("# Charliecloud version: %s" % ch_version) + print("# source tarball: %s" % ch_tarball) + os.rename("%s/%s" % (git_tmp, ch_tarball), + "%s/%s" % (rpm_sources, ch_tarball)) + + # Copy lint configuration. + # FIXME: Put version into destination sometime? + shutil.copy("%s/packaging/fedora/charliecloud.rpmlintrc" % CH_BASE, + "%s/charliecloud.rpmlintrc" % rpm_specs) + + # Remove temporary Git directory. + print("# rm -rf %s" % git_tmp) + shutil.rmtree(git_tmp) + + # Copy and patch spec file. + rpm_vr = "%s-%s" % (ch_version, rpm_release) + spec = "charliecloud-%s.spec" % rpm_vr + with open("%s/packaging/fedora/charliecloud.spec" % CH_BASE, "rt") as in_, \ + open("%s/%s" % (rpm_specs, spec), "wt") as out: + print("# writing %s" % out.name) + t = in_.read() + t = t.replace("@VERSION@", ch_version) + t = t.replace("@RELEASE@", rpm_release) + if ("~pre" in ch_version): + # Add dummy changelog entry. + timestamp = time.strftime("%a %b %d %Y") # required format + name = pwd.getpwuid(os.geteuid()).pw_gecos.split(",")[0] + moniker = pwd.getpwuid(os.geteuid()).pw_name + domain = re.sub(r"^[^.]+.", "", socket.getfqdn()) + t = t.replace("%changelog\n", """\ +%%changelog +* %s %s <%s@%s> %s +- Pre-release package. See Git history for what is going on. +""" % (timestamp, name, moniker, domain, rpm_vr)) + else: + # Last changelog entry must match version. + assert False, "unimplemented" + out.write(t) + + # Prepare build and rpmlint arguments. + container = [] + rpmbuild_args = [] + rpmlint_args = [] + if (not args.image): + rpms = args.rpmbuild + "/RPMS/x86_64" + rpmbuild_args += ["--define", "_topdir " + args.rpmbuild] + rpmlint_args += ["--file", "%s/charliecloud.rpmlintrc" % rpm_specs] + else: + # Use /usr/local/src because rpmbuild fails with "%{_topdir}/BUILD" + # shorter than "/usr/src/debug" (yes, really!) [1,2]. + # + # [1]: https://access.redhat.com/solutions/1426113 + # [2]: https://gbenson.net/?p=367 + rpms = "/usr/local/src/RPMS/x86_64" + rpm_specs = "/usr/local/src/SPECS" + rpm_sources = "/usr/local/src/SOURCES" + rpmbuild_args += ["--define", "_topdir /usr/local/src"] + rpmlint_args += ["--file", "%s/charliecloud.rpmlintrc" % rpm_specs] + container += [CH_BASE + "/bin/ch-run", "-w", + "-b", "%s:/usr/local/src" % args.rpmbuild, + args.image, "--"] + + # Build RPMs. + cmd(container, "rpmbuild", rpmbuild_args, "--version") + cmd(container, "rpmbuild", rpmbuild_args, "-ba", "%s/%s" % (rpm_specs, spec)) + cmd(container, "ls", "-lh", rpms) + + # Install RPMs. + if (args.install): + print("# uninstalling (most errors can be ignored)") + cmd_ok(container, "rpm", "--erase", PACKAGES) + print("# installing") + for p in PACKAGES: + cmd(container, "rpm", "--install", + "%s/%s-%s.*.rpm" % (rpms, p, rpm_vr)) + cmd(container, "rpm", "-qa", "charliecloud*") + + # Lint RPMs and spec file. Last so problems that don't result in program + # returning error are more obvious. + print("# linting") + cmd(container, "rpmlint", rpmlint_args, "%s/%s" % (rpm_specs, spec)) + for p in PACKAGES: + file_ = "%s/%s-%s.el7.x86_64.rpm" % (rpms, p, rpm_vr) + cmd(container, "test", "-e", file_) + cmd(container, "rpmlint", rpmlint_args, file_) + + # Success! + print("# done") + + +def cmd(*args, **kwargs): + cmd_real(subprocess.check_call, *args, **kwargs) + +def cmd_ok(*args, **kwargs): + rc = cmd_real(subprocess.call, *args, **kwargs) + return (rc == 0) + +def cmd_out(*args, **kwargs): + out = cmd_real(subprocess.check_output, *args, **kwargs) + return out.rstrip() # remove trailing newline + +def cmd_real(runf, *args, **kwargs): + # flatten any sub-lists (kludge) + args2 = [] + for arg in args: + if (isinstance(arg, list)): + args2 += arg + else: + args2.append(arg) + # print and run + print("$", end="") + for arg in args2: + arg = pipes.quote(arg) + print(" " + arg, end="") + print() + return runf(args2, **kwargs) + + +if (__name__ == "__main__"): + main() diff --git a/packaging/fedora/charliecloud.rpmlintrc b/packaging/fedora/charliecloud.rpmlintrc new file mode 100644 index 000000000..a25dd4050 --- /dev/null +++ b/packaging/fedora/charliecloud.rpmlintrc @@ -0,0 +1,25 @@ +# This file is used to supress false positive errors and warnings generated by +# rpmlint when used with our charliecloud packages. + +# charliecloud.spec + +# The RPM build script will generate invalid source URLs for non-release +# versions, e.g., '0.9.8~pre+epelpackage.41fe9fd'. +addFilter(r'invalid-url') + +# charliecloud-doc + +# Charliecloud is a container runtime. The libsotest objects are test suite +# resources that are injected into a container (guest), these objects are not +# used on the host. +addFilter(r'no-ldconfig-symlink') +addFilter(r'library-without-ldconfig-postin') +addFilter(r'library-without-ldconfig-postun') + +# The example/*.c and test/*.c files are example files that illustrate various +# details of the charliecloud runtime. For all intents and purposes, they are +# part of our documentation. +addFilter(r'devel-file-in-non-devel-package') + +# The symlink to /usr/bin is created and does exist. +addFilter(r'dangling-relative-symlink') diff --git a/packaging/fedora/charliecloud.spec b/packaging/fedora/charliecloud.spec new file mode 100644 index 000000000..bfaf91993 --- /dev/null +++ b/packaging/fedora/charliecloud.spec @@ -0,0 +1,81 @@ +Name: charliecloud +Version: @VERSION@ +Release: @RELEASE@%{?dist} +Summary: Lightweight user-defined software stacks for high-performance computing +License: ASL 2.0 +URL: https://hpc.github.io/%{name}/ +Source0: https://github.com/hpc/%{name}/archive/v%{version}/%{name}-%{version}.tar.gz +BuildRequires: gcc >= 4.8.5 +BuildRequires: make >= 3.82 + +%package doc +Summary: Charliecloud examples and test suite +Requires: %{name} = %{version} +Requires: bats >= 0.4.0 +Requires: bash >= 4.2.46 +Requires: wget >= 1.14 + +%description +Charliecloud uses Linux user namespaces to run containers with no privileged +operations or daemons and minimal configuration changes on center resources. +This simple approach avoids most security risks while maintaining access to +the performance and functionality already on offer. + +Container images can be built using Docker or anything else that can generate +a standard Linux filesystem tree. + +For more information: https://hpc.github.io/charliecloud/ + +%description doc +Charliecloud test suite and examples. + +# Voodoo to stop our python scripts from being byte compiled on Centos7, which +# otherwise results in rpmbuild failing to build the package. see: +# https://github.com/scylladb/scylla/issues/2235 +%global __os_install_post \ + /usr/lib/rpm/redhat/brp-compress \ + %{!?__debug_package:\ + /usr/lib/rpm/redhat/brp-strip %{__strip} \ + /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \ + } \ + /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \ + %{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \ +%{nil} + +%prep +%setup -q + +%build +%{__make} %{?mflags} + +%install +%{__make} %{?mflags_install} install PREFIX=%{_prefix} DESTDIR=%{buildroot} + +%files +# Documentation +%doc %{_datadir}/doc/%{name}/LICENSE +%doc %{_datadir}/doc/%{name}/README.rst +%doc %{_mandir}/man1/ch* + +# Helper scripts +%{_libexecdir}/%{name}/base.sh +%{_libexecdir}/%{name}/version.sh +%{_bindir}/ch-build +%{_bindir}/ch-build2dir +%{_bindir}/ch-docker2tar +%{_bindir}/ch-fromhost +%{_bindir}/ch-pull2dir +%{_bindir}/ch-pull2tar +%{_bindir}/ch-tar2dir + +# Binaries +%{_bindir}/ch-run +%{_bindir}/ch-ssh + +%files doc +%doc %{_datadir}/doc/%{name}/LICENSE +%doc %{_datadir}/doc/%{name}/README.rst +%{_libexecdir}/%{name}/examples +%{_libexecdir}/%{name}/test + +%changelog diff --git a/packaging/redhat/README b/packaging/redhat/README deleted file mode 100644 index c2a3cc275..000000000 --- a/packaging/redhat/README +++ /dev/null @@ -1,3 +0,0 @@ -You will need to replace @VERSION@ in the .spec file. - -See travis.sh for an example of a build, though under Travis' Ubuntu image. diff --git a/packaging/redhat/charliecloud.spec b/packaging/redhat/charliecloud.spec deleted file mode 100644 index ccc48a168..000000000 --- a/packaging/redhat/charliecloud.spec +++ /dev/null @@ -1,52 +0,0 @@ -Summary: Lightweight user-defined software stacks for high-performance computing -Name: charliecloud -Version: @VERSION@ -Release: %{?dist} -License: Apache-2.0 -Group: System Environment/Base -URL: https://hpc.github.io/charliecloud/ -Source: %{name}-%{version}.tar.gz -ExclusiveOS: linux -BuildRoot: %{?_tmppath}%{!?_tmppath:/var/tmp}/%{name}-%{version}-%{release}-root -BuildRequires: python python-sphinx python-sphinx_rtd_theme rsync - -%description -Charliecloud provides user-defined software stacks (UDSS) for -high-performance computing (HPC) centers. - -%prep -%setup -q -# Required for CentOS 7 and older, which don't know of Docker lexer yet. -#find doc-src -type f -print0 | xargs -0 sed -i '/.*:language: docker.*/d' - -%build -%{__make} %{?mflags} - -%install -LIBEXEC_POSTFIX=$(echo %{_libexecdir} | sed 's#^/usr/##') -PREFIX=/usr LIBEXEC_DIR=${LIBEXEC_POSTFIX}/charliecloud DESTDIR=$RPM_BUILD_ROOT %{__make} install %{?mflags_install} -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/examples -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/doc -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/test -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/COPYRIGHT -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/LICENSE -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/README.rst - -%clean -rm -rf $RPM_BUILD_ROOT - -#%check -#%{__make} -C test test-quick - -%files -%doc LICENSE README.rst examples -%{_mandir}/man1/* - -# Helper scripts -%{_libexecdir}/%{name}/base.sh -%{_libexecdir}/%{name}/version.sh - -# Binaries -%{_bindir}/ch-* - -%changelog diff --git a/packaging/redhat/travis.sh b/packaging/redhat/travis.sh deleted file mode 100755 index ca6530d9a..000000000 --- a/packaging/redhat/travis.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Build a .rpm from the current source code diretory. $PWD must be the root of -# the Charliecloud source code. -# -# Because this is designed to work on a Ubuntu box, which is Debian-based -# rather than Red Hat, some things are a little odd. - -set -e -set -x - -sudo apt-get install rpm - -RPMBUILD=~/rpmbuild -mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES,SPECS,SRPMS} - -make VERSION.full -VERSION=$(cat VERSION.full) - -tar czf $RPMBUILD/SOURCES/charliecloud-"${VERSION}".tar.gz \ - --xform "s#^\\.#charliecloud-${VERSION}#" \ - --exclude=.git \ - . - -cp packaging/redhat/charliecloud.spec $RPMBUILD/SPECS -sed -i "s#Version: @VERSION@#Version: ${VERSION}#g" $RPMBUILD/SPECS/charliecloud.spec - -# This is handled automatically on Red Hat systems. -sed -i "s#Release: %{?dist}#Release: 1#g" ~/rpmbuild/SPECS/charliecloud.spec - -# Build requirements cannot be satisfied on Debian derivatives. -sed -i 's#BuildRequires:.*##g' ~/rpmbuild/SPECS/charliecloud.spec - -#echo "Prepared rpmbuild directory tree:" -#find $RPMBUILD -#cho "Now starting build!" - -cd $RPMBUILD/SPECS - -rpmbuild -ba charliecloud.spec - -#echo "Done, tree now:" -#find $RPMBUILD - -# RPMLINT not available for 14.04 :-( -#echo "Running rpmlint:" -#cd $RPMBUILD/RPMS -#rpmlint * diff --git a/test/Dockerfile.centos7 b/test/Dockerfile.centos7 index ab16e0e29..dea831036 100644 --- a/test/Dockerfile.centos7 +++ b/test/Dockerfile.centos7 @@ -1,5 +1,15 @@ -# ch-test-scope: full +# ch-test-scope: standard FROM centos:7 -RUN yum -y install bc +# This image has two purposes: (1) demonstrate we can build a CentOS 7 image +# and (2) provide a build environment for Charliecloud RPMs. + +RUN yum -y install epel-release +RUN yum -y install \ + bats \ + gcc \ + make \ + rpm-build \ + rpmlint \ + wget RUN yum clean all diff --git a/test/run/build-rpms.bats b/test/run/build-rpms.bats new file mode 100644 index 000000000..435b10e94 --- /dev/null +++ b/test/run/build-rpms.bats @@ -0,0 +1,32 @@ +load ../common + +@test 'build RPMs' { + scope standard + prerequisites_ok centos7 + [[ -d ../.git ]] || skip "not in Git working directory" + command -v sphinx-build > /dev/null 2>&1 || skip 'Sphinx is not installed' + img=${ch_imgdir}/centos7 + + # Build and install RPMs into CentOS 7 image. + (cd .. && packaging/fedora/build --install --image="$img" \ + --rpmbuild="$BATS_TMPDIR/rpmbuild" HEAD) + + # Do installed RPMs look sane? + run ch-run "$img" -- rpm -qa "charliecloud*" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'charliecloud-'* ]] + [[ $output = *'charliecloud-doc-'* ]] + run ch-run "$img" -- rpm -ql "charliecloud" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'/usr/bin/ch-run'* ]] + [[ $output = *'/usr/libexec/charliecloud/base.sh'* ]] + [[ $output = *'/usr/share/man/man1/charliecloud.1.gz'* ]] + run ch-run "$img" -- rpm -ql "charliecloud-doc" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'/usr/libexec/charliecloud/examples/mpi/lammps/Dockerfile'* ]] + [[ $output = *'/usr/libexec/charliecloud/test/Build.centos7xz'* ]] + [[ $output = *'/usr/libexec/charliecloud/test/sotest/lib/libsotest.so.1.0'* ]] +} diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 670b4adf2..82d4d7ddc 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -35,12 +35,15 @@ EOF @test '/usr/bin/ch-ssh' { + # Note: --ch-ssh without /usr/bin/ch-ssh is in test "broken image errors". scope quick ls -l "$ch_bin/ch-ssh" - ch-run "$ch_timg" -- ls -l /usr/bin/ch-ssh - ch-run "$ch_timg" -- test -x /usr/bin/ch-ssh + ch-run --ch-ssh "$ch_timg" -- ls -l /usr/bin/ch-ssh + ch-run --ch-ssh "$ch_timg" -- test -x /usr/bin/ch-ssh + # Test bind-mount by comparing size rather than e.g. "ch-ssh --version" + # because ch-ssh won't run on Alpine (issue #4). host_size=$(stat -c %s "${ch_bin}/ch-ssh") - guest_size=$(ch-run "$ch_timg" -- stat -c %s /usr/bin/ch-ssh) + guest_size=$(ch-run --ch-ssh "$ch_timg" -- stat -c %s /usr/bin/ch-ssh) echo "host: ${host_size}, guest: ${guest_size}" [[ $host_size -eq "$guest_size" ]] } @@ -543,7 +546,7 @@ EOF dirs=$(echo {dev,proc,sys}) files=$(echo etc/{group,hosts,passwd,resolv.conf}) # shellcheck disable=SC2116 - files_optional=$(echo usr/bin/ch-ssh) + files_optional= # formerly for ch-ssh (#378), but leave infrastructure mkdir -p "$img" for d in $dirs; do mkdir -p "${img}/$d"; done mkdir -p "${img}/etc" "${img}/home" "${img}/usr/bin" "${img}/tmp" @@ -652,6 +655,12 @@ EOF [[ $status -eq 1 ]] [[ $output = *"can't execve(2): true: No such file or directory"* ]] + # --ch-ssh but no /usr/bin/ch-ssh + run ch-run --ch-ssh "$img" -- true + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"--ch-ssh: /usr/bin/ch-ssh not in image"* ]] + # Everything should be restored and back to the original error. run ch-run "$img" -- true echo "$output" diff --git a/test/travis.sh b/test/travis.sh index 84a8356f3..577e9dc17 100755 --- a/test/travis.sh +++ b/test/travis.sh @@ -26,6 +26,14 @@ case $TARBALL in tar xf charliecloud-*.tar.gz cd charliecloud-* ;; + export-bats) + (cd doc-src && make) + make export-bats + mv charliecloud-*.tar.gz "$PREFIX" + cd "$PREFIX" + tar xf charliecloud-*.tar.gz + cd charliecloud-* + ;; archive) # The Travis image already has Bats installed. git archive HEAD --prefix=charliecloud/ -o "$PREFIX/charliecloud.tar" @@ -35,20 +43,12 @@ case $TARBALL in ;; esac -if [[ $PKG_BUILD ]]; then - for i in packaging/*/travis.sh; do $i; done - # FIXME: If we continue with the rest of the tests after building the - # packages, they hang in "make test-all", I believe in test "ch-build - # python3" but I have not been able to verify this. - exit -fi - make bin/ch-run --version if [[ $INSTALL ]]; then sudo make install PREFIX="$PREFIX" - cd "$PREFIX/share/doc/charliecloud" + cd "$PREFIX/libexec/charliecloud" fi cd test diff --git a/test/travis.yml b/test/travis.yml index 294cd3f8a..1e904f3bb 100644 --- a/test/travis.yml +++ b/test/travis.yml @@ -10,29 +10,28 @@ compiler: gcc # it up in parallel. It would be nice if "make test-build" could be done # serially before splitting into parallel jobs. # -# TARBALL= # build in Git checkout & use embedded Bats -# TARBALL=archive # build from "git archive" tarball & use system Bats -# TARBALL=export # build from "make export" tarball & use embedded Bats -# INSTALL= # run from build directory -# INSTALL=yes # make install to /usr/local, run that one +# TARBALL= # build in Git checkout & use embedded Bats +# TARBALL=archive # build from "git archive" tarball & use system Bats +# TARBALL=export # build from "make export" tarball & use system Bats +# TARBALL=export-bats # build from "make export" tarball & use embedded Bats +# INSTALL= # run from build directory +# INSTALL=yes # make install to /usr/local, run that one # -# Package builds can also be tested. In this case, INSTALL is ignored, as is -# the main test suite. These tests are much faster. +# Additional options: # # PKG_BUILD=yes # build (but don't test) distribution packages, then exit # MINIMAL_DEPS=yes # test with a minimal dependency set (no fancy tools) # env: -# Package building (fast). - - TARBALL= PKG_BUILD=yes - - TARBALL=archive PKG_BUILD=yes # Complete matrix of TARBALL and INSTALL. - - TARBALL= INSTALL= - - TARBALL= INSTALL=yes - - TARBALL=archive INSTALL= - - TARBALL=archive INSTALL=yes - - TARBALL=export INSTALL= - - TARBALL=export INSTALL=yes + - TARBALL= INSTALL= + - TARBALL= INSTALL=yes + - TARBALL=archive INSTALL= +# - TARBALL=archive INSTALL=yes + - TARBALL=export INSTALL= + - TARBALL=export INSTALL=yes + - TARBALL=export-bats INSTALL= +# - TARBALL=export-bats INSTALL=yes # Extra conditions - TARBALL= INSTALL= MINIMAL_DEPS=yes # One full-scope test. This will finish last by a lot.