rewrite embedded paths at install time

.github/workflows/main.yml

@@ -234,7 +234,7 @@ jobs:
           test -f ./lib/lark/lark.py
           make $ch_makej
           sudo make $ch_makej install
-          [[ $(bin/ch-image -v --dependencies) = 'lark path: /home/runner/work/charliecloud/charliecloud/lib/charliecloud/lark/__init__.py' ]]
+          [[ $(bin/ch-image -v --dependencies) = 'lark path: /home/runner/work/charliecloud/charliecloud/lib/lark/__init__.py' ]]
           [[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = 'lark path: /usr/local/lib/python3.10/dist-packages/lark/__init__.py' ]]
           make clean
           # no install, enable: build fails, untested
@@ -263,7 +263,15 @@ jobs:
           else
               LDD=ldd
           fi
-          ./configure --prefix=$ch_prefix/from-git
+          # Go wild with install directories to make sure it works (see #683).
+          # We do have a normal install below.
+          ./configure --prefix=$ch_prefix/from-git \
+                      --datarootdir=$ch_prefix/from-git_dataroot \
+                      --libexecdir=$ch_prefix/from-git_libexec \
+                      --libdir=$ch_prefix/from-git_lib \
+                      --mandir=$ch_prefix/from-git_man \
+                      --docdir=$ch_prefix/from-git_doc \
+                      --htmldir=$ch_prefix/from-git_html
           set -x
           fgrep 'documentation: yes' config.log
           [[ $CH_TEST_BUILDER = buildah* ]] && fgrep 'with Buildah: yes' config.log
@@ -299,9 +307,9 @@ jobs:
         run: |
           set -x
           [[ $(bin/ch-image gestalt python-path) = /usr/bin/python3 ]]
-          [[ $(bin/ch-image -v --dependencies) = "lark path: /home/runner/work/charliecloud/charliecloud/lib/charliecloud/lark/__init__.py" ]]
+          [[ $(bin/ch-image -v --dependencies) = "lark path: /home/runner/work/charliecloud/charliecloud/lib/lark/__init__.py" ]]
           [[ $($ch_prefix/from-git/bin/ch-image gestalt python-path) = /usr/bin/python3 ]]
-          [[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = "lark path: /var/tmp/from-git/lib/charliecloud/lark/__init__.py" ]]
+          [[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = "lark path: /var/tmp/from-git_lib/charliecloud/lark/__init__.py" ]]
 
       - name: late setup & validation, all
         run: |
@@ -344,7 +352,7 @@ jobs:
           mv charliecloud-*.tar.gz $ch_prefix
           cd $ch_prefix
           tar xf charliecloud-*.tar.gz
-          rm charliecloud-*.tar.gz  # else cd fails with "too many arguments"
+          rm charliecloud-*.tar.gz  # else cd fails with “too many arguments”
           cd charliecloud-*
           pwd
           # Configure and verify output.

.gitignore

@@ -19,7 +19,6 @@ a.out
 /doc/doctrees/
 /doc/html/
 /doc/man/*.[17]
-/lib/charliecloud
 /lib/contributors.bash
 /lib/lark
 /lib/lark-stubs

Makefile.am

@@ -5,3 +5,23 @@ SUBDIRS = lib bin doc examples misc packaging test
 EXTRA_DIST = .github/PERUSEME .github/workflows/main.yml
 
 EXTRA_DIST += LICENSE README.rst VERSION autogen.sh
+
+# Embedded paths are in the source code suitable for running from the source
+# directory (i.e., without install). When installing, those paths are often
+# wrong, so re-write them with the correct paths we got from configure. Note:
+# Some variables are in both Python and sh, so we use syntax valid for both;
+# others are just sh.
+install-exec-hook:
+	@echo '### re-writing embedded paths ###'
+	for i in $(DESTDIR)@bindir@/ch-convert \
+	         $(DESTDIR)@bindir@/ch-fromhost \
+	         $(DESTDIR)@bindir@/ch-image \
+	         $(DESTDIR)@bindir@/ch-run-oci \
+	         $(DESTDIR)@bindir@/ch-test \
+	         $(DESTDIR)@libdir@/charliecloud/base.sh; \
+	do \
+	  sed -Ei -e 's|^(ch_lib ?= ?).+/lib"?$$|\1"@libdir@/charliecloud"|' \
+	          -e 's|^(CHTEST_DIR=).+$$|\1@libexecdir@/charliecloud|' \
+	          -e 's|^(CHTEST_EXAMPLES_DIR=).+$$|\1@docdir@/examples|' \
+              $$i; \
+	done

bin/ch-convert

@@ -2,8 +2,8 @@
 
 ## preamble ##################################################################
 
-lib=$(cd "$(dirname "$0")" && pwd)/../lib/charliecloud
-. "${lib}/base.sh"
+ch_lib=$(cd "$(dirname "$0")" && pwd)/../lib
+. "${ch_lib}/base.sh"
 PATH=${ch_bin}:$PATH
 
 usage=$(cat <<EOF

bin/ch-fromhost

@@ -34,8 +34,8 @@
 #   3. Here string, e.g. 'while IFS= read -r FILE; do ... done <<< "$FILES"'.
 #      This is a bashism.
 
-lib=$(cd "$(dirname "$0")" && pwd)/../lib/charliecloud
-. "${lib}/base.sh"
+ch_lib=$(cd "$(dirname "$0")" && pwd)/../lib
+. "${ch_lib}/base.sh"
 
 set -e
 

bin/ch-image.py.in

@@ -5,8 +5,8 @@ import inspect
 import os.path
 import sys
 
-sys.path.insert(0, (  os.path.dirname(os.path.abspath(__file__))
-                    + "/../lib/charliecloud"))
+ch_lib = os.path.dirname(os.path.abspath(__file__)) + "/../lib"
+sys.path.insert(0, ch_lib)
 import build_cache as bu
 import charliecloud as ch
 import build

bin/ch-run-oci.py.in

@@ -11,8 +11,8 @@ import sys
 import time
 import types
 
-sys.path.insert(0, (  os.path.dirname(os.path.abspath(__file__))
-                    + "/../lib/charliecloud"))
+ch_lib = os.path.dirname(os.path.abspath(__file__)) + "/../lib"
+sys.path.insert(0, ch_lib)
 import charliecloud as ch
 import misc
 import filesystem as fs

bin/ch-test

@@ -31,7 +31,7 @@ warning () {
 
 ### Setup
 
-ch_lib=$(cd "$(dirname "$0")" && pwd)/../lib/charliecloud
+ch_lib=$(cd "$(dirname "$0")" && pwd)/../lib
 if [[ ! -f ${ch_lib}/base.sh ]]; then
     fatal "install or build problem: not found: ${ch_lib}/base.sh"
 fi
@@ -612,7 +612,7 @@ test_make_tag_from_path () {
     case $basicname in
         Build|Dockerfile)
             case $dir in
-                .|test|examples)  # dot is directory “test”
+                .|examples|charliecloud|test)  # dot is directory “test”
                     if [[ -z $extension ]]; then
                         fatal "can't compute tag: $1"
                     else
@@ -761,15 +761,13 @@ if [[ ! -d $TMP_ ]]; then
     chmod 700 "$TMP_"
 fi
 
-# Find test directories. (At one point we tried to unify the paths between the
-# two conditions using a deeper directory hierarchy and symlinks to . in the
-# source code to keep it from being even deeper, but this became too unruly.)
-if [[ -d ${ch_base}/share ]]; then
+# Find test directories. Note some of this gets rewritten at install time.
+CHTEST_DIR=${ch_base}/test
+CHTEST_EXAMPLES_DIR=${ch_base}/examples
+if [[ ! -f ${ch_base}/VERSION ]]; then
     # installed
     CHTEST_INSTALLED=yes
     CHTEST_GITWD=
-    CHTEST_DIR=${ch_base}/libexec/charliecloud/test
-    CHTEST_EXAMPLES_DIR=${ch_base}/share/doc/charliecloud/examples
 else
     # build dir
     CHTEST_INSTALLED=
@@ -778,8 +776,6 @@ else
     else
         CHTEST_GITWD=
     fi
-    CHTEST_DIR=${ch_base}/test
-    CHTEST_EXAMPLES_DIR=${ch_base}/examples
 fi
 export ch_base
 export CHTEST_INSTALLED

lib/Makefile.am

@@ -16,7 +16,6 @@ dist_mylib_DATA = base.sh \
                   pull.py \
                   push.py \
                   registry.py
-noinst_DATA = charliecloud
 mylib_DATA = contributors.bash \
              version.py \
              version.sh \
@@ -89,12 +88,7 @@ if ENABLE_LARK
 nobase_dist_mylib_DATA = $(LARK)
 endif
 
-CLEANFILES = $(mylib_DATA) $(noinst_DATA)
-
-# This symlink is so scripts can use "lib/charliecloud" whether they are
-# installed or not.
-charliecloud:
-	ln -s . charliecloud
+CLEANFILES = $(mylib_DATA)
 
 contributors.bash: ../README.rst
 	rm -f $@

lib/base.sh

@@ -5,8 +5,8 @@ ch_bin="$(cd "$(dirname "$0")" && pwd)"
 # shellcheck disable=SC2034
 ch_base=${ch_bin%/*}
 
-lib="${ch_bin}/../lib/charliecloud"
-. "${lib}/version.sh"
+ch_lib=${ch_bin}/../lib
+. "${ch_lib}/version.sh"
 
 
 # Verbosity level; works the same as the Python code.
@@ -54,7 +54,7 @@ exist_p () {
 parse_basic_arg () {
     case $1 in
         --_lib-path)  # undocumented
-            echo "$lib"
+            echo "$ch_lib"
             exit 0
             ;;
         --help)

packaging/fedora/charliecloud.spec

@@ -169,7 +169,7 @@ ln -s "${sphinxdir}/js"    %{buildroot}%{_pkgdocdir}/html/_static/js
 
 %files test
 %{_bindir}/ch-test
-%{_libexecdir}/%{name}/test
+%{_libexecdir}/%{name}
 %{_mandir}/man1/ch-test.1*
 
 %changelog

test/Makefile.am

@@ -1,4 +1,4 @@
-testdir = $(pkglibexecdir)/test
+testdir = $(pkglibexecdir)
 
 # These test files require no special handling.
 testfiles = \