Move containers build to CI

.github/workflows/containers.yaml

@@ -0,0 +1,63 @@
+name: Docker Containers
+
+on:
+  push:
+    paths:
+      - build/containers/**
+
+jobs:
+  changed-containers:
+    runs-on: ubuntu-latest
+    outputs:
+      containers: ${{ steps.determine-containers.outputs.containers }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - id: determine-containers
+        name: Determine changed containers
+        run: |
+          containers=$( build/containers/build.sh --changed-containers-as-json)
+          echo "containers=${containers}" >> "$GITHUB_OUTPUT"
+
+  build-containers:
+    runs-on: ubuntu-latest
+    needs: changed-containers
+    strategy:
+      max-parallel: 4
+      fail-fast: true
+      matrix:
+        container: ${{ fromJSON(needs.changed-containers.outputs.containers) }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Build Docker image
+        run: build/containers/build.sh --containers ${{ matrix.container }}
+
+  push-containers:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    needs:
+      - changed-containers
+      - build-containers
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to DockerHub
+        if: ${{ env.DOCKER_USERNAME != '' }}
+        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
+      - name: Push Docker images
+        if: ${{ env.DOCKER_USERNAME != '' }}
+        run: |
+          containers=${{ needs.changed-containers.outputs.containers }}
+          containers=$( echo "${containers}" | sed 's/[][]//g' )
+          build/containers/build.sh --dry-run --skip-build --push --containers "${containers}"

build/containers/build.sh

@@ -1,19 +1,173 @@
 #!/usr/bin/env bash
+
+# Automatically exit on any failure
 set -e
 
-FOLDER=$1
-IMAGE=$2
+# Those ANSI codes are needed to print with colours
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+CYAN='\033[0;36m'
+GREY='\033[38;5;241m'
+RED='\033[0;31m'
+# WHITE='\033[0;37m'
+BOLD='\033[1m'
+RESET='\033[0m' # Reset to default color
+
+# ---------------------------------------------------------------------------------------
+# Global variables ----------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
+
+# Set CONTAINERS_DIR to the path where this script is located
+CONTAINERS_DIR=$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)
+
+# List of containers (By default all folders with git changes)
+CONTAINERS=""
+
+# Whether to run or not docker comands
+DRY_RUN="false"
+
+# Whether to skip the build or not
+SKIP_BUILD="no"
+
+# Whether to push the containers into the DockerHub or not
+PUSH="false"
+
+# What docker binary to use
+DOCKER_BIN="docker"
+
+# BuildKit’s advanced features and output capabilities for the Docker builds
+export DOCKER_BUILDKIT=1
+export BUILDKIT_PROGRESS=plain
+
+# ---------------------------------------------------------------------------------------
+# Function to determine which containers have changes -----------------------------------
+# ---------------------------------------------------------------------------------------
+
+changed_containers() {
+	(git diff --cached --name-only && git diff --name-only HEAD~1 HEAD) |
+		grep -E "^(build/containers|combination|core)" | xargs dirname | sort -u |
+		sed -E 's/(combination|core)/intogen-\1/' | sort |
+		sed -E 's|build/containers/?||' | grep -v '^$'
+}
+
+# ---------------------------------------------------------------------------------------
+# Function to format a list of containers for display -----------------------------------
+# ---------------------------------------------------------------------------------------
+
+format_display() {
+	echo "$1" | tr '\n' ',' | sed -E 's/,$//'
+}
+
+# ---------------------------------------------------------------------------------------
+# Function to format a list of containers into JSON -------------------------------------
+# ---------------------------------------------------------------------------------------
+
+format_json() {
+	echo "$1" | jq --raw-input . | jq --slurp --compact-output .
+}
+
+# ---------------------------------------------------------------------------------------
+# Function to build the Docker images ---------------------------------------------------
+# ---------------------------------------------------------------------------------------
+
+# This will build each container sequentially.
+# It assumes that every container folder will contain a Dockerfile.
+# The Docker images will be named after its folder and tagged with the git tag.
+# Some folders are just symlinks to upper level folders.
+build() {
+	container=$1
+	tag=$2
+
+	image="${container}:${tag}"
+	echo -e "${GREEN}${BOLD}Building Docker image for ${YELLOW}${image}${GREEN} ...${RESET}"
+	command="${DOCKER_BIN} build -t ${image} ${CONTAINERS_DIR}/${container}"
+	echo -e "${GREY}> ${command}${RESET}"
+	if [ "${DRY_RUN}" != "yes" ]; then
+        eval "${command}"
+    fi
+}
+
+# ---------------------------------------------------------------------------------------
+# Function to push the containers to the DockerHub --------------------------------------
+# ---------------------------------------------------------------------------------------
+
+push() {
+	container=$1
+	tag=$2
+
+	image="${container}:${tag}"
+	echo -e "${GREEN}${BOLD}Pushing Docker image for ${YELLOW}${image}${GREEN} ...${RESET}"
+	command="${DOCKER_BIN} push ${image}"
+	echo -e "${GREY}> ${command}${RESET}"
+	if [ "${DRY_RUN}" != "yes" ]; then
+        eval "${command}"
+    fi
+}
+
+# ---------------------------------------------------------------------------------------
+# Parse command line arguments ----------------------------------------------------------
+# ---------------------------------------------------------------------------------------
+
+while [[ $# -gt 0 ]]; do
+	case $1 in
+	--changed-containers-as-json)
+		format_json "$(changed_containers)"
+		exit 0
+		;;
+	--containers)
+        CONTAINERS="${2//,/$'\n'}"
+		shift
+		shift
+		;;
+	-n | --dry-run)
+		DRY_RUN="yes"
+		shift
+		;;
+	--skip-build)
+		SKIP_BUILD="yes"
+		shift
+		;;
+	--push)
+		PUSH="yes"
+		shift
+		;;
+	--podman)
+		DOCKER_BIN="podman"
+		shift
+		;;
+	*)
+		echo -e "${RED}${BOLD}Unknown option: ${YELLOW}$1${RESET}" >&2
+		exit 1
+		;;
+	esac
+done
+
+# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
+
+if [ -z "${CONTAINERS}" ]; then
+	CONTAINERS=$(changed_containers)
+fi
+
+CONTAINERS_FMT=$(format_display "$CONTAINERS")
+
+# Set TAG to either the current git tag name or else the git commit short SHA
+TAG=$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short HEAD)
 
-tmpdir=`mktemp -d`
+echo -e "${CYAN}${BOLD}Containers: ${YELLOW}${CONTAINERS_FMT}${CYAN}${RESET}"
 
-## if you have root permission you can uncomment the following:
+if [ "${SKIP_BUILD}" == "no" ]; then
+	for container in ${CONTAINERS}; do
+		build "${container}" "${TAG}"
+	done
 
-# (cd ${FOLDER} && sudo singularity build ${tmpdir}/$(basename ${IMAGE}) Singularity)
-# mv ${tmpdir}/$(basename ${IMAGE}) ${IMAGE}
-# sudo chown ${USER}: ${IMAGE}
+	echo -e "${GREEN}${BOLD}All images built successfully: ${YELLOW}${CONTAINERS_FMT}${CYAN}${RESET}"
+fi
 
-## if you DO NOT have root permission then you could use the --fakeroot flag as long as your user is specified in the /etc/subuid and /etc/subgid files. See https://docs.sylabs.io/guides/3.6/admin-guide/user_namespace.html#user-namespaces-fakeroot for more info. 
+if [ "${PUSH}" == "yes" ]; then
+	for container in ${CONTAINERS}; do
+		push "${container}" "${TAG}"
+	done
 
-(cd ${FOLDER} && singularity build --fakeroot ${tmpdir}/$(basename ${IMAGE}) Singularity)
-mv ${tmpdir}/$(basename ${IMAGE}) ${IMAGE}
-chown ${USER}: ${IMAGE}
+	echo -e "${GREEN}${BOLD}All images pushed successfully: ${YELLOW}${CONTAINERS_FMT}${CYAN}${RESET}"
+fi

build/containers/cbase/Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:buster-slim
+
+ENV LC_ALL=C.UTF-8
+
+RUN apt-get update && \
+    apt-get install -y python-scipy python-statsmodels python-mpmath && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p cbase
+COPY cbase.py cbase/
+RUN chmod -R a+r cbase
+
+ENTRYPOINT ["/usr/bin/python", "/cbase/cbase.py"]