docker: data dir & user permissions, install ca-certificates, better …

…data dir location, more. (#58)

Improvements to the docker container and associated utilities.

.github/workflows/goreleaser.yml

@@ -30,11 +30,17 @@ jobs:
       - name: Setup Docker Buildx (docker multi-arch dependency)
         uses: docker/setup-buildx-action@v2
 
+      - name: dockerhub-login
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - uses: goreleaser/goreleaser-action@v4
         with:
           distribution: goreleaser
           version: latest
-          args: release --clean --skip-docker
+          args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.goreleaser.yaml

@@ -42,6 +42,8 @@ dockers:
     - --label=org.opencontainers.image.created={{ .Date }}
     - --label=org.opencontainers.image.revision={{ .FullCommit }}
     - --label=org.opencontainers.image.licenses=MIT
+    extra_files:
+    - docker/docker-entrypoint.sh
   - use: buildx
     goos: linux
     goarch: arm64
@@ -55,6 +57,8 @@ dockers:
     - --label=org.opencontainers.image.created={{ .Date }}
     - --label=org.opencontainers.image.revision={{ .FullCommit }}
     - --label=org.opencontainers.image.licenses=MIT
+    extra_files:
+    - docker/docker-entrypoint.sh
 
 # automatically select amd64/arm64 when requesting "algorand/conduit"
 docker_manifests:

Dockerfile

@@ -1,14 +1,27 @@
-# This dockerfile is used by goreleaser
+# Build this Dockerfile with goreleaser.
+# The binary must be present at /conduit
 FROM debian:bullseye-slim
 
-RUN useradd conduit
-RUN mkdir -p /conduit/data && \
-    chown -R conduit.conduit /conduit
+# Hard code UID/GID to 999 for consistency in advanced deployments.
+# Install ca-certificates to enable using infra providers.
+# Install gosu for fancy data directory management.
+RUN groupadd --gid=999 --system algorand && \
+    useradd --uid=999 --no-log-init --create-home --system --gid algorand algorand && \
+    mkdir -p /data && \
+    chown -R algorand.algorand /data && \
+    apt-get update && \
+    apt-get install -y gosu ca-certificates && \
+    update-ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# binary is passed into the build
-COPY conduit /conduit/conduit
+COPY conduit /usr/local/bin/conduit
+COPY docker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-USER conduit
-WORKDIR /conduit
-ENTRYPOINT ["./conduit"]
-CMD ["-d", "data"]
+ENV CONDUIT_DATA_DIR /data
+WORKDIR /data
+# Note: docker-entrypoint.sh calls 'conduit'. Similar entrypoint scripts
+# accept the binary as the first argument in order to surface a suite of
+# tools (i.e. algod, goal, algocfg, ...). Maybe this will change in the
+# future, but for now this approach seemed simpler.
+ENTRYPOINT ["docker-entrypoint.sh"]

docker/docker-entrypoint.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+# To allow mounting the data directory we need to change permissions
+# to our algorand user. The script is initially run as the root user
+# in order to change permissions; afterwards, the script is re-launched
+# as the algorand user.
+if [ "$(id -u)" = '0' ]; then
+  chown -R algorand:algorand $CONDUIT_DATA_DIR
+  exec gosu algorand "$0" "$@"
+fi
+
+# always run the conduit command
+exec conduit "$@"

docs/Docker.md

@@ -1,5 +1,3 @@
-**This container is a work in progress and not yet deployed to docker hub.**
-
 # Docker Image
 
 Algorand's Conduit data pipeline packaged for docker.
@@ -42,8 +40,25 @@ docker run algorand/conduit init --importer algod --processors filter_processor
 
 ## Run with conduit.yml
 
-With `conduit.yml` in your current working directory,
-launch the container:
+With `conduit.yml` in your current working directory, it can be mounted directly to `/data/conduit.yml`. This is good for testing and some deployments which override the starting round. For a more complete deployment see the next section which explains how to mount the entire data directory.
+
+Mount `conduit.yml` with the following command:
+```
+docker run -it -v $(pwd)/conduit.yml:/data/conduit.yml algorand/conduit
 ```
-docker run -it -v $(pwd)/conduit.yml:/conduit/data/conduit.yml algorand/conduit
+
+# Mounting the Data Directory
+
+For production deployments, you should consider mounting the entire data directory. This way you can persist state across images during an upgrade, or for backups. The data directory is located at `/data`. When mounting a data directory, it must contain the `conduit.yml` file.
+
 ```
+docker run -it -v $(pwd)/local_data_dir:/data algorand/conduit
+```
+
+## Volume Permissions
+
+The container executes in the context of the `algorand` user with UID=999 and GID=999 which is handled differently depending on your operating system or deployment platform. During startup the container temporarily runs as root in order to modify the permissions of `/data`. It then changes to the `algorand` user. This can sometimes cause problems, for example if your deployment platform doesn't allow containers to run as the root user.
+
+### Use specific UID and GID
+
+If you do not want the container to start as the root user you can specify a UID and GID. On the host system, ensure the directory being mounted uses UID=999 and GID=999. If the directory already has these permissions you may override the default user with `-u 999:999`.

go.sum

@@ -84,16 +84,13 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/algorand/avm-abi v0.1.1/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g=
 github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k=
 github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g=
-github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230228201805-5b8c99b1412c h1:KAX6gb3+DLCTBcVhjDtuhcdbCeKnwIYKdj5Dv2JA/nI=
 github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230228201805-5b8c99b1412c/go.mod h1:Nt3EHpP8AznLs0/EFfhr0/xsVf5ucnvjNeRygGgbUzM=
 github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230324200319-055c8d2b174a h1:fv15GJlyepaaP517PeiJuPX0Q1Wmr17T8uZzevep/TU=
 github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230324200319-055c8d2b174a/go.mod h1:Nt3EHpP8AznLs0/EFfhr0/xsVf5ucnvjNeRygGgbUzM=
 github.com/algorand/go-codec v1.1.8 h1:XDSreeeZY8gMst6Edz4RBkl08/DGMJOeHYkoXL2B7wI=
 github.com/algorand/go-codec v1.1.8/go.mod h1:XhzVs6VVyWMLu6cApb9/192gBjGRVGm5cX5j203Heg4=
 github.com/algorand/go-codec/codec v1.1.8 h1:lsFuhcOH2LiEhpBH3BVUUkdevVmwCRyvb7FCAAPeY6U=
 github.com/algorand/go-codec/codec v1.1.8/go.mod h1:tQ3zAJ6ijTps6V+wp8KsGDnPC2uhHVC7ANyrtkIY0bA=
-github.com/algorand/indexer v0.0.0-20230306212826-146c4d38c5b4 h1:BLzw/1gSbntKblR4ywXdSSxTM/GeKhdkchXNtKUUnzs=
-github.com/algorand/indexer v0.0.0-20230306212826-146c4d38c5b4/go.mod h1:ULZ8Qt539rs+FNkSYdoe9HuZ/z1cRAFsWCysylz0nDg=
 github.com/algorand/indexer v0.0.0-20230315150109-cf0074cfd4ed h1:aZ5FURJNLUmyayj10ahbVuPJtFQ6YBdp0mP3zJz7yyY=
 github.com/algorand/indexer v0.0.0-20230315150109-cf0074cfd4ed/go.mod h1:ULZ8Qt539rs+FNkSYdoe9HuZ/z1cRAFsWCysylz0nDg=
 github.com/algorand/oapi-codegen v1.12.0-algorand.0 h1:W9PvED+wAJc+9EeXPONnA+0zE9UhynEqoDs4OgAxKhk=