From 2868c13375786e17dfe7a2cc5a2f6cb9707cc5e4 Mon Sep 17 00:00:00 2001
From: Nick Hale <4175918+njhale@users.noreply.github.com>
Date: Thu, 9 Jan 2025 17:15:33 -0500
Subject: [PATCH] enhance: support enterprise docker builds

Add support for:
- Pulling multiple tool registries into the image at build-time
  via the `TOOL_REGISTY_REPOS` build arg
- Cloning private tool registry repos via the `GITHUB_TOKEN` Docker
  secret

e.g. Building the "enterprise" Obot image

```bash
export GITHUB_TOKEN=$(gh auth token)
docker build --build-arg TOOL_REGISTRY_REPOS='github.com/obot-platform/enterprise-tools,github.com/obot-platform/tools' \
  --secret id=GITHUB_TOKEN \
  -t obot-enterprise:latest .
```

Signed-off-by: Nick Hale <4175918+njhale@users.noreply.github.com>
---
 Dockerfile             |  12 ++---
 run.sh                 |  20 ++++----
 tools/package-tools.sh | 112 ++++++++++++++++++++++++++++++++++-------
 3 files changed, 109 insertions(+), 35 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b02498f78..b912ab13b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 FROM cgr.dev/chainguard/wolfi-base AS base
 
 RUN apk add --no-cache go make git npm pnpm
@@ -12,6 +13,7 @@ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
     make all
 
 FROM base AS tools
+ARG TOOL_REGISTRY_REPOS='github.com/obot-platform/tools'
 RUN apk add --no-cache curl python-3.13 py3.13-pip
 WORKDIR /app
 COPY . .
@@ -19,7 +21,8 @@ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/root/.cache/uv \
     --mount=type=cache,target=/root/go/pkg/mod \
-    UV_LINK_MODE=copy BIN_DIR=/bin make package-tools
+    --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
+    UV_LINK_MODE=copy BIN_DIR=/bin TOOL_REGISTRY_REPOS=$TOOL_REGISTRY_REPOS make package-tools
 
 FROM cgr.dev/chainguard/postgres:latest-dev AS build-pgvector
 RUN apk add build-base git postgresql-dev
@@ -42,6 +45,7 @@ COPY --from=build-pgvector /usr/share/postgresql17/extension/vector* /usr/share/
 
 RUN apk add --no-cache git python-3.13 py3.13-pip openssh-server npm bash tini procps libreoffice docker
 COPY --chmod=0755 /tools/package-chrome.sh /
+
 RUN /package-chrome.sh && rm /package-chrome.sh
 RUN sed -E 's/^#(PermitRootLogin)no/\1yes/' /etc/ssh/sshd_config -i
 RUN ssh-keygen -A
@@ -54,13 +58,9 @@ COPY --from=bin /app/bin/obot /bin/
 
 EXPOSE 22
 # libreoffice executables
-ENV PATH=/obot-tools/venv/bin:$PATH:/usr/lib/libreoffice/program
+ENV PATH=$PATH:/usr/lib/libreoffice/program
 ENV HOME=/data
 ENV XDG_CACHE_HOME=/data/cache
-ENV GPTSCRIPT_SYSTEM_TOOLS_DIR=/obot-tools/
-ENV OBOT_SERVER_WORKSPACE_TOOL=/obot-tools/workspace-provider
-ENV OBOT_SERVER_DATASETS_TOOL=/obot-tools/datasets
-ENV OBOT_SERVER_TOOL_REGISTRY=/obot-tools
 ENV OBOT_SERVER_ENCRYPTION_CONFIG_FILE=/encryption.yaml
 ENV BAAAH_THREADINESS=20
 ENV TERM=vt100
diff --git a/run.sh b/run.sh
index 664294f5c..b57d49e0d 100644
--- a/run.sh
+++ b/run.sh
@@ -14,6 +14,16 @@ check_postgres_active() {
   exit 1
 }
 
+source /obot-tools/.envrc.tools
+export PATH=$TOOLS_VENV_BIN:$PATH
+
+# double echo to remove trailing whitespace
+export OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
+"chrome": "$(echo $(/opt/google/chrome/chrome --version))"
+${OBOT_SERVER_VERSIONS}
+VERSIONS
+)"
+
 # Only enable sshd in Render. Remove sshd entirely once we have migrated out of Render.
 if [[ -v ENABLE_SSHD ]]; then
   mkdir -p /run/sshd
@@ -21,16 +31,6 @@ if [[ -v ENABLE_SSHD ]]; then
 fi
 
 mkdir -p /data/cache
-# This is YAML
-export OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
-"github.com/obot-platform/tools": "$(cd /obot-tools && git rev-parse HEAD)"
-"github.com/gptscript-ai/workspace-provider": "$(cd /obot-tools/workspace-provider && git rev-parse HEAD)"
-"github.com/gptscript-ai/datasets": "$(cd /obot-tools/datasets && git rev-parse HEAD)"
-"github.com/kubernetes-sigs/aws-encryption-provider": "$(cd /obot-tools/aws-encryption-provider && git rev-parse HEAD)"
-# double echo to remove trailing whitespace
-"chrome": "$(echo $(/opt/google/chrome/chrome --version))"
-VERSIONS
-)"
 
 if [ -z "$OBOT_SERVER_DSN" ]; then
   echo "OBOT_SERVER_DSN is not set. Starting PostgreSQL process..."
diff --git a/tools/package-tools.sh b/tools/package-tools.sh
index 7a6675164..a193a31eb 100755
--- a/tools/package-tools.sh
+++ b/tools/package-tools.sh
@@ -3,51 +3,117 @@ set -e -x -o pipefail
 
 BIN_DIR=${BIN_DIR:-./bin}
 
+# Check if TOOL_REGISTRY_REPOS is set and non-empty
+if [[ -z "${TOOL_REGISTRY_REPOS}" ]]; then
+    echo "Error: TOOL_REGISTRY_REPOS environment variable is not set or is empty."
+    exit 1
+fi
+
+if [[ -n "${GITHUB_TOKEN}" ]]; then
+    set +x
+    git config --global url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" 2>/dev/null
+    set -x
+fi
+
 cd $(dirname $0)/..
 
 if [ ! -e obot-tools ]; then
-    git clone --depth=1 https://github.com/obot-platform/tools obot-tools
+    mkdir obot-tools
 fi
+cd obot-tools
 
-./obot-tools/scripts/build.sh
-
-for pj in $(find obot-tools -name package.json | grep -v node_modules); do
-    if [ $(basename $(dirname $pj)) == common ]; then
-        continue
+# Convert TOOL_REGISTRY_REPOS into an array by replacing commas with spaces
+read -r -a TOOL_REPOS <<< "${TOOL_REGISTRY_REPOS//,/ }"
+REGISTRY_REMAP=()
+LOCAL_REGISTRIES=()
+OBOT_SERVER_VERSIONS=""
+
+# Iterate over the repositories
+for REPO in "${TOOL_REPOS[@]}"; do
+    # Extract the repo name (e.g., tools, enterprise-tools)
+    REPO_NAME=$(basename "${REPO}")
+    REPO_DIR="obot-tools/${REPO_NAME}"
+
+    # Clone the repository into the target directory
+    echo "Cloning ${REPO} into ${REPO_DIR}..."
+    if git clone --depth=1 "https://${REPO}" "${REPO_NAME}"; then
+        # Change to the repository directory
+        # Check if the build script exists and is executable
+        if [[ -x "./${REPO_NAME}/scripts/build.sh" ]]; then
+          (
+            echo "Running build script for ${REPO}..."
+            cd "${REPO_NAME}"
+            ./scripts/build.sh
+            echo "Build script for ${REPO} complete!"
+          )
+        else
+            echo "No build script found in ${REPO}"
+        fi
+
+        OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
+"${REPO}": "$(cd "${REPO_NAME}" && git rev-parse --short HEAD)"
+${OBOT_SERVER_VERSIONS}
+VERSIONS
+)"
+
+    else
+        echo "Failed to clone $REPO. Aborting..."
+        exit 1
     fi
-    (
-        cd $(dirname $pj)
-        echo Building $PWD
-        pnpm i
-    )
+
+    REGISTRY_REMAP+=("${REPO}=/${REPO_DIR}")
+    LOCAL_REGISTRIES+=("/${REPO_DIR}")
 done
 
+cd ..
+for pj in $(find obot-tools -name package.json | grep -v node_modules); do
+  if [ $(basename $(dirname $pj)) == common ]; then
+    continue
+  fi
+  (
+    cd $(dirname $pj)
+    echo Building $PWD
+    pnpm i
+  )
+done
 cd obot-tools
+
 if [ ! -e workspace-provider ]; then
     git clone --depth=1 https://github.com/gptscript-ai/workspace-provider
 fi
-
 cd workspace-provider
 go build -ldflags="-s -w" -o bin/gptscript-go-tool .
-
+REGISTRY_REMAP+=('github.com/gptscript-ai/workspace-provider=/obot-tools/workspace-provider')
+OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
+"github.com/gptscript-ai/workspace-provider": "$(git rev-parse --short HEAD)"
+${OBOT_SERVER_VERSIONS}
+VERSIONS
+)"
 cd ..
 
 if [ ! -e datasets ]; then
     git clone --depth=1 https://github.com/gptscript-ai/datasets
 fi
-
 cd datasets
 go build -ldflags="-s -w" -o bin/gptscript-go-tool .
-
+REGISTRY_REMAP+=('github.com/gptscript-ai/datasets=/obot-tools/datasets')
+OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
+"github.com/gptscript-ai/datasets": "$(git rev-parse --short HEAD)"
+${OBOT_SERVER_VERSIONS}
+VERSIONS
+)"
 cd ..
 
 if [ ! -e aws-encryption-provider ]; then
     git clone --depth=1 https://github.com/kubernetes-sigs/aws-encryption-provider
 fi
-
 cd aws-encryption-provider
-go build -o ${BIN_DIR}/aws-encryption-provider cmd/server/main.go
-
+go build -o "${BIN_DIR}/aws-encryption-provider" cmd/server/main.go
+OBOT_SERVER_VERSIONS="$(cat <<VERSIONS
+"github.com/kubernetes-sigs/aws-encryption-provider": "$(git rev-parse --short HEAD)"
+${OBOT_SERVER_VERSIONS}
+VERSIONS
+)"
 cd ../..
 
 if ! command -v uv; then
@@ -59,6 +125,14 @@ if [ ! -e obot-tools/venv ]; then
 fi
 
 source obot-tools/venv/bin/activate
-
 find obot-tools -name requirements.txt -exec cat {} \; -exec echo \; | sort -u > requirements.txt
 uv pip install -r requirements.txt
+
+cd obot-tools
+cat <<EOF > .envrc.tools
+export GPTSCRIPT_SYSTEM_TOOLS_DIR=/obot-tools/
+export GPTSCRIPT_TOOL_REMAP="$(IFS=','; echo "${REGISTRY_REMAP[*]}")"
+export OBOT_SERVER_TOOL_REGISTRIES="${TOOL_REGISTRY_REPOS}"
+export OBOT_SERVER_VERSIONS="${OBOT_SERVER_VERSIONS}"
+export TOOLS_VENV_BIN=/obot-tools/venv/bin
+EOF
\ No newline at end of file