fix: mc fails on uploading large files to gcs

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:bionic
 
-ARG MC_VERSION=RELEASE.2023-08-08T17-23-59Z
+ARG GCLOUD_VERSION=442.0.0
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
@@ -15,17 +15,23 @@ RUN useradd -u 1234 -m notroot && \
     curl=7.58.0-2ubuntu3.24 \
     gnupg=2.2.4-1ubuntu1.6 \
     mydumper=0.9.1-5 \
+    gettext-base=0.19.8.1-6ubuntu0.3 \
     mariadb-client=1:10.1.48-0ubuntu0.18.04.1 && \
-    curl -sSL "https://dl.min.io/client/mc/release/$TARGETOS-$TARGETARCH$TARGETVARIANT/archive/mc.$MC_VERSION" \
-      --create-dirs \
-      -o "$HOME/minio-binaries/mc" && \
-    chmod +x "$HOME/minio-binaries/mc" && \
-    mv "$HOME/minio-binaries/mc" /usr/bin/mc && \
+    curl -sSL -O "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-$GCLOUD_VERSION-$TARGETOS-${TARGETARCH/amd64/x86_64}.tar.gz" && \
+    mkdir -p /usr/local/gcloud && \
+    tar -C /usr/local/gcloud -xvf "google-cloud-cli-$GCLOUD_VERSION-$TARGETOS-${TARGETARCH/amd64/x86_64}.tar.gz" && \
+    /usr/local/gcloud/google-cloud-sdk/install.sh && \
+    rm -rf "google-cloud-cli-$GCLOUD_VERSION-$TARGETOS--${TARGETARCH/amd64/x86_64}.tar.gz" && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
 USER notroot
+
+COPY --chown=notroot .boto.template /home/notroot/.boto.template
+
 WORKDIR /app
-COPY src/ /app
+COPY --chown=notroot src/ /app
 
 ENV DB_PORT="3306" \
     DB_HOST="localhost" \

ci/docker-compose.yml

@@ -59,7 +59,8 @@ services:
       - DO_UPLOAD=1
       - BACKUP_KEY=abc123
       - EXPECTED_FILES=metadata,my_wiki.page.sql,my_wiki.page-schema.sql
-      - STORAGE_ENDPOINT=http://storage:9000
+      - STORAGE_ENDPOINT=http://storage
+      - STORAGE_PORT=9000
       - STORAGE_ACCESS_KEY=test
       - STORAGE_SECRET_KEY=secret-key-value
       - STORAGE_BUCKET_NAME=backup

src/configure_gsutil.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -eu
+
+envsubst < ~/.boto.template > ~/.boto

src/entrypoint.sh

@@ -9,9 +9,9 @@ fi
 
 ./backup.sh
 
-## GCS bucket is mounted by chart
 # We can just move the artifacts
 if [ "$DO_UPLOAD" -eq "1" ]; then
+    ./configure_gsutil.sh
     ./upload.sh
 else
     echo "Skipping upload as DO_UPLOAD is not set..."

src/upload.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 
 set -e
-mc alias set remote "$STORAGE_ENDPOINT" "$STORAGE_ACCESS_KEY" "$STORAGE_SECRET_KEY" --api "$STORAGE_SIGNATURE_VERSION"
-mc cp --recursive /backups/output/ remote/"$STORAGE_BUCKET_NAME"
+gsutil cp -R /backups/output/ "s3://$STORAGE_BUCKET_NAME"