fix: mc fails on uploading large files to gcs (#25)

* fix: mc fails on uploading large files to gcs * chore: use awscli * fix: use proper default value for storage endpoint
wmde · Aug 16, 2023 · 0544201 · 0544201
1 parent 4dc571a
commit 0544201
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 14 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:bionic
 
-ARG MC_VERSION=RELEASE.2023-08-08T17-23-59Z
+ARG AWS_CLI_VERSION=2.13.9
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
@@ -15,23 +15,25 @@ RUN useradd -u 1234 -m notroot && \
     curl=7.58.0-2ubuntu3.24 \
     gnupg=2.2.4-1ubuntu1.6 \
     mydumper=0.9.1-5 \
-    mariadb-client=1:10.1.48-0ubuntu0.18.04.1 && \
-    curl -sSL "https://dl.min.io/client/mc/release/$TARGETOS-$TARGETARCH$TARGETVARIANT/archive/mc.$MC_VERSION" \
-      --create-dirs \
-      -o "$HOME/minio-binaries/mc" && \
-    chmod +x "$HOME/minio-binaries/mc" && \
-    mv "$HOME/minio-binaries/mc" /usr/bin/mc && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    mariadb-client=1:10.1.48-0ubuntu0.18.04.1 \
+    unzip=6.0-21ubuntu1.2 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    curl "https://awscli.amazonaws.com/awscli-exe-$TARGETOS-${TARGETARCH/amd64/x86_64}-$AWS_CLI_VERSION.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -rf ./aws awscliv2.zip
 
 USER notroot
+
 WORKDIR /app
-COPY src/ /app
+
+COPY --chown=notroot src/ /app
 
 ENV DB_PORT="3306" \
     DB_HOST="localhost" \
     DO_UPLOAD="1" \
     MYDUMPER_VERBOSE_LEVEL="1" \
-    STORAGE_ENDPOINT="storage.googleapis.com" \
+    STORAGE_ENDPOINT="https://storage.googleapis.com" \
     STORAGE_SIGNATURE_VERSION="S3v2" \
     REPLICATION_THRESHOLD="60" \
     SECONDARY_HOST="sql-mariadb-secondary.default.svc.cluster.local" \

diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ Variable                             | Default
 `DB_USER`                            | NONE                                                                | User for mariadb
 `DO_UPLOAD`                          | 1                                                                   | Flag for uploading to GCS_BUCKET_NAME or not 
 `STORAGE_BUCKET_NAME`                | NONE                                                                | Bucket name that uploading happens to
-`STORAGE_ENDPOINT`                   | storage.googleapis.com                                              | S3 compatible storage endpoint
+`STORAGE_ENDPOINT`                   | https://storage.googleapis.com                                      | S3 compatible storage endpoint
 `STORAGE_ACCESS_KEY`                 | NONE                                                                | Storage Access Key
 `STORAGE_SECRET_KEY`                 | NONE                                                                | Storage Secret Key
 `STORAGE_SIGNATURE_VERSION`          | S3v2                                                                | S3 signature version to use

diff --git a/src/entrypoint.sh b/src/entrypoint.sh
@@ -9,7 +9,6 @@ fi
 
 ./backup.sh
 
-## GCS bucket is mounted by chart
 # We can just move the artifacts
 if [ "$DO_UPLOAD" -eq "1" ]; then
     ./upload.sh

diff --git a/src/upload.sh b/src/upload.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
 set -e
-mc alias set remote "$STORAGE_ENDPOINT" "$STORAGE_ACCESS_KEY" "$STORAGE_SECRET_KEY" --api "$STORAGE_SIGNATURE_VERSION"
-mc cp --recursive /backups/output/ remote/"$STORAGE_BUCKET_NAME"
+AWS_ACCESS_KEY_ID=$STORAGE_ACCESS_KEY \
+    AWS_SECRET_ACCESS_KEY=$STORAGE_SECRET_KEY \
+    AWS_DEFAULT_REGION=us-east-1 \
+    aws s3 cp --recursive --endpoint "$STORAGE_ENDPOINT" /backups/output/ "s3://$STORAGE_BUCKET_NAME"