feat: upload file using an s3 client (#24)

* feat: upload file using an s3 client

* test: update integration tests to spin up s3 server

* docs: add key values

* chore: lock mc version to be downloaded

* fix: copy directory contents instead of directory

.github/workflows/docker.test.yml

@@ -19,7 +19,7 @@ jobs:
         name: Start test container
         run: |
           cd ci
-          docker-compose up -d wikibase mysql
+          docker-compose up -d wikibase mysql storage
           cd -
 
       - name: Wait for server to accept connections
@@ -34,16 +34,12 @@ jobs:
       -
         name: Run backup and show some files
         run: |
-          #create writable dir for non-root
-          mkdir -p ci/backup-outputs
-          chmod 777 ci/backup-outputs
-
           # run a backup
           cd ci
           docker-compose up wbaas_backup
 
           # decompress and list some files
-          FILE=$(ls backup-outputs/mydumper-backup-*)
+          FILE=$(ls backup-outputs/backup/mydumper-backup-*)
           cd ..
           BACKUP_KEY=abc123 ./src/decompress_archive.sh ci/$FILE /tmp/output
           ls /tmp/output/

Dockerfile

@@ -1,42 +1,40 @@
 FROM ubuntu:bionic
 
-ENV GCSFUSE_REPO gcsfuse-bionic
+ARG MC_VERSION=RELEASE.2023-08-08T17-23-59Z
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN useradd -u 1234 notroot && \
+RUN useradd -u 1234 -m notroot && \
     mkdir -p /backups/tmp && \
     mkdir -p /backups/output && \
-    mkdir -p /mnt/backup-bucket && \
     chown notroot /backups -R && \
-    chown notroot /mnt/backup-bucket && \
     apt-get update && apt-get install --yes --no-install-recommends \
     ca-certificates=20230311ubuntu0.18.04.1 \
     curl=7.58.0-2ubuntu3.24 \
     gnupg=2.2.4-1ubuntu1.6 \
     mydumper=0.9.1-5 \
-    mariadb-client=1:10.1.48-0ubuntu0.18.04.1 \
-  && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" \
-    | tee /etc/apt/sources.list.d/gcsfuse.list \
-  && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
-  && apt-get update \
-  && apt-get install --yes gcsfuse=0.40.0 --no-install-recommends \
-  && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    mariadb-client=1:10.1.48-0ubuntu0.18.04.1 && \
+    curl -sSL "https://dl.min.io/client/mc/release/$TARGETOS-$TARGETARCH$TARGETVARIANT/archive/mc.$MC_VERSION" \
+      --create-dirs \
+      -o "$HOME/minio-binaries/mc" && \
+    chmod +x "$HOME/minio-binaries/mc" && \
+    mv "$HOME/minio-binaries/mc" /usr/bin/mc && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 USER notroot
 WORKDIR /app
 COPY src/ /app
 
-ENV DB_PORT=3306 \
+ENV DB_PORT="3306" \
     DB_HOST="localhost" \
-    DB_PASSWORD="" \
-    DB_USER="" \
     DO_UPLOAD="1" \
-    GCS_BUCKET_NAME="" \
-    BACKUP_KEY="" \
     MYDUMPER_VERBOSE_LEVEL="1" \
-    EXPECTED_FILES="" \
-    REPLICATION_THRESHOLD=60 \
-    SECONDARY_HOST=sql-mariadb-secondary.default.svc.cluster.local \
+    STORAGE_ENDPOINT="storage.googleapis.com" \
+    STORAGE_SIGNATURE_VERSION="S3v2" \
+    REPLICATION_THRESHOLD="60" \
+    SECONDARY_HOST="sql-mariadb-secondary.default.svc.cluster.local" \
     DO_CHECK_SECONDARY="1"
 
 ENTRYPOINT [ "/app/entrypoint.sh" ]

README.md

@@ -9,7 +9,11 @@ Variable                             | Default
 `DB_PASSWORD`                        | NONE                                                                | Password of mariadb DB_USER
 `DB_USER`                            | NONE                                                                | User for mariadb
 `DO_UPLOAD`                          | 1                                                                   | Flag for uploading to GCS_BUCKET_NAME or not 
-`GCS_BUCKET_NAME`                    | NONE                                                                | Bucket name that uploading happens to
+`STORAGE_BUCKET_NAME`                | NONE                                                                | Bucket name that uploading happens to
+`STORAGE_ENDPOINT`                   | storage.googleapis.com                                              | S3 compatible storage endpoint
+`STORAGE_ACCESS_KEY`                 | NONE                                                                | Storage Access Key
+`STORAGE_SECRET_KEY`                 | NONE                                                                | Storage Secret Key
+`STORAGE_SIGNATURE_VERSION`          | S3v2                                                                | S3 signature version to use
 `BACKUP_KEY`                         | NONE                                                                | Key used for openssl encryption and decryption
 `MYDUMPER_VERBOSE_LEVEL`             | 1                                                                   | mydumper verbosity level ( 0 = silent, 1 = errors, 2 = warnings, 3 = info)
 `EXPECTED_FILES`                     | see [validate_expected_files.sh](src/validate_expected_files.sh)    | Files to expect after backup is taken.

ci/.env

@@ -48,3 +48,5 @@ QUICKSTATEMENTS_PORT=8840
 ## ElasticSearch
 MW_ELASTIC_HOST=elasticsearch.svc
 MW_ELASTIC_PORT=9200
+
+MINIO_IMAGE_NAME=minio/minio:RELEASE.2020-08-04T23-10-51Z

ci/docker-compose.yml

@@ -59,9 +59,22 @@ services:
       - DO_UPLOAD=1
       - BACKUP_KEY=abc123
       - EXPECTED_FILES=metadata,my_wiki.page.sql,my_wiki.page-schema.sql
+      - STORAGE_ENDPOINT=http://storage:9000
+      - STORAGE_ACCESS_KEY=test
+      - STORAGE_SECRET_KEY=secret-key-value
+      - STORAGE_BUCKET_NAME=backup
       - DO_CHECK_SECONDARY=0
+
+  storage:
+    image: "${MINIO_IMAGE_NAME}"
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: secret-key-value
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - ./backup-outputs:/mnt/backup-bucket/:rw
+      - ./backup-outputs:/data
 
 volumes:
-  mediawiki-mysql-data:
+  mediawiki-mysql-data:

src/entrypoint.sh

@@ -12,9 +12,9 @@ fi
 ## GCS bucket is mounted by chart
 # We can just move the artifacts
 if [ "$DO_UPLOAD" -eq "1" ]; then
-    mv /backups/output/* /mnt/backup-bucket/
+    ./upload.sh
 else
-    echo "Skip uploading..."
+    echo "Skipping upload as DO_UPLOAD is not set..."
 fi
 
 TIMESTAMP=$(date '+%Y-%m-%d_%H%M%S')

src/upload.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+mc alias set remote "$STORAGE_ENDPOINT" "$STORAGE_ACCESS_KEY" "$STORAGE_SECRET_KEY" --api "$STORAGE_SIGNATURE_VERSION"
+mc cp --recursive /backups/output/ remote/"$STORAGE_BUCKET_NAME"