test: update integration tests to spin up s3 server

.github/workflows/docker.test.yml

@@ -19,7 +19,7 @@ jobs:
         name: Start test container
         run: |
           cd ci
-          docker-compose up -d wikibase mysql
+          docker-compose up -d wikibase mysql storage
           cd -
 
       - name: Wait for server to accept connections
@@ -34,16 +34,13 @@ jobs:
       -
         name: Run backup and show some files
         run: |
-          #create writable dir for non-root
-          mkdir -p ci/backup-outputs
-          chmod 777 ci/backup-outputs
-
           # run a backup
           cd ci
           docker-compose up wbaas_backup
 
           # decompress and list some files
-          FILE=$(ls backup-outputs/mydumper-backup-*)
+          tree backup-outputs
+          FILE=$(ls backup-outputs/backup/output/mydumper-backup-*)
           cd ..
           BACKUP_KEY=abc123 ./src/decompress_archive.sh ci/$FILE /tmp/output
           ls /tmp/output/

Dockerfile

@@ -13,9 +13,9 @@ RUN useradd -u 1234 -m notroot && \
     mariadb-client=1:10.1.48-0ubuntu0.18.04.1 && \
     curl https://dl.min.io/client/mc/release/linux-amd64/mc \
       --create-dirs \
-      -o $HOME/minio-binaries/mc && \
-    chmod +x $HOME/minio-binaries/mc && \
-    mv $HOME/minio-binaries/mc /usr/bin/mc && \
+      -o "$HOME/minio-binaries/mc" && \
+    chmod +x "$HOME/minio-binaries/mc" && \
+    mv "$HOME/minio-binaries/mc" /usr/bin/mc && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 USER notroot

ci/.env

@@ -48,3 +48,5 @@ QUICKSTATEMENTS_PORT=8840
 ## ElasticSearch
 MW_ELASTIC_HOST=elasticsearch.svc
 MW_ELASTIC_PORT=9200
+
+MINIO_IMAGE_NAME=minio/minio:RELEASE.2020-08-04T23-10-51Z

ci/docker-compose.yml

@@ -59,9 +59,22 @@ services:
       - DO_UPLOAD=1
       - BACKUP_KEY=abc123
       - EXPECTED_FILES=metadata,my_wiki.page.sql,my_wiki.page-schema.sql
+      - STORAGE_ENDPOINT=http://storage:9000
+      - STORAGE_ACCESS_KEY=test
+      - STORAGE_SECRET_KEY=secret-key-value
+      - STORAGE_BUCKET_NAME=backup
       - DO_CHECK_SECONDARY=0
+
+  storage:
+    image: "${MINIO_IMAGE_NAME}"
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: secret-key-value
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - ./backup-outputs:/mnt/backup-bucket/:rw
+      - ./backup-outputs:/data
 
 volumes:
-  mediawiki-mysql-data:
+  mediawiki-mysql-data:

src/entrypoint.sh

@@ -12,7 +12,7 @@ fi
 ## GCS bucket is mounted by chart
 # We can just move the artifacts
 if [ "$DO_UPLOAD" -eq "1" ]; then
-    ./copy.sh
+    ./upload.sh
 else
     echo "Skipping upload as DO_UPLOAD is not set..."
 fi

src/upload.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
 set -e
-mc config host add "$STORAGE_ENDPOINT" "$STORAGE_HMAC_KEY" "$STORAGE_HMAC_SECRET" "$STORAGE_SIGNATURE_VERSION"
-mc config add alias remote "$STORAGE_ENDPOINT"
-mc cp /backups/output remote/"$STORAGE_BUCKET_NAME"
+mc alias set remote "$STORAGE_ENDPOINT" "$STORAGE_ACCESS_KEY" "$STORAGE_SECRET_KEY" --api "$STORAGE_SIGNATURE_VERSION"
+mc cp --recursive /backups/output remote/"$STORAGE_BUCKET_NAME"