feat(s3-client): add option to rotate dumps (#1052)

s3-client/Dockerfile

@@ -37,6 +37,8 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 USER 1001
 WORKDIR /home/ubuntu
 
+COPY entrypoint.sh .
 COPY dump-upload.sh .
+COPY rotate-dumps.sh .
 
-CMD ["./dump-upload.sh"]
+CMD ["./entrypoint.sh"]

s3-client/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+# check mandatory environment variables
+MANDATORY_VARS="AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_ENDPOINT_URL DESTINATION_PATH"
+for VAR in $MANDATORY_VARS; do
+  if [[ -z "${!VAR}" ]]; then
+    echo "${VAR} environment variable is empty"
+    exit 1
+  fi
+done
+
+./dump-upload.sh
+
+if ! [[ -z $RETENTION_DAYS ]]; then
+  ./rotate-dumps.sh
+fi

s3-client/rotate-dumps.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+aws s3 ls $DESTINATION_PATH/ | grep " DIR " -v | while read -r line;
+  do
+    createDate=`echo $line | awk {'print $1" "$2'}`
+    createDate=`date -d "$createDate" +%s`
+    olderThan=`date -d "-$RETENTION_DAYS days" +%s`
+
+    if [[ $createDate -lt $olderThan ]]
+      then 
+        fileName=`echo $line | awk {'print $4'}`
+        if [[ $fileName != "" ]]
+          then
+            echo "Deleting $DESTINATION_PATH/$fileName"
+            aws s3 rm "$DESTINATION_PATH/$fileName" 
+        fi
+    fi
+  done;

s3-client/tests/container-structure-test.yml

@@ -13,3 +13,9 @@ fileExistenceTests:
   - name: "dump script"
     path: "/home/ubuntu/dump-upload.sh"
     isExecutableBy: "any"
+  - name: "rotate dumps script"
+    path: "/home/ubuntu/rotate-dumps.sh"
+    isExecutableBy: "any"
+  - name: "entrypoint sript"
+    path: "/home/ubuntu/entrypoint.sh"
+    isExecutableBy: "any"