Merge pull request #27 from Ultimaker/restore_helper

Restore helper

.flake8

@@ -0,0 +1,25 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# This file contains the flake8 rules used in the stardust projects.
+
+# To configure Flake8 as an external tool in PyCharm, create a new External Tool with the settings:
+#
+# Name: Flake8
+# Program: Check with 'which flake8'. For example: /home/dschiavini/.local/bin/flake8
+# Arguments: $ContentRoot$/$FileDirRelativeToProjectRoot$ --config=$ContentRoot$/lib/stardustCommons/.flake8 --count
+# Working directory: $ContentRoot$
+# Output filters: $FILE_PATH$:$LINE$:$COLUMN$:.*
+#
+# You can add a keyboard shortcut in the keymap settings. Select the folder you want to check (e.g. the root of the
+# project) before running the external tool.
+#
+# If you find a better way to configure the external tool please edit this file.
+
+[flake8]
+# E251 allows us to add extra spaces in constructors and method calls, i.e. method(value = result).
+# F401 is ignored because it gives false positives for # type comments
+# W503 forbids line breaks before binary operators, we use instead W504 to forbid breaks after the operators
+ignore = E251, F401, W503
+exclude = .git,__pycache__,node_modules,libCharon,venv
+max-complexity = 6
+max-line-length = 120
+inline-quotes = double

.gitignore

@@ -106,4 +106,4 @@ ENV/
 .idea
 
 # Google credentials
-google_credentials.json
+google-credentials.json

.pylintrc

@@ -0,0 +1,102 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# This file contains the Pylint rules used in the stardust projects.
+
+# To configure PyLint as an external tool in PyCharm, create a new External Tool with the settings:
+#
+# Name: PyLint
+# Program: Check with 'which pylint'. For example: /home/dschiavini/.local/bin/pylint
+# Arguments: $FileDirName$ --rcfile=lib/stardustCommons/.pylintrc --msg-template='{abspath}:{line}:{column}:({symbol}):{msg_id}:{msg}'
+# Working directory: $ContentRoot$
+# Output filters: $FILE_PATH$:$LINE$:$COLUMN$:.*
+#
+# You can add a keyboard shortcut in the keymap settings. To run Pylint to a project, select the module
+# you want to check (e.g. stardustAuthServerApi folder) before running the external tool.
+#
+# If you find a better way to configure the external tool please edit this file.
+
+[MASTER]
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=pylint_quotes
+
+# We expect double string quotes
+string-quote=double-avoid-escape
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Add files or directories to the blacklist. They should be base names, not paths.
+ignore=node_modules,tests
+
+[REFACTORING]
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=3
+
+[MESSAGES CONTROL]
+# C0326: No space allowed around keyword argument assignment
+# C0111: Missing module docstring (because we use a file per class, this would lead to duplicate docstrings)
+# C0411: Ignore import order because the rules are different than in PyCharm, so automatic imports break lots of builds
+# R0201: Method could be a function (no-self-use)
+# R0401: Cyclic imports (cyclic-import) are used for typing
+# R0801: Unfortunately the error is triggered for a lot of similar models (duplicate-code)
+# R1710: Either all return statements in a function should return an expression, or none of them should.
+# W0221: Parameters differ from overridden method (tornado http methods have a flexible number of parameters)
+# W0511: Ignore warnings generated for TODOs in the code
+disable=C0326,C0411,C0111,R0201,R0401,R0801,R1710,W0221,W0511
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=500
+
+good-names=os
+
+[BASIC]
+# allow modules and functions to use pascalCase
+module-rgx=[a-zA-Z0-9_]+$
+function-rgx=
+method-rgx=([a-z_][a-z0-9_]{2,30}|([a-z_][A-Za-z0-9]{2,30}))$
+
+[DESIGN]
+# Maximum number of arguments for function / method.
+max-args=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=8
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (R0903).
+# We set this to 0 because our models and fields do not have methods.
+min-public-methods=0
+
+[CLASSES]
+defining-attr-methods=__init__,__new__,setUp,initialize
+
+[TYPECHECK]
+ignored-classes=NotImplemented
+
+[VARIABLES]
+dummy-variables-rgx=_+[a-z0-9_]{2,30}

Dockerfile

@@ -2,31 +2,34 @@
 FROM python:3.6-stretch AS base
 WORKDIR /usr/src/app
 
-# install MongoDB tools
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
-RUN echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list
-RUN apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    mongodb-org-tools \
-    mongodb-org-shell
+# Install MongoDB tools
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 && \
+    echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mongodb-org-tools mongodb-org-shell
 
+# Install Python dependencies
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 
 # This is the container build that will run the "unit tests"
 FROM base AS tests
 WORKDIR /usr/src/app
 COPY requirements-testing.txt ./
-RUN pip install -r requirements-testing.txt
 ARG cache=1
 ARG KUBERNETES_SERVICE_HOST="localhost"
 ARG KUBERNETES_SERVICE_PORT=8081
-RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount
-RUN echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/token
-RUN echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+RUN pip install -r requirements-testing.txt && \
+    mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+    echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/token && \
+    echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 ADD . .
-RUN ENV_NAME=testing ASYNC_TEST_TIMEOUT=15 coverage run --source="mongoOperator" -m pytest
-RUN coverage report --skip-covered --show-missing  --fail-under=100
+RUN ENV_NAME=testing ASYNC_TEST_TIMEOUT=15 coverage run --source="mongoOperator" -m pytest -vvx && \
+    coverage report --skip-covered --show-missing  --fail-under=100
+
+# Linting
+RUN flake8 . --count
+RUN pylint mongoOperator
 
 # This is the container build statements that will create the container meant for deployment
 FROM base AS build

Dockerfile.local

@@ -0,0 +1,23 @@
+# Copyright (c) 2018 Ultimaker B.V.
+FROM python:3.6-stretch AS base
+WORKDIR /usr/src/app
+
+COPY requirements*.txt ./
+# install MongoDB tools
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 && \
+    echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mongodb-org-tools mongodb-org-shell
+
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements-testing.txt
+
+ARG cache=1
+ARG KUBERNETES_SERVICE_HOST="localhost"
+ARG KUBERNETES_SERVICE_PORT=8081
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+    echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/token && \
+    echo "unit-test" >> /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+ADD . .
+
+ENTRYPOINT ["python", "./main.py"]

README.md

@@ -75,6 +75,8 @@ The following options are available to use in the `spec` section of the `yaml` c
 | `mongodb.replicas` | - | The amount of MongoDB replicas that should be available in the replica set. Must be an uneven positive integer and minimum 3. |
 | * `backups.cron` | - | The cron on which to create a backup to cloud storage.
 | * `backups.gcs.bucket` | - | The GCS bucket to upload the backup to. |
+| `backups.gcs.restore_bucket` | - | The GCS bucket that contains the backup we wish to restore. If not specified, the value of backups.gcs.bucket is used. |
+| `backups.gcs.restore_from` | - | Filename of the backup in the bucket we wish to restore. If not specified, or set to 'latest', the last backup created is used. |
 | `backups.gcs.prefix` | backups/ | The file name prefix for the backup file. |
 
 > Please read https://docs.mongodb.com/manual/administration/production-notes/#allocate-sufficient-ram-and-cpu for details about why setting the WiredTiger cache size is important when you change the container memory limit from the default value.
@@ -94,7 +96,7 @@ minikube start
 
 Then you can run our test script to deploy the operator and execute some end-to-end tests.
 
-Note that this script assumes there is a file `google_credentials.json` in this directory that will be uploaded to Kubernetes as the secret for the backups.
+Note that this script assumes there is a file `google-credentials.json` in this directory that will be uploaded to Kubernetes as the secret for the backups.
 You will need to download this file from Google in order to run the script.
 
 ```bash

Settings.py

@@ -3,19 +3,18 @@
 # -*- coding: utf-8 -*-
 import os
 
-
 STRING_TO_BOOL_DICT = {"True", "true", "yes", "1"}
 
 
 class Settings:
     """
     Class responsible for keeping the application settings.
     """
-    
+
     # Custom resource (CRD) API config.
     CUSTOM_OBJECT_API_GROUP = "operators.ultimaker.com"
     CUSTOM_OBJECT_API_VERSION = "v1"
     CUSTOM_OBJECT_RESOURCE_PLURAL = "mongos"
-    
+
     # Kubernetes config.
     KUBERNETES_SERVICE_DEBUG = os.getenv("KUBERNETES_SERVICE_DEBUG") in STRING_TO_BOOL_DICT

build-and-deploy-local.sh

@@ -1,38 +1,56 @@
 #!/usr/bin/env bash
+set -eo pipefail
+
+EXAMPLE_FILE=${1:-examples/mongo-3-replicas.yaml}
+
+if ! [[ -e "google-credentials.json" ]]; then
+    echo "google-credentials.json file is missing, aborting."
+    exit -1
+fi
 
 # set the environment of the minikube docker
 eval $(minikube docker-env)
 
+readonly NAMESPACE="mongo-operator-cluster"
+readonly KUBECTL="kubectl --namespace=${NAMESPACE}"
+
 # build the docker image
 docker build --tag ultimaker/k8s-mongo-operator:local .
 
 # print out the Kubernetes client and server versions
-kubectl version
+${KUBECTL} version
+
+if ! kubectl get namespace ${NAMESPACE}; then
+    kubectl create namespace ${NAMESPACE}
+fi
 
 # remove the deployment, if needed, and apply the new one
-kubectl delete deployment mongo-operator 2>/dev/null
-kubectl apply --filename=kubernetes/operators/mongo-operator/service-account.yaml
-kubectl apply --filename=kubernetes/operators/mongo-operator/cluster-role.yaml
-kubectl apply --filename=kubernetes/operators/mongo-operator/cluster-role-binding.yaml
-kubectl apply --filename=kubernetes/operators/mongo-operator/deployment.yaml
+${KUBECTL} delete deployment mongo-operator 1>/dev/null
+${KUBECTL} apply --filename=kubernetes/operators/mongo-operator/service-account.yaml
+${KUBECTL} apply --filename=kubernetes/operators/mongo-operator/cluster-role.yaml
+${KUBECTL} apply --filename=kubernetes/operators/mongo-operator/cluster-role-binding.yaml
+${KUBECTL} apply --filename=kubernetes/operators/mongo-operator/deployment.yaml
 
 # show some details about the deployment
-kubectl describe deploy mongo-operator
+${KUBECTL} describe deploy mongo-operator
 
 # create a secret with the google account credentials
-kubectl create secret generic storage-serviceaccount --from-file=json=google_credentials.json
+if ${KUBECTL} get secret storage-serviceaccount 1>/dev/null; then
+    ${KUBECTL} delete secret storage-serviceaccount
+fi
+${KUBECTL} create secret generic storage-serviceaccount --from-file=json=google-credentials.json
 
 # wait for the pod to startup to retrieve its name
 sleep 10
-POD_NAME=$(kubectl get pods | grep -e "mongo-operator.*Running" | cut --fields=1 --delimiter=" ")
+POD_NAME=$(${KUBECTL} get pod -l app=mongo-operator -o jsonpath="{.items[0].metadata.name}")
 if [ -z $POD_NAME ]; then
     echo "The operator pod is not running!"
-    kubectl get pods
+    ${KUBECTL} get pods
     exit 1
 fi
 
 # apply the example file
-kubectl apply --filename=examples/mongo-3-replicas.yaml
+${KUBECTL} apply --filename=${EXAMPLE_FILE}
 
 # show the pod logs
-kubectl logs ${POD_NAME} --follow
+${KUBECTL} logs ${POD_NAME} --follow

examples/mongo-3-replicas-from-latest-backup.yaml

@@ -0,0 +1,25 @@
+apiVersion: "operators.ultimaker.com/v1"
+kind: Mongo
+metadata:
+  name: mongo-cluster
+  namespace: mongo-operator-cluster
+  labels:
+    app: mongo-cluster
+spec:
+  mongodb:
+    replicas: 3  # Must be between 3 and 50
+    cpu_limit: "200m"
+    memory_limit: "64Mi"
+  backups:
+    cron: "0 * * * *"  # every hour at 0 minutes
+    gcs:
+      bucket: "ultimaker-mongo-backups"
+      # Set restore_from to 'latest' to use the last backup created when initializing the replicaset.
+      restore_from: "latest"
+      # set restore_bucket if the file in restore_from is in another bucket.
+      # restore_bucket:
+      prefix: "test-backups"
+      serviceAccount:
+        secretKeyRef:
+          name: storage-serviceaccount
+          key: json

examples/mongo-3-replicas-from-specific-backup.yaml

@@ -0,0 +1,25 @@
+apiVersion: "operators.ultimaker.com/v1"
+kind: Mongo
+metadata:
+  name: mongo-cluster
+  namespace: mongo-operator-cluster
+  labels:
+    app: mongo-cluster
+spec:
+  mongodb:
+    replicas: 3  # Must be between 3 and 50
+    cpu_limit: "200m"
+    memory_limit: "64Mi"
+  backups:
+    cron: "0 * * * *"  # every hour at 0 minutes
+    gcs:
+      bucket: "ultimaker-mongo-backups"
+      # Set restore_from to 'latest' to use the last backup created when initializing the replicaset.
+      restore_from: mongodb-backup-default-mongo-cluster-2019-02-07_132931.archive.gz
+      # set restore_bucket if the file in restore_from is in another bucket.
+      # restore_bucket:
+      prefix: "test-backups"
+      serviceAccount:
+        secretKeyRef:
+          name: storage-serviceaccount
+          key: json

examples/mongo-3-replicas.yaml

@@ -2,7 +2,9 @@ apiVersion: "operators.ultimaker.com/v1"
 kind: Mongo
 metadata:
   name: mongo-cluster
-  namespace: default
+  namespace: mongo-operator-cluster
+  labels:
+    app: mongo-cluster
 spec:
   mongodb:
     replicas: 3  # Must be between 3 and 50

examples/mongo-5-replicas.yaml

@@ -2,7 +2,9 @@ apiVersion: "operators.ultimaker.com/v1"
 kind: Mongo
 metadata:
   name: mongo-cluster
-  namespace: default
+  namespace: mongo-operator-cluster
+  labels:
+    app: mongo-cluster
 spec:
   mongodb:
     replicas: 5  # Must be between 3 and 50