Merge branch 'master' into master-github

.gitignore

@@ -12,3 +12,4 @@ checkstyle.txt
 .env
 .direnv
 .envrc
+.elasticbeanstalk/

.gitlab-ci.yml

@@ -25,7 +25,7 @@ deploydevelop:
         - develop
 
 deployawsstaging:
-    image: twstuart/elasticbeanstalk-pipenv
+    image: bullettrain/elasticbeanstalk-pipenv
     stage: deploy-aws
     script:
         - export AWS_ACCESS_KEY_ID=$AWS_STAGING_ACCESS_KEY_ID
@@ -45,7 +45,7 @@ deployawsstaging:
         - staging
 
 deployawsmaster:
-    image: twstuart/elasticbeanstalk-pipenv
+    image: bullettrain/elasticbeanstalk-pipenv
     stage: deploy-aws
     script:
         - export DATABASE_URL=$DATABASE_URL_PRODUCTION

Pipfile

@@ -16,6 +16,7 @@ pylint = "*"
 "autopep8" = "*"
 pytest = "*"
 pytest-django = "*"
+django-test-migrations = "*"
 
 [packages]
 appdirs = "*"
@@ -33,9 +34,7 @@ sendgrid-django = "*"
 psycopg2-binary = "*"
 coreapi = "*"
 Django = "<3.0"
-numpy = "*"
 django-simple-history = "*"
-twisted = {version = "*",extras = ["tls"]}
 django-debug-toolbar = "*"
 google-api-python-client = "*"
 "oauth2client" = "*"
@@ -46,8 +45,8 @@ chargebee = "*"
 python-http-client = "<3.2.0" # 3.2.0 is the latest but throws an error on installation saying that it's not found
 django-health-check = "*"
 django-storages = "*"
-boto3 = "*"
 django-environ = "*"
 django-trench = "*"
 djoser = "*"
 influxdb-client = "*"
+django-ordered-model = "*"

readme.md

@@ -120,7 +120,6 @@ The application relies on the following environment variables to run:
 * `INFLUXDB_URL`: The URL for your InfluxDB database
 * `INFLUXDB_ORG`: The organisation string for your InfluxDB API call.
 * `GA_TABLE_ID`: GA table ID (view) to query when looking for organisation usage
-* `USE_S3_STORAGE`: 'True' to store static files in s3
 * `AWS_STORAGE_BUCKET_NAME`: bucket name to store static files. Required if `USE_S3_STORAGE' is true.
 * `AWS_S3_REGION_NAME`: region name of the static files bucket. Defaults to eu-west-2.
 * `ALLOWED_ADMIN_IP_ADDRESSES`: restrict access to the django admin console to a comma separated list of IP addresses (e.g. `127.0.0.1,127.0.0.2`) 

src/api/serializers.py

@@ -0,0 +1,5 @@
+from rest_framework import serializers
+
+
+class ErrorSerializer(serializers.Serializer):
+    message = serializers.CharField()

src/app/middleware.py

@@ -1,6 +1,10 @@
 from django.conf import settings
 from django.core.exceptions import PermissionDenied
 
+from util.logging import get_logger
+
+logger = get_logger(__name__)
+
 
 class AdminWhitelistMiddleware:
     def __init__(self, get_response):
@@ -12,6 +16,7 @@ def __call__(self, request):
             ip = x_forwarded_for.split(',')[0] if x_forwarded_for else request.META.get('REMOTE_ADDR')
             if settings.ALLOWED_ADMIN_IP_ADDRESSES and ip not in settings.ALLOWED_ADMIN_IP_ADDRESSES:
                 # IP address not allowed!
+                logger.info('Denying access to admin for ip address %s' % ip)
                 raise PermissionDenied()
 
         return self.get_response(request)

src/app/settings/common.py

@@ -9,7 +9,6 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/1.9/ref/settings/
 """
-import logging
 import os
 import warnings
 from importlib import reload
@@ -29,6 +28,8 @@
 PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
 ENV = env('ENVIRONMENT', default='local')
+if ENV not in ('local', 'dev', 'staging', 'production'):
+    warnings.warn('ENVIRONMENT env variable must be one of local, dev, staging or production')
 
 if 'DJANGO_SECRET_KEY' not in os.environ:
     secret_key_gen()
@@ -106,6 +107,9 @@
     # health check plugins
     'health_check',
     'health_check.db',
+
+    # Used for ordering models (e.g. FeatureSegment)
+    'ordered_model',
 ]
 
 if GOOGLE_ANALYTICS_KEY or INFLUXDB_TOKEN:
@@ -125,7 +129,10 @@
     ),
     'PAGE_SIZE': 10,
     'UNICODE_JSON': False,
-    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination'
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
+    'DEFAULT_THROTTLE_RATES': {
+        'login': '1/s'
+    }
 }
 
 MIDDLEWARE = [
@@ -147,7 +154,9 @@
 if INFLUXDB_TOKEN:
     MIDDLEWARE.append('analytics.middleware.InfluxDBMiddleware')
 
-if ENV != 'local':
+ALLOWED_ADMIN_IP_ADDRESSES = env.list('ALLOWED_ADMIN_IP_ADDRESSES', default=list())
+if len(ALLOWED_ADMIN_IP_ADDRESSES) > 0:
+    warnings.warn('Restricting access to the admin site for ip addresses %s' % ', '.join(ALLOWED_ADMIN_IP_ADDRESSES))
     MIDDLEWARE.append('app.middleware.AdminWhitelistMiddleware')
 
 ROOT_URLCONF = 'app.urls'
@@ -320,16 +329,6 @@
     }
 }
 
-if env.bool('USE_S3_STORAGE', default=False):
-    STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-    AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
-    AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', 'eu-west-2')
-    AWS_LOCATION = 'static'
-    AWS_DEFAULT_ACL = 'public-read'
-    AWS_S3_ADDRESSING_STYLE = 'virtual'
-
-ALLOWED_ADMIN_IP_ADDRESSES = env.list('ALLOWED_ADMIN_IP_ADDRESSES', default=list())
-
 LOG_LEVEL = env.str('LOG_LEVEL', 'WARNING')
 
 TRENCH_AUTH = {
@@ -368,3 +367,8 @@
         'user_list': ['custom_auth.permissions.CurrentUser'],
     }
 }
+
+
+# Github OAuth credentials
+GITHUB_CLIENT_ID = env.str('GITHUB_CLIENT_ID', '')
+GITHUB_CLIENT_SECRET = env.str('GITHUB_CLIENT_SECRET', '')

src/app/settings/master.py

@@ -40,4 +40,4 @@
 REST_FRAMEWORK['PAGE_SIZE'] = 999
 
 SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$']  # root is exempt as it's used for EB health checks
+SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/app/settings/staging.py

@@ -40,4 +40,4 @@
 REST_FRAMEWORK['PAGE_SIZE'] = 999
 
 SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$']  # root is exempt as it's used for EB health checks
+SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/app/urls.py

@@ -22,10 +22,5 @@
 if settings.DEBUG:
     import debug_toolbar
     urlpatterns = [
-        # Django 2
-        # path('__debug__/', include(debug_toolbar.urls)),
-
-        # For django versions before 2.0:
         url(r'^__debug__/', include(debug_toolbar.urls)),
-
     ] + urlpatterns

src/audit/models.py

@@ -9,7 +9,7 @@
 FEATURE_UPDATED_MESSAGE = "Flag / Remote Config updated: %s"
 SEGMENT_CREATED_MESSAGE = "New Segment created: %s"
 SEGMENT_UPDATED_MESSAGE = "Segment updated: %s"
-FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s"
+FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s in environment: %s"
 ENVIRONMENT_CREATED_MESSAGE = "New Environment created: %s"
 ENVIRONMENT_UPDATED_MESSAGE = "Environment updated: %s"
 FEATURE_STATE_UPDATED_MESSAGE = "Flag state / Remote Config value updated for feature: %s"
@@ -45,3 +45,15 @@ class Meta:
 
     def __str__(self):
         return "Audit Log %s" % self.id
+
+    @classmethod
+    def create_record(cls, obj, obj_type, log_message, author, project=None, environment=None):
+        cls.objects.create(
+            related_object_id=obj.id,
+            related_object_type=obj_type.name,
+            log=log_message,
+            author=author,
+            project=project,
+            environment=environment
+        )
+

src/audit/signals.py

@@ -5,10 +5,10 @@
 
 from audit.models import AuditLog
 from audit.serializers import AuditLogSerializer
+from util.logging import get_logger
 from webhooks.webhooks import call_organisation_webhooks, WebhookEventType
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger = get_logger(__name__)
 
 
 @receiver(post_save, sender=AuditLog)

src/custom_auth/oauth/exceptions.py

@@ -0,0 +1,10 @@
+class GithubError(Exception):
+    pass
+
+
+class GoogleError(Exception):
+    pass
+
+
+class OAuthError(Exception):
+    pass

src/custom_auth/oauth/github.py

@@ -0,0 +1,78 @@
+import requests
+from django.conf import settings
+from requests import RequestException
+
+from custom_auth.oauth.exceptions import GithubError
+from custom_auth.oauth.helpers.github_helpers import convert_response_data_to_dictionary, get_first_and_last_name
+from util.logging import get_logger
+
+GITHUB_API_URL = "https://api.github.com"
+GITHUB_OAUTH_URL = "https://github.com/login/oauth"
+
+NON_200_ERROR_MESSAGE = "Github returned {} status code when getting an access token."
+
+logger = get_logger(__name__)
+
+
+class GithubUser:
+    def __init__(self, code: str, client_id: str = None, client_secret: str = None):
+        self.client_id = client_id or settings.GITHUB_CLIENT_ID
+        self.client_secret = client_secret or settings.GITHUB_CLIENT_SECRET
+
+        self.access_token = self._get_access_token(code)
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        }
+
+    def _get_access_token(self, code) -> str:
+        data = {
+            "code": code,
+            "client_id": self.client_id,
+            "client_secret": self.client_secret
+        }
+        response = requests.post(f"{GITHUB_OAUTH_URL}/access_token", data=data)
+
+        if response.status_code != 200:
+            raise GithubError(NON_200_ERROR_MESSAGE.format(response.status_code))
+
+        response_json = convert_response_data_to_dictionary(response.text)
+        if "error" in response_json:
+            error_message = response_json["error_description"].replace("+", " ")
+            raise GithubError(error_message)
+
+        return response_json["access_token"]
+
+    def get_user_info(self) -> dict:
+        # TODO: use threads?
+        try:
+            return {
+                **self._get_user_name_and_id(),
+                "email": self._get_primary_email()
+            }
+        except RequestException:
+            raise GithubError("Failed to communicate with the Github API.")
+
+    def _get_user_name_and_id(self):
+        user_response = requests.get(f"{GITHUB_API_URL}/user", headers=self.headers)
+        user_response_json = user_response.json()
+        full_name = user_response_json.get("name")
+        first_name, last_name = get_first_and_last_name(full_name) if full_name else ["", ""]
+        return {
+            "first_name": first_name,
+            "last_name": last_name,
+            "github_user_id": user_response_json.get("id")
+        }
+
+    def _get_primary_email(self):
+        emails_response = requests.get(f"{GITHUB_API_URL}/user/emails", headers=self.headers)
+
+        # response from github should be a list of dictionaries, this will find the first entry that is both verified
+        # and marked as primary (there should only be one).
+        primary_email_data = next(
+            filter(lambda email_data: email_data["primary"] and email_data["verified"], emails_response.json()), None
+        )
+
+        if not primary_email_data:
+            raise GithubError("User does not have a verified email address with Github.")
+
+        return primary_email_data["email"]

src/custom_auth/oauth/google.py

@@ -1,15 +1,27 @@
 import requests
+from requests import RequestException
+from rest_framework import status
+
+from custom_auth.oauth.exceptions import GoogleError
 
 USER_INFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json&"
+NON_200_ERROR_MESSAGE = "Google returned {} status code when getting an access token."
 
 
 def get_user_info(access_token):
-    headers = {"Authorization": f"Bearer {access_token}"}
-    response = requests.get(USER_INFO_URL, headers=headers)
-    response_json = response.json()
-    return {
-        "email": response_json["email"],
-        "first_name": response_json.get("given_name", ""),
-        "last_name": response_json.get("family_name", ""),
-        "google_user_id": response_json["id"]
-    }
+    try:
+        headers = {"Authorization": f"Bearer {access_token}"}
+        response = requests.get(USER_INFO_URL, headers=headers)
+
+        if response.status_code != status.HTTP_200_OK:
+            raise GoogleError(NON_200_ERROR_MESSAGE.format(response.status_code))
+
+        response_json = response.json()
+        return {
+            "email": response_json["email"],
+            "first_name": response_json.get("given_name", ""),
+            "last_name": response_json.get("family_name", ""),
+            "google_user_id": response_json["id"]
+        }
+    except RequestException:
+        raise GoogleError("Failed to communicate with the Google API.")

src/custom_auth/oauth/helpers/github_helpers.py

@@ -0,0 +1,23 @@
+from custom_auth.oauth.exceptions import GithubError
+from util.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def convert_response_data_to_dictionary(text: str) -> dict:
+    try:
+        response_data = {}
+        for key, value in [param.split("=") for param in text.split("&")]:
+            response_data[key] = value
+        return response_data
+    except ValueError:
+        logger.warning("Malformed data received from Github (%s)" % text)
+        raise GithubError("Malformed data received from Github")
+
+
+def get_first_and_last_name(full_name: str) -> list:
+    if not full_name:
+        return ["", ""]
+
+    names = full_name.strip().split(" ")
+    return names if len(names) == 2 else [full_name, ""]