Merge branch 'master' into master-github

.gitlab-ci.yml

@@ -1,65 +1,63 @@
 stages:
-    - test
-    - deploy
-    - deploy-aws
+  - test
+  - deploy
+
+cache:
+  key: ${CI_COMMIT_REF_SLUG}
+  paths:
+    - .venv/
 
 test:
-  image: python:3.7
+  image: python:3.7-slim
   stage: test
   services:
-    - postgres:10.9
+    - postgres:10.9-alpine
   variables:
     DJANGO_SETTINGS_MODULE: "app.settings.test"
     DATABASE_URL: postgres://testuser:testpass@postgres/test_db
     POSTGRES_USER: testuser
     POSTGRES_PASSWORD: testpass
   script:
+    - pip install virtualenv # set up local venv dir for caching of packages
+    - virtualenv .venv
+    - source .venv/bin/activate 
     - pip install -r requirements-dev.txt
     - black --check .
     - pytest src -p no:warnings
 
 deploydevelop:
-    image: ilyasemenov/gitlab-ci-git-push
-    stage: deploy
-    script: git-push [email protected]:bullet-train
-    only:
-        - develop
+  image: ilyasemenov/gitlab-ci-git-push
+  stage: deploy
+  script: git-push [email protected]:bullet-train
+  only:
+    - develop
+
+.deploy_to_beanstalk: &deploy_to_beanstalk |
+  echo "Deploying to beanstalk with label $CI_COMMIT_SHORT_SHA"
+  cp requirements.txt ./src/requirements.txt
+  cd src
+  eb deploy $ENVIRONMENT_NAME -l "$CI_COMMIT_SHORT_SHA"
 
 deployawsstaging:
-    image: bullettrain/elasticbeanstalk-pipenv  # TODO: remove pipenv from this docker image
-    stage: deploy-aws
-    script:
-        - export AWS_ACCESS_KEY_ID=$AWS_STAGING_ACCESS_KEY_ID
-        - export AWS_SECRET_ACCESS_KEY=$AWS_STAGING_SECRET_ACCESS_KEY
-        - export DATABASE_URL=$DATABASE_URL_STAGING
-        - export GOOGLE_ANALYTICS_CLIENT_ID=$GOOGLE_ANALYTICS_CLIENT_ID_STAGING
-        - export GOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY_STAGING
-        - export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE_STAGING
-        - cp requirements.txt ./src/requirements.txt
-        - sh generate.sh
-        - git config --global user.email "[email protected]" 
-        - git config --global user.name "Gitlab"
-        - git add . && git commit -m "Commit to EB"
-        - cd src
-        - eb deploy $EB_ENVIRONMENT_STAGING
-    only:
-        - staging
+  image: flagsmith/eb-cli:latest
+  stage: deploy
+  variables:
+    ENVIRONMENT_NAME: staging-api
+    AWS_ACCESS_KEY_ID: "$AWS_STAGING_ACCESS_KEY_ID"
+    AWS_SECRET_ACCESS_KEY: "$AWS_STAGING_SECRET_ACCESS_KEY"
+  script:
+    - *deploy_to_beanstalk
+  only:
+    - staging
 
 deployawsmaster:
-    image: bullettrain/elasticbeanstalk-pipenv
-    stage: deploy-aws
-    script:
-        - export DATABASE_URL=$DATABASE_URL_PRODUCTION
-        - export GOOGLE_ANALYTICS_CLIENT_ID=$GOOGLE_ANALYTICS_CLIENT_ID_PRODUCTION
-        - export GOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY_PRODUCTION
-        - export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE_PRODUCTION
-        - cp requirements.txt ./src/requirements.txt
-        - sh generate.sh
-        - git config --global user.email "[email protected]" 
-        - git config --global user.name "Gitlab"
-        - git add . && git commit -m "Commit to EB"
-        - cd src
-        - eb deploy $EB_ENVIRONMENT_PRODUCTION --timeout 30
-    only:
-        - master
-        - master-aws
+  image: flagsmith/eb-cli:latest
+  stage: deploy
+  variables:
+    ENVIRONMENT_NAME: production-api
+    AWS_ACCESS_KEY_ID: "$AWS_PRODUCTION_ACCESS_KEY_ID"
+    AWS_SECRET_ACCESS_KEY: "$AWS_PRODUCTION_SECRET_ACCESS_KEY"
+  script:
+    - *deploy_to_beanstalk
+  only:
+    - master

.gitlab/merge_request_templates/general.md

@@ -0,0 +1,7 @@
+## Overview of the Merge Request
+
+This Merge Request works by...
+
+## Have you written tests?
+
+If not, explain here!

docker-compose.dev.yml

@@ -12,9 +12,8 @@ services:
   api:
     build:
       context: .
-      dockerfile: Dockerfile.dev
-    command: ./bin/docker-dev
-    # command: sleep 9999
+      dockerfile: docker/Dockerfile.dev
+    command: docker/bin/docker-dev
     volumes:
       - .:/app
     environment:

docker/Dockerfile

@@ -1,20 +1,13 @@
-FROM python:3.7
+FROM python:3.8-slim as application
 
-RUN rm /var/lib/dpkg/info/format
-RUN printf "1\n" > /var/lib/dpkg/info/format
-RUN dpkg --configure -a
-RUN  apt-get clean && apt-get update \
-    && apt-get install -y --no-install-recommends \
-        postgresql-client \
-    && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY requirements.txt /app/
+COPY src/ /app/src/
+COPY docker/bin/ /app/bin/
 
-RUN pip install pipenv
-
-WORKDIR /usr/src
-COPY src/ ./
-COPY Pipfile* ./
-RUN pipenv install --deploy
+RUN pip install -r requirements.txt
 
 ENV DJANGO_SETTINGS_MODULE=app.settings.master-docker
-
 EXPOSE 8000
+
+CMD ["./bin/docker"]

docker/Dockerfile.ci

@@ -0,0 +1,18 @@
+FROM python:3-slim
+
+ENV BUILD_DIR=/build
+
+RUN apt-get update
+RUN apt-get install -y \
+        build-essential zlib1g-dev libssl-dev libncurses-dev git \
+        libffi-dev libsqlite3-dev libreadline-dev libbz2-dev curl
+
+RUN mkdir $BUILD_DIR
+WORKDIR ${BUILD_DIR}
+
+RUN git clone https://github.com/aws/aws-elastic-beanstalk-cli-setup.git
+
+RUN pip install virtualenv
+RUN python aws-elastic-beanstalk-cli-setup/scripts/ebcli_installer.py --location $BUILD_DIR
+
+ENV PATH="${BUILD_DIR}/.ebcli-virtual-env/bin:${PATH}"

docker/Dockerfile.dev

@@ -0,0 +1,13 @@
+FROM python:3.8
+ENV PYTHONUNBUFFERED 1
+
+RUN mkdir /app
+WORKDIR /app
+COPY requirements.txt requirements-dev.txt /app/
+COPY src/ /app/src/
+COPY docker/bin/ /app/bin/
+
+RUN pip install -r requirements-dev.txt
+
+ENV DJANGO_SETTINGS_MODULE=app.settings.master-docker
+EXPOSE 8000

readme.md

@@ -122,6 +122,14 @@ docker-compose up
 This will use some default settings created in the `docker-compose.yml` file located in the root of 
 the project. These should be changed before using in any production environments.
 
+You can work on the project itself using Docker:
+
+```bash
+docker-compose -f docker-compose.dev.yml up
+```
+
+This gets an environment up and running along with Postgres and enables hot reloading etc. 
+
 ### Environment Variables
 
 The application relies on the following environment variables to run:
@@ -151,6 +159,31 @@ The application relies on the following environment variables to run:
 * `ALLOWED_ADMIN_IP_ADDRESSES`: restrict access to the django admin console to a comma separated list of IP addresses (e.g. `127.0.0.1,127.0.0.2`) 
 * `USER_CREATE_PERMISSIONS`: set the permissions for creating new users, using a comma separated list of djoser or rest_framework permissions. Use this to turn off public user creation for self hosting. e.g. `'djoser.permissions.CurrentUserOrAdmin'` Defaults to `'rest_framework.permissions.AllowAny'`.
 * `ENABLE_EMAIL_ACTIVATION`: new user registration will go via email activation flow, default False
+* `SENTRY_SDK_DSN`: If using Sentry, set the project DSN here.
+* `SENTRY_TRACE_SAMPLE_RATE`: Float. If using Sentry, sets the trace sample rate. Defaults to 1.0.
+
+## Pre commit
+
+The application uses pre-commit configuration ( `.pre-commit-config.yaml` ) to run black formatting before commits.
+
+To install pre-commit:
+
+```bash
+pip install pre-commit
+pre-commit install
+```
+
+### Creating a secret key
+
+It is important to also set an environment variable on whatever platform you are using for 
+`DJANGO_SECRET_KEY`. There is a function to create one in `app.settings.common` if none exists in 
+the environment variables, however, this is not suitable for use in production. To generate a new 
+secret key, you can use the function defined in `src/secret-key-gen.py` by simply running it from a 
+command prompt:
+
+```bash
+python secret-key-gen.py
+```
 
 ## Adding dependencies
 
@@ -171,9 +204,9 @@ given project. The number of seconds this is cached for is configurable using th
 
 ## Stack
 
-- Python 2.7.14
-- Django 1.11.13
-- DjangoRestFramework 3.8.2
+- Python 3.8
+- Django 2.2.17
+- DjangoRestFramework 3.12.1
 
 ## Static Files
 
@@ -186,7 +219,7 @@ that the static files are hosted in.
 
 ## Documentation
 
-Further documentation can be found [here](https://docs.flagsmith.com). 
+Further documentation can be found [here](https://docs.bullet-train.io).
 
 ## Contributing
 

requirements.txt

@@ -22,7 +22,6 @@ chargebee==2.7.7
 python-http-client<3.2.0 # 3.2.0 is the latest but throws an error on installation saying that it's not found
 django-health-check==3.14.3
 django-storages==1.10.1
-django-environ==0.4.5
 django-trench==0.2.3
 djoser==2.0.5
 influxdb-client==1.11.0
@@ -31,4 +30,6 @@ django-ses==1.0.3
 django-axes==5.8.0
 django-admin-sso==3.0.0
 drf-yasg2==1.19.3
-django-debug-toolbar==3.1.1
+django-debug-toolbar==3.1.1
+sentry-sdk==0.19.4
+environs==9.2.0

src/.ebextensions/db-migrate.config

@@ -1,6 +1,9 @@
 container_commands:
   01_migrate:
-    command: "django-admin.py migrate"
+    command: django-admin.py migrate
     leader_only: true
   02_collectstatic:
-    command: "source /opt/python/run/venv/bin/activate && python manage.py collectstatic --noinput"
+    command: python manage.py collectstatic --noinput
+option_settings:
+  aws:elasticbeanstalk:application:environment:
+    DJANGO_SETTINGS_MODULE: app.settings.production

src/analytics/influxdb_wrapper.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from django.conf import settings
 from influxdb_client import InfluxDBClient, Point
 from influxdb_client.client.write_api import SYNCHRONOUS
@@ -88,14 +90,13 @@ def get_event_list_for_organisation(organisation_id: int):
         drop_columns='"organisation", "organisation_id", "type", "project", "project_id"',
         extra="|> aggregateWindow(every: 24h, fn: sum)",
     )
-    dataset = []
+    dataset = defaultdict(list)
     labels = []
     for result in results:
         for record in result.records:
-            dataset.append(
-                {"t": record.values["_time"].isoformat(), "y": record.values["_value"]}
-            )
-            labels.append(record.values["_time"].strftime("%Y-%m-%d"))
+            dataset[record["resource"]].append(record["_value"])
+            if len(labels) != 31:
+                labels.append(record.values["_time"].strftime("%Y-%m-%d"))
     return dataset, labels
 
 
@@ -120,6 +121,6 @@ def get_multiple_event_list_for_organisation(organisation_id: int):
 
     for result in results:
         for i, record in enumerate(result.records):
-            dataset[i][record.values["resource"]] = record.values["_value"]
-            dataset[i]["name"] = record.values["_time"].isoformat()
+            dataset[i][record.values["resource"].capitalize()] = record.values["_value"]
+            dataset[i]["name"] = record.values["_time"].strftime("%Y-%m-%d")
     return dataset

src/api/urls/v1.py

@@ -13,7 +13,7 @@
 
 schema_view = get_schema_view(
     openapi.Info(
-        title="Bullet Train API",
+        title="Flagsmith API",
         default_version="v1",
         description="",
         license=openapi.License(name="BSD License"),