Added yml config for Green Metrics Tool

.env.test → .env.ci

@@ -1,12 +1,10 @@
 SECRET_KEY='some-development-secret'
 DATABASE_URL=mysql://root:[email protected]:3306/greencheck
+DATABASE_URL_READ_ONLY=mysql://root:[email protected]:3306/greencheck
+RABBITMQ_URL=amqp://guest:guest@localhost:5672/
 MAILGUN_API_KEY="nope"
 DJANGO_SETTINGS_MODULE='greenweb.settings.testing'
 SENTRY_DSN="https://[email protected]/test"
 SENTRY_AUTH_TOKEN="nope"
 SENTRY_ORG="nope"
 PYTHONDONTWRITEBYTECODE=1
-RABBITMQ_URL=amqp://guest:guest@localhost:5672/
-
-MAXMIND_USER_ID = 123456
-MAXMIND_LICENCE_KEY = "xxxxxxxxxxxxxxxx"

.github/workflows/run-tests.yml

@@ -60,23 +60,9 @@ jobs:
           uv venv
           uv pip install -r requirements/requirements.linux.generated.txt
 
-      - name: Run tests
+      - name: Run tests, explictly using the python version of dotenv
         run: |
           source .venv/bin/activate
-          .venv/bin/dotenv -f .env.test run -- pytest
+          pytest
         env:
-          DOMAIN_SNAPSHOT_BUCKET: ${{ secrets.TESTING_DOMAIN_SNAPSHOT_BUCKET }}
-          DATABASE_URL_READ_ONLY: ${{ secrets.TESTING_DATABASE_URL_READ_ONLY }}
-          OBJECT_STORAGE_ACCESS_KEY_ID: ${{ secrets.TESTING_OBJECT_STORAGE_ACCESS_KEY_ID }}
-          OBJECT_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TESTING_OBJECT_STORAGE_SECRET_ACCESS_KEY }}
-          OBJECT_STORAGE_ENDPOINT: ${{ secrets.TESTING_OBJECT_STORAGE_ENDPOINT }}
-          OBJECT_STORAGE_REGION: ${{ secrets.TESTING_OBJECT_STORAGE_REGION }}
-          OBJECT_STORAGE_BUCKET_NAME: ${{ secrets.TESTING_OBJECT_STORAGE_BUCKET_NAME }}
-          AMAZON_PROVIDER_ID: ${{ secrets.AMAZON_PROVIDER_ID }}
-          AMAZON_REMOTE_API_ENDPOINT: ${{ secrets.AMAZON_REMOTE_API_ENDPOINT }}
-          MICROSOFT_PROVIDER_ID: ${{ secrets.MICROSOFT_PROVIDER_ID }}
-          MICROSOFT_LOCAL_FILE_DIRECTORY: ${{ secrets.MICROSOFT_LOCAL_FILE_DIRECTORY }}
-          EQUINIX_PROVIDER_ID: ${{ secrets.EQUINIX_PROVIDER_ID }}
-          EQUINIX_REMOTE_API_ENDPOINT: ${{ secrets.EQUINIX_REMOTE_API_ENDPOINT }}
-          AWS_SHARED_CREDENTIALS_FILE: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
-          AWS_CONFIG_FILE: ${{ secrets.AWS_CONFIG_FILE }}
+          RUNNING_IN_CI: true

Dockerfile

@@ -17,8 +17,10 @@ RUN apt-get clean
 # Delete index files we don't need anymore:
 RUN rm -rf /var/lib/apt/lists/*
 
-# Install dependencies in a virtualenv
+# Declare the path for our virtual environment
 ENV VIRTUAL_ENV=/app/.venv
+
+# RUN useradd deploy --create-home && mkdir /app /app/.venv && chown -R deploy /app /app/.venv
 RUN useradd deploy --create-home && mkdir /app $VIRTUAL_ENV && chown -R deploy /app $VIRTUAL_ENV
 
 WORKDIR /app
@@ -40,15 +42,15 @@ USER deploy
 RUN python -m venv $VIRTUAL_ENV
 
 # Add our python libraries for managing dependencies
-uv 0.1.43 is triggering bad certificate errors, so we pin to 0.1.39
+# uv 0.1.43 is triggering bad certificate errors, so we pin to 0.1.39
 RUN python -m pip install uv==0.1.39 wheel --upgrade
 
 # Copy application code, with dockerignore filtering out the stuff we don't want
 # from our final build artefact
 COPY --chown=deploy . .
 
-# Install dependencies via uv
-RUN uv pip install -r requirements/requirements.linux.generated.txt 
+# Install dependencies via uv into /app/.venv/
+RUN python -m pip install -r requirements/requirements.linux.generated.txt 
 
 # Set up front end pipeline
 RUN python ./manage.py tailwind install
@@ -67,4 +69,7 @@ RUN python ./manage.py collectstatic --noinput --clear
 
 # Use the shell form of CMD, so we have access to our environment variables
 # $GUNICORN_CMD_ARGS allows us to add additional arguments to the gunicorn command
-CMD gunicorn greenweb.wsgi --bind $GUNICORN_BIND_IP:$PORT --config gunicorn.conf.py $GUNICORN_CMD_ARGS
+
+
+
+CMD /app/.venv/bin/gunicorn greenweb.wsgi --bind $GUNICORN_BIND_IP:$PORT --config gunicorn.conf.py $GUNICORN_CMD_ARGS

compose.yaml

@@ -1,6 +1,6 @@
 
 services:
-  db:
+  mariadb:
     image: mariadb:10.11
     restart: always
     environment:
@@ -18,8 +18,18 @@ services:
     ports:
       - 5672:5672
   django:
-    env_file:
-    - path: ./.env.docker
+    # uncomment to pass in your own .env file instead of using the environment variables below
+    # env_file:
+    # - path: ./.env.docker
+    environment:
+      - PORT=9000
+      - GUNICORN_BIND_IP=0.0.0.0
+      - PYTHONDONTWRITEBYTECODE=1
+      - PYTHONUNBUFFERED=1
+      - DATABASE_URL=mysql://deploy:deploy@mariadb:3306/greencheck
+      - DATABASE_URL_READ_ONLY=mysql://deploy:deploy@mariadb:3306/greencheck
+      - RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672/
+      - DJANGO_SETTINGS_MODULE=greenweb.settings.development
     build:
       context: .
       dockerfile: Dockerfile
@@ -34,5 +44,5 @@ services:
       - ./greenweb:/app/greenweb
     restart: always
     depends_on:
-      - db
+      - mariadb
       - rabbitmq

green_metric_tests/greencheck_test.spec.js

@@ -0,0 +1,18 @@
+const request = require('supertest');
+const expect = require('chai').expect;
+
+describe('Greencheck API', () => {
+  it('Has the correct URL property', () => {
+    // 
+    request('http://django:9000')
+      .get('/greencheck/climateaction.tech')
+      .expect(200)
+      .expect('Content-Type', 'application/json')
+      .expect(function (res) {
+        if (!res.body.hasOwnProperty('url')) throw new Error("Expected 'url' key!");
+      })
+      .end(function (err, res) {
+        if (err) throw err;
+      })
+  });
+});

green_metric_tests/package.json

@@ -0,0 +1,16 @@
+{
+    "name": "green_metric_tests",
+    "version": "1.0.0",
+    "description": "Integration tests for the Greencheck API, intended for use with the Green Metrics Tool from Green Code Berlin. (https://www.green-coding.io/projects/green-metrics-tool/)",
+    "main": "index.js",
+    "scripts": {
+      "test": "mocha '/*.spec.js'"
+    },
+    "author": "James Camilleri",
+    "license": "SEE LICENSE IN ../LICENCE",
+    "dependencies": {
+      "chai": "^4.2.0",
+      "mocha": "^10.4.0",
+      "supertest": "^7.0.0"
+    }
+  }

greenweb/settings/common.py

@@ -10,7 +10,6 @@
 https://docs.djangoproject.com/en/dev/ref/settings/
 """
 
-import os
 import environ
 import pathlib
 from dramatiq import middleware as dramatiq_middleware
@@ -19,27 +18,52 @@
 ROOT = environ.Path(__file__) - 3
 env = environ.Env(
     DEBUG=(bool, False),
-    SECRET_KEY=(str, os.getenv("SECRET_KEY")),
-    DATABASE_URL=(str, os.getenv("DATABASE_URL")),
-    DATABASE_URL_READ_ONLY=(str, os.getenv("DATABASE_URL_READ_ONLY")),
-    DOMAIN_SNAPSHOT_BUCKET=(str, os.getenv("DOMAIN_SNAPSHOT_BUCKET")),
-    # add for object storage
-    OBJECT_STORAGE_ENDPOINT=(str, os.getenv("OBJECT_STORAGE_ENDPOINT")),
-    OBJECT_STORAGE_REGION=(str, os.getenv("OBJECT_STORAGE_REGION")),
-    OBJECT_STORAGE_ACCESS_KEY_ID=(str, os.getenv("OBJECT_STORAGE_ACCESS_KEY_ID")),
+    SECRET_KEY=(str, "some-key"),
+    DJANGO_LOG_LEVEL=(str, "INFO"),
+    # databases
+    # TODO: this is only added to debug a github action issue with failing tests
+    DATABASE_URL=(
+        str,
+        "mysql://root:[email protected]:3306/greencheck",
+    ),
+    DATABASE_URL_READ_ONLY=(
+        str,
+        "mysql://root:[email protected]:3306/greencheck",
+    ),
+    EXPLORER_TOKEN=(str, "some-token"),
+    # object storage
+    OBJECT_STORAGE_ENDPOINT=(str, "https://s3.nl-ams.scw.cloud"),
+    OBJECT_STORAGE_REGION=(str, "nl-ams"),
+    OBJECT_STORAGE_ACCESS_KEY_ID=(str, "xxxxxxxxxxxxxxxxxxxx"),
     OBJECT_STORAGE_SECRET_ACCESS_KEY=(
         str,
-        os.getenv("OBJECT_STORAGE_SECRET_ACCESS_KEY"),
+        "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
     ),
-    # add for basicauth
+    OBJECT_STORAGE_BUCKET_NAME=(str, "tgwf-green-domains-dev"),
+    DOMAIN_SNAPSHOT_BUCKET=(str, "tgwf-green-domains-dev"),
+    # basicauth for staging environments
     BASICAUTH_DISABLE=(bool, True),
     BASICAUTH_USER=(str, "staging_user"),
     BASICAUTH_PASSWORD=(str, "strong_password"),
-    API_URL=(str, os.getenv("API_URL")),
+    # Swagger API docs url
+    API_URL=(str, "https://greenweb.localhost"),
     TRELLO_REGISTRATION_EMAIL_TO_BOARD_ADDRESS=(
         str,
-        os.getenv("TRELLO_REGISTRATION_EMAIL_TO_BOARD_ADDRESS"),
+        "mail-to-board@localhost",
     ),
+    RABBITMQ_URL=(str, "amqp://USERNAME:PASSWORD@localhost:5672/"),
+    # cloud providers updated on cronjobs
+    # we use very high numbers to minimise chance of collision
+    # with an actual provider id
+    GOOGLE_PROVIDER_ID=(int, 10000001),
+    GOOGLE_DATASET_ENDPOINT=(str, "https://www.gstatic.com/ipranges/cloud.json"),
+    MICROSOFT_PROVIDER_ID=(int, 1000002),
+    EQUINIX_PROVIDER_ID=(int, 1000003),
+    EQUINIX_REMOTE_API_ENDPOINT=(str, "https://domain/link/to/file.txt"),
+    AMAZON_PROVIDER_ID=(int, 1000004),
+    AMAZON_REMOTE_API_ENDPOINT=(str, "https://domain/link/to/file.json"),
+    MAXMIND_USER_ID=(str, "123456"),
+    MAXMIND_LICENCE_KEY=(str, "xxxxxxxxxxxxxxxx"),
 )
 
 # in some cases we don't have a .env file to work from - the environment
@@ -237,7 +261,7 @@
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 # only support API access with the sql explorer if we
 # explicitly set the token
-EXPLORER_TOKEN = os.getenv("EXPLORER_TOKEN")
+EXPLORER_TOKEN = env("EXPLORER_TOKEN")
 if EXPLORER_TOKEN:
     EXPLORER_TOKEN_AUTH_ENABLED = True
 
@@ -246,8 +270,8 @@
 GEOIP_PROVIDER_DOWNLOAD_URL = (
     "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
 )
-GEOIP_USER = env("MAXMIND_USER_ID", default=None)
-GEOIP_PASSWORD = env("MAXMIND_LICENCE_KEY", default=None)
+GEOIP_USER = env("MAXMIND_USER_ID")
+GEOIP_PASSWORD = env("MAXMIND_LICENCE_KEY")
 
 # Allow requests from any origin, but only make the API urls available
 # CORS_URLS_REGEX = r"^/api/.*$"
@@ -319,24 +343,22 @@
 
 # Importer variables
 # Microsoft
-MICROSOFT_PROVIDER_ID = env("MICROSOFT_PROVIDER_ID", default=None)
+MICROSOFT_PROVIDER_ID = env("MICROSOFT_PROVIDER_ID")
 
 # Equinix
-EQUINIX_PROVIDER_ID = env("EQUINIX_PROVIDER_ID", default=None)
-EQUINIX_REMOTE_API_ENDPOINT = env("EQUINIX_REMOTE_API_ENDPOINT", default=None)
+EQUINIX_PROVIDER_ID = env("EQUINIX_PROVIDER_ID")
+EQUINIX_REMOTE_API_ENDPOINT = env("EQUINIX_REMOTE_API_ENDPOINT")
 
 # Amazon
-AMAZON_PROVIDER_ID = env("AMAZON_PROVIDER_ID", default=None)
-AMAZON_REMOTE_API_ENDPOINT = env("AMAZON_REMOTE_API_ENDPOINT", default=None)
+AMAZON_PROVIDER_ID = env("AMAZON_PROVIDER_ID")
+AMAZON_REMOTE_API_ENDPOINT = env("AMAZON_REMOTE_API_ENDPOINT")
 
 # Google
-GOOGLE_PROVIDER_ID = env("GOOGLE_PROVIDER_ID", default=None)
-GOOGLE_DATASET_ENDPOINT = env(
-    "GOOGLE_DATASET_ENDPOINT", default="https://www.gstatic.com/ipranges/cloud.json"
-)
+GOOGLE_PROVIDER_ID = env("GOOGLE_PROVIDER_ID")
+GOOGLE_DATASET_ENDPOINT = env("GOOGLE_DATASET_ENDPOINT")
 
 
-RABBITMQ_URL = env("RABBITMQ_URL", default=None)
+RABBITMQ_URL = env("RABBITMQ_URL")
 
 
 REST_FRAMEWORK = {
@@ -380,7 +402,7 @@
     "root": {"handlers": ["console"], "level": "INFO"},
     "handlers": {
         "console": {
-            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
+            "level": env("DJANGO_LOG_LEVEL"),
             "class": "logging.StreamHandler",
             "formatter": "simple",
         },

greenweb/settings/development.py

@@ -5,7 +5,10 @@
 INTERNAL_IPS = ["127.0.0.1"]
 hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
 INTERNAL_IPS += [ip[:-1] + "1" for ip in ips]
-ALLOWED_HOSTS.extend(["127.0.0.1", "localhost"])  # noqa
+
+# django is added to allow for the app to server requests in a
+# docker compose environment, or in green metrics tool
+ALLOWED_HOSTS.extend(["127.0.0.1", "localhost", "django"])  # noqa
 
 INSTALLED_APPS.append("debug_toolbar")  # noqa
 

greenweb/settings/production.py

@@ -2,6 +2,15 @@
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration
 
+import environ
+
+
+prod_env = environ.Env(
+    SENTRY_DSN=(str, ""),
+    SENTRY_ENVIRONMENT=(str, "production"),
+    SENTRY_RELEASE=(str, "[email protected]"),
+    SENTRY_SAMPLE_RATE=(str, 0),
+)
 
 ANYMAIL = {
     "MAILGUN_API_KEY": env("MAILGUN_API_KEY"),  # noqa
@@ -42,9 +51,9 @@
 AWS_S3_FILE_OVERWRITE = False
 
 # report when things asplode
-SENTRY_DSN = os.environ.get("SENTRY_DSN", False)  # noqa
-SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "production")  # noqa
-SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "[email protected]")  # noqa
+SENTRY_DSN = prod_env("SENTRY_DSN")
+SENTRY_ENVIRONMENT = prod_env("SENTRY_ENVIRONMENT")
+SENTRY_RELEASE = prod_env("SENTRY_RELEASE")
 
 # Set to a value between 0 for 0% of request and
 # 1.0 to capture 100% of requests and annotate
@@ -56,7 +65,7 @@
 # For more:
 # https://docs.sentry.io/platforms/python/guides/django/
 # https://docs.sentry.io/platforms/python/guides/django/performance/
-sentry_sample_rate = os.environ.get("SENTRY_SAMPLE_RATE", 0)  # noqa
+sentry_sample_rate = prod_env("SENTRY_SAMPLE_RATE")
 
 if SENTRY_DSN:
     sentry_sdk.init(

greenweb/settings/testing.py

@@ -1,4 +1,5 @@
 from .common import *  # noqa
+import os
 
 INTERNAL_IPS = ["127.0.0.1"]
 ALLOWED_HOSTS.extend(["127.0.0.1", "localhost"])  # noqa
@@ -30,7 +31,8 @@
     ],
 }
 
-# we replace this with the autogenerated address for a specific trello board in production
-TRELLO_REGISTRATION_EMAIL_TO_BOARD_ADDRESS = "mail-to-board@localhost"
+DRF_LOGGER_INTERVAL = 1
 
-DRF_LOGGER_INTERVAL=1
+if os.getenv("RUNNING_IN_CI", False):
+    # add the settings specifcally for CI in github actions
+    environ.Env.read_env(".env.ci")  # noqa