(refactor) Replace AppScheduler with Redis + Celery

  * introduce redis + celery for running background tasks
  * remove appscheduler and pyppeteer patch
  * refactor scraper

.github/workflows/django.yml

@@ -1,10 +1,6 @@
 name: Django CI
 
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
+on: [push]
 
 jobs:
   #
@@ -58,7 +54,8 @@ jobs:
           pip install -r requirements/ci.txt
       - name: Run tests
         env:
-          DJANGO_ENV: "CI"
+          SECRET_KEY: dummy
+          DJANGO_ENV: BASE
         run: pytest articles/tests/
   #
   # Migrations
@@ -91,4 +88,5 @@ jobs:
         run: |
           python manage.py makemigrations --check --dry-run
         env:
-          DJANGO_ENV: "CI"
+          SECRET_KEY: dummy
+          DJANGO_ENV: BASE

.gitignore

@@ -12,6 +12,9 @@ staticfiles/
 dev/
 logs/*.log*
 
+### celery etc.
+celerybeat-schedule
+
 ### Unit test / coverage reports
 htmlcov/
 .tox/

articles/data.py

@@ -0,0 +1,16 @@
+sources = {
+    "en": [
+        "Al Jazeera",
+        "Associated Press",
+        "Christian Science Monitor",
+        "Consortium News",
+        "Current Affairs",
+        "New York Times",
+        "NPR",
+        "Reuters",
+        "The Atlantic",
+        "The Intercept",
+        "UPI",
+        "Wall Street Journal",
+    ]
+}

articles/tasks.py

@@ -0,0 +1,44 @@
+import json
+
+from celery import group, shared_task
+from django.utils import timezone
+
+import scraper
+
+from .data import sources
+from .models import Article, Source
+
+
+@shared_task
+def get_articles_for_source(source_title: str):
+    source: Source = Source.objects.get(name=source_title)
+    sitemap = source.to_dict()
+    starting_urls = [
+        sitemap["base_url"] + path for path in sitemap["paths"]
+    ]
+
+    spider = scraper.Spider(starting_urls, sitemap)
+    scraper.run(spider)
+    data = [json.loads(article) for article in spider.articles]
+
+    Article.objects.bulk_create([
+        Article(
+            headline=article_data["headline"],
+            slug=article_data["slug"],
+            source=Source.objects.get(link=article_data["source_link"]),
+            summary=article_data["summary"],
+            language=article_data["language"],
+            link=article_data["link"],
+            created_at=timezone.now(),
+        ) for article_data in data
+    ], ignore_conflicts=True)
+
+
+@shared_task
+def get_articles(language: str):
+    task_group = group(
+        get_articles_for_source.s(source_title=title) for title in sources[language]
+    )
+    promise = task_group.apply_async()
+    if promise.ready():
+        return promise.get()

articles/tests/test_parsers.py

@@ -1,8 +1,9 @@
 import json
 
+from scraper.parser import parse
+
 from ..constants import Language
 from ..models import Source
-from ..scraper.parser import parse
 
 
 def test_parse(source_values):

nous_aggregator/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

nous_aggregator/celery.py

@@ -0,0 +1,11 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "nous_aggregator.settings.local")
+
+app = Celery("nous_aggregator")
+
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+app.autodiscover_tasks()

nous_aggregator/settings/__init__.py

@@ -10,11 +10,9 @@
 DJANGO_ENV = config('DJANGO_ENV', default="")
 
 match DJANGO_ENV:
-    case "CI":
-        from .ci import *
+    case "BASE":
+        from .base import *
     case "LOCAL":
         from .local import *
-    case "STAGING":
-        from .staging import *
     case "":
         from .production import *

nous_aggregator/settings/base.py

@@ -1,11 +1,39 @@
 import os
 from pathlib import Path
 
+from decouple import Csv, config
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 # (modified because settings files are nested one level deeper)
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
 
+SECRET_KEY = config("SECRET_KEY", default="")
+
+DEBUG = config("DEBUG", default=False, cast=bool)
+
+ALLOWED_HOSTS = config("ALLOWED_HOSTS", default="", cast=Csv())
+
+CSRF_TRUSTED_ORIGINS = config("CSRF_TRUSTED_ORIGINS", default="", cast=Csv())
+
+SESSION_COOKIE_SECURE = config("SESSION_COOKIE_SECURE", default=True, cast=bool)
+
+CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=True, cast=bool)
+
+SECURE_SSL_REDIRECT = config("SECURE_SSL_REDIRECT", default=True, cast=bool)
+
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+
+DATABASES = {
+    "default": {
+        "ENGINE": config("DATABASE_ENGINE", default="django.db.backends.postgresql"),
+        "NAME": config("DATABASE_NAME", default="postgres"),
+        "USER": config("DATABASE_USER", default="postgres"),
+        "PASSWORD": config("DATABASE_PASSWORD", default="postgres"),
+        "HOST": config("DATABASE_HOST", default="localhost"),
+        "PORT": config("DATABASE_PORT", default=5432, cast=int),
+    },
+}
+
 
 # Application definition
 
@@ -17,10 +45,8 @@
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.humanize",
-    # My Apps
+    # nous_aggregator apps
     "articles.apps.ArticlesConfig",
-    # Third Party Apps
-    "django_apscheduler",
 ]
 
 MIDDLEWARE = [
@@ -39,11 +65,6 @@
     "127.0.0.1",
 ]
 
-# Timeouts (connection timeout, read timeout) in seconds for requests
-# made with the requests library
-REQUESTS_TIMEOUT = (6, 18)
-REQUESTS_TIMEOUT_JS = (6, 60)
-
 TEMPLATES = [
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
@@ -154,3 +175,21 @@
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+
+# Timeouts (connection timeout, read timeout) in seconds for requests
+REQUESTS_TIMEOUT = (30, 60)
+
+# Celery
+CELERY_BROKER_URL = config("CELERY_BROKER_URL")
+CELERY_RESULT_BACKEND = config("CELERY_RESULT_BACKEND")
+# TODO: retrieve schedule from env var or module
+CELERY_BEAT_SCHEDULE = {
+    "get_articles_en": {
+        "task": "articles.tasks.get_articles",
+        "schedule": 60,
+        "kwargs": {
+            "language": "en",
+        }
+    }
+}

nous_aggregator/settings/ci.py

@@ -1,18 +1,18 @@
-from .base import *
+# from .base import *
 
-SECRET_KEY = "hush-hush"
+# SECRET_KEY = "hush-hush"
 
-DEBUG = False
+# DEBUG = False
 
-ALLOWED_HOSTS = ['*']
+# ALLOWED_HOSTS = ['*']
 
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.postgresql",
-        "NAME": "nous_aggregator",
-        "USER": "postgres",
-        "PASSWORD": "postgres",
-        "HOST": "localhost",
-        "PORT": 5432,
-    },
-}
+# DATABASES = {
+#     "default": {
+#         "ENGINE": "django.db.backends.postgresql",
+#         "NAME": "nous_aggregator",
+#         "USER": "postgres",
+#         "PASSWORD": "postgres",
+#         "HOST": "localhost",
+#         "PORT": 5432,
+#     },
+# }

nous_aggregator/settings/local.py

@@ -1,3 +1,5 @@
+import socket
+
 from .base import *
 
 SECRET_KEY = "hush-hush"
@@ -13,8 +15,13 @@
 
 INSTALLED_APPS += [
     "debug_toolbar",
+    "django_extensions",
 ]
 
 MIDDLEWARE += [
     "debug_toolbar.middleware.DebugToolbarMiddleware",
 ]
+
+# for django_debug_toolbar
+hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + ["127.0.0.1", "10.0.2.2"]

nous_aggregator/settings/production.py

@@ -2,7 +2,7 @@
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration
 
-from .staging import *
+from .base import *
 
 sentry_sdk.init(
     dsn="https://[email protected]/6748377",