Refactor project structure

.github/workflows/django.yml

@@ -19,7 +19,8 @@ jobs:
           pip install -r requirements/ci.txt
       - name: Run Bandit
         run: |
-          bandit -r articles/ -x tests
+          bandit -r src/articles/ -x tests
+          bandit -r src/scraper/ -x tests
   #
   # Tests
   #
@@ -58,9 +59,9 @@ jobs:
           DJANGO_ENV: BASE
           SECURE_SSL_REDIRECT: False
         run: |
-          pytest articles/tests/unit/
-          pytest articles/tests/integration/
-          pytest scraper/tests/
+          pytest src/articles/tests/unit/
+          pytest src/articles/tests/integration/
+          pytest src/scraper/tests/
 
   #
   # Migrations

nous_aggregator/asgi.py → config/asgi.py

@@ -1,5 +1,5 @@
 """
-ASGI config for nous_aggregator project.
+ASGI config for nous-aggregator project.
 
 It exposes the ASGI callable as a module-level variable named ``application``.
 
@@ -12,6 +12,6 @@
 from django.core.asgi import get_asgi_application
 from django.core.handlers.asgi import ASGIHandler
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
 
 application: ASGIHandler = get_asgi_application()

nous_aggregator/celery.py → config/celery.py

@@ -2,9 +2,9 @@
 
 from celery import Celery
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "nous_aggregator.settings.local")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
 
-app = Celery("nous_aggregator")
+app = Celery("config")
 
 app.config_from_object("django.conf:settings", namespace="CELERY")
 

config/scraper.py

@@ -0,0 +1,21 @@
+tasks = {
+    "magazines": {
+        "en": {
+            "schedule": 120,
+            "titles": [
+                "Al Jazeera",
+                "Associated Press",
+                "Christian Science Monitor",
+                "Consortium News",
+                "Current Affairs",
+                "New York Times",
+                "NPR",
+                "Reuters",
+                "The Atlantic",
+                "The Intercept",
+                "UPI",
+                "Wall Street Journal",
+            ]
+        },
+    },
+}

nous_aggregator/settings/base.py → config/settings/base.py

@@ -1,14 +1,18 @@
 import os
+import sys
 from pathlib import Path
-from typing import Any, Dict, List, Union
+from typing import Any, Dict
 
 from decouple import Csv, config
 
-from scraper import tasks as scraper_tasks
+from .. import scraper
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 # (modified because settings files are nested one level deeper)
-BASE_DIR: Path = Path(__file__).resolve().parent.parent.parent
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+# Add "src" to Python path
+sys.path.insert(0, os.path.join(BASE_DIR, "src"))
 
 SECRET_KEY = config("SECRET_KEY", default="")
 
@@ -62,7 +66,7 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
-ROOT_URLCONF = "nous_aggregator.urls"
+ROOT_URLCONF = "config.urls"
 
 INTERNAL_IPS = [
     "127.0.0.1",
@@ -84,7 +88,7 @@
     },
 ]
 
-WSGI_APPLICATION = "nous_aggregator.wsgi.application"
+WSGI_APPLICATION = "config.wsgi.application"
 
 
 # Logging
@@ -188,7 +192,7 @@
 CELERY_BEAT_SCHEDULE = {
     "get_articles_en": {
         "task": "articles.tasks.get_articles",
-        "schedule": scraper_tasks.magazines["en"]["schedule"],
+        "schedule": scraper.tasks["magazines"]["en"]["schedule"],
         "kwargs": {
             "language": "en",
         }

nous_aggregator/urls.py → config/urls.py

@@ -1,4 +1,4 @@
-"""nous_aggregator URL Configuration
+"""nous-aggregator URL Configuration
 
 The `urlpatterns` list routes URLs to views. For more information please see:
     https://docs.djangoproject.com/en/4.0/topics/http/urls/

nous_aggregator/wsgi.py → config/wsgi.py

@@ -1,5 +1,5 @@
 """
-WSGI config for nous_aggregator project.
+WSGI config for nous-aggregator project.
 
 It exposes the WSGI callable as a module-level variable named ``application``.
 
@@ -12,6 +12,6 @@
 from django.core.handlers.wsgi import WSGIHandler
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
 
 application: WSGIHandler = get_wsgi_application()

heroku.yml

@@ -12,7 +12,7 @@ release:
   image: web
 
 run:
-  web: gunicorn nous_aggregator.wsgi
+  web: gunicorn config.wsgi
   worker:
     command:
       - python manage.py scrape

manage.py

@@ -6,7 +6,7 @@
 
 def main() -> None:
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

pyproject.toml

@@ -1,6 +1,18 @@
+# mypy
 [tool.mypy]
+python_version = 3.11
 plugins = ["mypy_django_plugin.main"]
-python_executable="home/pi-sigma/.virtualenvs/nous/bin/python"
+ignore_missing_imports=true
 
 [tool.django-stubs]
-django_settings_module = "nous_aggregator.settings.local"
+django_settings_module = "config.settings.local"
+
+# pytest
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "config.settings.local"
+
+# pytype
+[too.pytype]
+python_version = 3.11
+inputs = ["src/articles/management", "src/articles/scraper", "src/articles/*.py"]
+disable = "import-error"