From 0b00000b1ede10ec40a4096e62f4ce569a72ecb2 Mon Sep 17 00:00:00 2001 From: Paul Schilling Date: Thu, 21 Mar 2024 21:47:34 +0100 Subject: [PATCH] Refactor project structure --- .github/workflows/django.yml | 9 +- articles/static/css/main.css | 137 ------------------ {nous_aggregator => config}/__init__.py | 0 {nous_aggregator => config}/asgi.py | 4 +- {nous_aggregator => config}/celery.py | 4 +- config/scraper.py | 21 +++ .../settings/__init__.py | 0 {nous_aggregator => config}/settings/base.py | 14 +- {nous_aggregator => config}/settings/local.py | 0 .../settings/production.py | 0 {nous_aggregator => config}/urls.py | 2 +- {nous_aggregator => config}/wsgi.py | 4 +- heroku.yml | 2 +- manage.py | 2 +- pyproject.toml | 16 +- requirements/pyppeteer_deps.txt | 38 ----- scraper/tasks.py | 19 --- setup.cfg | 35 ----- {articles => src}/__init__.py | 0 .../migrations => src/articles}/__init__.py | 0 {articles => src/articles}/admin.py | 0 {articles => src/articles}/apps.py | 2 +- {articles => src/articles}/constants.py | 0 .../articles}/migrations/0001_initial.py | 0 ...options_alter_article_headline_and_more.py | 0 .../articles/migrations}/__init__.py | 0 {articles => src/articles}/models.py | 0 {articles => src/articles}/tasks.py | 0 .../articles}/templates/articles/base.html | 0 .../articles}/templates/articles/index.html | 0 .../articles}/templates/articles/navbar.html | 0 .../templates/articles/search_results.html | 0 .../articles/tests}/__init__.py | 0 {articles => src/articles}/tests/conftest.py | 0 .../articles/tests/integration}/__init__.py | 0 .../articles}/tests/integration/test_tasks.py | 0 .../articles/tests/unit}/__init__.py | 0 .../articles}/tests/unit/test_models.py | 0 .../articles}/tests/unit/test_views.py | 0 {articles => src/articles}/urls.py | 0 {articles => src/articles}/views.py | 0 {scraper => src/scraper}/__init__.py | 0 {scraper => src/scraper}/headers.py | 0 {scraper => src/scraper}/parser.py | 0 {scraper => src/scraper}/spiders.py | 2 +- {scraper => src/scraper}/tests/__init__.py | 0 {scraper => src/scraper}/tests/conftest.py | 0 .../tests/files/articles/aj/_start.html | 0 .../tests/files/articles/aj/asian_cup.html | 0 .../tests/files/articles/aj/football.html | 0 .../tests/files/articles/aj/footprints.html | 0 .../tests/files/articles/aj/girl.html | 0 .../tests/files/articles/aj/indonesia.html | 0 .../tests/files/articles/aj/israel.html | 0 .../tests/files/articles/aj/israel2.html | 0 .../tests/files/articles/aj/lunar.html | 0 .../tests/files/articles/aj/resistance.html | 0 .../tests/files/articles/aj/russia.html | 0 .../tests/files/articles/aj/student.html | 0 .../tests/files/articles/aj/taiwan.html | 0 .../scraper/tests/integration}/__init__.py | 0 .../scraper}/tests/integration/test_spider.py | 0 {scraper => src/scraper}/tests/mocks.py | 0 src/scraper/tests/unit/__init__.py | 0 .../scraper}/tests/unit/test_parsers.py | 0 .../scraper}/tests/unit/test_spider.py | 0 {scraper => src/scraper}/tests/utils.py | 0 67 files changed, 58 insertions(+), 253 deletions(-) delete mode 100644 articles/static/css/main.css rename {nous_aggregator => config}/__init__.py (100%) rename {nous_aggregator => config}/asgi.py (75%) rename {nous_aggregator => config}/celery.py (53%) create mode 100644 config/scraper.py rename {nous_aggregator => config}/settings/__init__.py (100%) rename {nous_aggregator => config}/settings/base.py (93%) rename {nous_aggregator => config}/settings/local.py (100%) rename {nous_aggregator => config}/settings/production.py (100%) rename {nous_aggregator => config}/urls.py (96%) rename {nous_aggregator => config}/wsgi.py (75%) delete mode 100644 requirements/pyppeteer_deps.txt delete mode 100644 scraper/tasks.py delete mode 100644 setup.cfg rename {articles => src}/__init__.py (100%) rename {articles/migrations => src/articles}/__init__.py (100%) rename {articles => src/articles}/admin.py (100%) rename {articles => src/articles}/apps.py (82%) rename {articles => src/articles}/constants.py (100%) rename {articles => src/articles}/migrations/0001_initial.py (100%) rename {articles => src/articles}/migrations/0002_alter_source_options_alter_article_headline_and_more.py (100%) rename {articles/tests => src/articles/migrations}/__init__.py (100%) rename {articles => src/articles}/models.py (100%) rename {articles => src/articles}/tasks.py (100%) rename {articles => src/articles}/templates/articles/base.html (100%) rename {articles => src/articles}/templates/articles/index.html (100%) rename {articles => src/articles}/templates/articles/navbar.html (100%) rename {articles => src/articles}/templates/articles/search_results.html (100%) rename {articles/tests/integration => src/articles/tests}/__init__.py (100%) rename {articles => src/articles}/tests/conftest.py (100%) rename {articles/tests/unit => src/articles/tests/integration}/__init__.py (100%) rename {articles => src/articles}/tests/integration/test_tasks.py (100%) rename {scraper/tests/integration => src/articles/tests/unit}/__init__.py (100%) rename {articles => src/articles}/tests/unit/test_models.py (100%) rename {articles => src/articles}/tests/unit/test_views.py (100%) rename {articles => src/articles}/urls.py (100%) rename {articles => src/articles}/views.py (100%) rename {scraper => src/scraper}/__init__.py (100%) rename {scraper => src/scraper}/headers.py (100%) rename {scraper => src/scraper}/parser.py (100%) rename {scraper => src/scraper}/spiders.py (98%) rename {scraper => src/scraper}/tests/__init__.py (100%) rename {scraper => src/scraper}/tests/conftest.py (100%) rename {scraper => src/scraper}/tests/files/articles/aj/_start.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/asian_cup.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/football.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/footprints.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/girl.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/indonesia.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/israel.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/israel2.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/lunar.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/resistance.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/russia.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/student.html (100%) rename {scraper => src/scraper}/tests/files/articles/aj/taiwan.html (100%) rename {scraper/tests/unit => src/scraper/tests/integration}/__init__.py (100%) rename {scraper => src/scraper}/tests/integration/test_spider.py (100%) rename {scraper => src/scraper}/tests/mocks.py (100%) create mode 100644 src/scraper/tests/unit/__init__.py rename {scraper => src/scraper}/tests/unit/test_parsers.py (100%) rename {scraper => src/scraper}/tests/unit/test_spider.py (100%) rename {scraper => src/scraper}/tests/utils.py (100%) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 343eb2d..76afdd7 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -19,7 +19,8 @@ jobs: pip install -r requirements/ci.txt - name: Run Bandit run: | - bandit -r articles/ -x tests + bandit -r src/articles/ -x tests + bandit -r src/scraper/ -x tests # # Tests # @@ -58,9 +59,9 @@ jobs: DJANGO_ENV: BASE SECURE_SSL_REDIRECT: False run: | - pytest articles/tests/unit/ - pytest articles/tests/integration/ - pytest scraper/tests/ + pytest src/articles/tests/unit/ + pytest src/articles/tests/integration/ + pytest src/scraper/tests/ # # Migrations diff --git a/articles/static/css/main.css b/articles/static/css/main.css deleted file mode 100644 index 9b2da56..0000000 --- a/articles/static/css/main.css +++ /dev/null @@ -1,137 +0,0 @@ -body { - font-family: sans-serif; - font-size: 14px; - color: #000; -} - -h1,h2,h3,h4,h5 { - font-family: "Ubuntu", sans-serif; -} - -a { - text-decoration: none; -} - -ul { - list-style-type: none; - margin: 0; - padding: 0; -} - -hr { - margin: 0; - padding: 4px 0; -} - - -/* ****** - * Navbar - * ******/ -.navbar { - padding: 0; - background: #333333; - overflow: hidden; - position: fixed; - width: 100%; -} - -.navbar-brand, -.navbar-brand:hover { - margin-right: 4rem; - padding: 5px 7px; - font-family: "Ubuntu", sans-serif; - background: #E83A14; - color: #FFF; -} - -.navbar-nav .nav-item .nav-link { - font-family: "Ubuntu", sans-serif; - color: #FFF; -} - -.search-field, -.search-field:focus { - height: 32px; - box-shadow: none; -} - -.search-button { - height: 32px; - line-height: 15px; - color: #FFF; - border-color: #FFF; -} -.search-button:focus, -.search-button:hover { - border-color: #E83A14; - background: #E83A14; - box-shadow: none!important; -} - -.navbar-toggler { - background: #FFF; -} -.navbar-toggler-icon { - width: 20px; - height: 20px; -} - -.undo-button { - margin-left: 1rem; -} -.fa-undo { - margin-top: 8px; - color: #fff; -} -.fa-undo:hover { - color: #E83A14; -} - -@media (max-width: 990px) { - .search-field { - margin-bottom: 10px; - } -} - -/* ******** - * Articles - * ********/ -.container-sources{ - padding: 4rem 0 2.5rem 0rem; -} - -.container-articles-outer { - max-width: 30rem; - padding: 1rem; -} - -.container-articles-inner { - padding: 0; - max-height: 18rem; -} - -/* scrolling */ -.container-scrolling { - -ms-overflow-style: none; /* for Internet Explorer, Edge */ - scrollbar-width: none; /* for Firefox */ - overflow-y: scroll; -} -.container-scrolling::-webkit-scrollbar { - display: none; /* for Chrome, Safari, Opera */ -} - -.source-name { - font-weight: 600; - font-size: 19px; - color: #E83A14; -} - -.article-link { - color: #000; -} - -.article-headline { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} diff --git a/nous_aggregator/__init__.py b/config/__init__.py similarity index 100% rename from nous_aggregator/__init__.py rename to config/__init__.py diff --git a/nous_aggregator/asgi.py b/config/asgi.py similarity index 75% rename from nous_aggregator/asgi.py rename to config/asgi.py index 751c449..ca3c31e 100644 --- a/nous_aggregator/asgi.py +++ b/config/asgi.py @@ -1,5 +1,5 @@ """ -ASGI config for nous_aggregator project. +ASGI config for nous-aggregator project. It exposes the ASGI callable as a module-level variable named ``application``. @@ -12,6 +12,6 @@ from django.core.asgi import get_asgi_application from django.core.handlers.asgi import ASGIHandler -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') application: ASGIHandler = get_asgi_application() diff --git a/nous_aggregator/celery.py b/config/celery.py similarity index 53% rename from nous_aggregator/celery.py rename to config/celery.py index fd5d514..cfd95e5 100644 --- a/nous_aggregator/celery.py +++ b/config/celery.py @@ -2,9 +2,9 @@ from celery import Celery -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "nous_aggregator.settings.local") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") -app = Celery("nous_aggregator") +app = Celery("config") app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/config/scraper.py b/config/scraper.py new file mode 100644 index 0000000..0113853 --- /dev/null +++ b/config/scraper.py @@ -0,0 +1,21 @@ +tasks = { + "magazines": { + "en": { + "schedule": 120, + "titles": [ + "Al Jazeera", + "Associated Press", + "Christian Science Monitor", + "Consortium News", + "Current Affairs", + "New York Times", + "NPR", + "Reuters", + "The Atlantic", + "The Intercept", + "UPI", + "Wall Street Journal", + ] + }, + }, +} diff --git a/nous_aggregator/settings/__init__.py b/config/settings/__init__.py similarity index 100% rename from nous_aggregator/settings/__init__.py rename to config/settings/__init__.py diff --git a/nous_aggregator/settings/base.py b/config/settings/base.py similarity index 93% rename from nous_aggregator/settings/base.py rename to config/settings/base.py index e83a2e4..01d65f9 100644 --- a/nous_aggregator/settings/base.py +++ b/config/settings/base.py @@ -1,14 +1,14 @@ import os from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Dict from decouple import Csv, config -from scraper import tasks as scraper_tasks +from .. import scraper # Build paths inside the project like this: BASE_DIR / 'subdir'. # (modified because settings files are nested one level deeper) -BASE_DIR: Path = Path(__file__).resolve().parent.parent.parent +BASE_DIR = Path(__file__).resolve().parent.parent.parent SECRET_KEY = config("SECRET_KEY", default="") @@ -49,7 +49,7 @@ "django.contrib.staticfiles", "django.contrib.humanize", # nous_aggregator apps - "articles.apps.ArticlesConfig", + "src.articles.apps.ArticlesConfig", ] MIDDLEWARE = [ @@ -62,7 +62,7 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = "nous_aggregator.urls" +ROOT_URLCONF = "config.urls" INTERNAL_IPS = [ "127.0.0.1", @@ -84,7 +84,7 @@ }, ] -WSGI_APPLICATION = "nous_aggregator.wsgi.application" +WSGI_APPLICATION = "config.wsgi.application" # Logging @@ -188,7 +188,7 @@ CELERY_BEAT_SCHEDULE = { "get_articles_en": { "task": "articles.tasks.get_articles", - "schedule": scraper_tasks.magazines["en"]["schedule"], + "schedule": scraper.tasks["magazines"]["en"]["schedule"], "kwargs": { "language": "en", } diff --git a/nous_aggregator/settings/local.py b/config/settings/local.py similarity index 100% rename from nous_aggregator/settings/local.py rename to config/settings/local.py diff --git a/nous_aggregator/settings/production.py b/config/settings/production.py similarity index 100% rename from nous_aggregator/settings/production.py rename to config/settings/production.py diff --git a/nous_aggregator/urls.py b/config/urls.py similarity index 96% rename from nous_aggregator/urls.py rename to config/urls.py index 2523e9d..1c93a18 100644 --- a/nous_aggregator/urls.py +++ b/config/urls.py @@ -1,4 +1,4 @@ -"""nous_aggregator URL Configuration +"""nous-aggregator URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/4.0/topics/http/urls/ diff --git a/nous_aggregator/wsgi.py b/config/wsgi.py similarity index 75% rename from nous_aggregator/wsgi.py rename to config/wsgi.py index 3c4105a..9dd7450 100644 --- a/nous_aggregator/wsgi.py +++ b/config/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for nous_aggregator project. +WSGI config for nous-aggregator project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -12,6 +12,6 @@ from django.core.handlers.wsgi import WSGIHandler from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') application: WSGIHandler = get_wsgi_application() diff --git a/heroku.yml b/heroku.yml index 85e1991..56cca5b 100644 --- a/heroku.yml +++ b/heroku.yml @@ -12,7 +12,7 @@ release: image: web run: - web: gunicorn nous_aggregator.wsgi + web: gunicorn config.wsgi worker: command: - python manage.py scrape diff --git a/manage.py b/manage.py index 5b2728f..17ccd1b 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,7 @@ def main() -> None: """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/pyproject.toml b/pyproject.toml index b335971..91438d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,18 @@ +# mypy [tool.mypy] +python_version = 3.11 plugins = ["mypy_django_plugin.main"] -python_executable="home/pi-sigma/.virtualenvs/nous/bin/python" +ignore_missing_imports=true [tool.django-stubs] -django_settings_module = "nous_aggregator.settings.local" +django_settings_module = "config.settings.local" + +# pytest +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "config.settings.local" + +# pytype +[too.pytype] +python_version = 3.11 +inputs = ["src/articles/management", "src/articles/scraper", "src/articles/*.py"] +disable = "import-error" diff --git a/requirements/pyppeteer_deps.txt b/requirements/pyppeteer_deps.txt deleted file mode 100644 index 10cce9d..0000000 --- a/requirements/pyppeteer_deps.txt +++ /dev/null @@ -1,38 +0,0 @@ -gconf-service -libasound2 -libatk1.0-0 -libc6 -libcairo2 -libcups2 -libdbus-1-3 -libexpat1 -libfontconfig1 -libgcc1 -libgconf-2-4 -libgdk-pixbuf2.0-0 -libglib2.0-0 -libgtk-3-0 -libnspr4 -libpango-1.0-0 -libpangocairo-1.0-0 -libstdc++6 -libx11-6 -libx11-xcb1 -libxcb1 -libxcomposite1 -libxcursor1 -libxdamage1 -libxext6 -libxfixes3 -libxi6 -libxrandr2 -libxrender1 -libxss1 -libxtst6 -ca-certificates -fonts-liberation -libappindicator1 -libnss3 -lsb-release -xdg-utils -wget diff --git a/scraper/tasks.py b/scraper/tasks.py deleted file mode 100644 index c13fd7d..0000000 --- a/scraper/tasks.py +++ /dev/null @@ -1,19 +0,0 @@ -magazines = { - "en": { - "schedule": 120, - "titles": [ - "Al Jazeera", - "Associated Press", - "Christian Science Monitor", - "Consortium News", - "Current Affairs", - "New York Times", - "NPR", - "Reuters", - "The Atlantic", - "The Intercept", - "UPI", - "Wall Street Journal", - ] - } -} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index db6fe90..0000000 --- a/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[pytype] -python_version = 3.10 -inputs = - articles/management - articles/scraper - articles/*.py - -disable = - import-error - -[mypy] -python_version = 3.10 -files = articles/*.py, articles/scraper/*.py, articles/management/commands/*.py -plugins = - mypy_django_plugin.main -ignore_missing_imports=true - -# mypy: stubs -[mypy.plugins.django-stubs] -django_settings_module = "nous_aggregator.settings" - -# mypy: ignore imports -[mypy-apscheduler.*] -ignore_missing_imports = True -[mypy-django_apscheduler.*] -ignore_missing_imports = True - -[flake8] -max-line-length = 120 -exclude = tests/*, articles/migrations/*, settings/*, sitemaps.py, headers.py -max-complexity = 10 -ignore = W391, E116 - -[tool:pytest] -DJANGO_SETTINGS_MODULE = nous_aggregator.settings diff --git a/articles/__init__.py b/src/__init__.py similarity index 100% rename from articles/__init__.py rename to src/__init__.py diff --git a/articles/migrations/__init__.py b/src/articles/__init__.py similarity index 100% rename from articles/migrations/__init__.py rename to src/articles/__init__.py diff --git a/articles/admin.py b/src/articles/admin.py similarity index 100% rename from articles/admin.py rename to src/articles/admin.py diff --git a/articles/apps.py b/src/articles/apps.py similarity index 82% rename from articles/apps.py rename to src/articles/apps.py index 9baf7c9..cab2063 100644 --- a/articles/apps.py +++ b/src/articles/apps.py @@ -3,4 +3,4 @@ class ArticlesConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "articles" + name = "src.articles" diff --git a/articles/constants.py b/src/articles/constants.py similarity index 100% rename from articles/constants.py rename to src/articles/constants.py diff --git a/articles/migrations/0001_initial.py b/src/articles/migrations/0001_initial.py similarity index 100% rename from articles/migrations/0001_initial.py rename to src/articles/migrations/0001_initial.py diff --git a/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py b/src/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py similarity index 100% rename from articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py rename to src/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py diff --git a/articles/tests/__init__.py b/src/articles/migrations/__init__.py similarity index 100% rename from articles/tests/__init__.py rename to src/articles/migrations/__init__.py diff --git a/articles/models.py b/src/articles/models.py similarity index 100% rename from articles/models.py rename to src/articles/models.py diff --git a/articles/tasks.py b/src/articles/tasks.py similarity index 100% rename from articles/tasks.py rename to src/articles/tasks.py diff --git a/articles/templates/articles/base.html b/src/articles/templates/articles/base.html similarity index 100% rename from articles/templates/articles/base.html rename to src/articles/templates/articles/base.html diff --git a/articles/templates/articles/index.html b/src/articles/templates/articles/index.html similarity index 100% rename from articles/templates/articles/index.html rename to src/articles/templates/articles/index.html diff --git a/articles/templates/articles/navbar.html b/src/articles/templates/articles/navbar.html similarity index 100% rename from articles/templates/articles/navbar.html rename to src/articles/templates/articles/navbar.html diff --git a/articles/templates/articles/search_results.html b/src/articles/templates/articles/search_results.html similarity index 100% rename from articles/templates/articles/search_results.html rename to src/articles/templates/articles/search_results.html diff --git a/articles/tests/integration/__init__.py b/src/articles/tests/__init__.py similarity index 100% rename from articles/tests/integration/__init__.py rename to src/articles/tests/__init__.py diff --git a/articles/tests/conftest.py b/src/articles/tests/conftest.py similarity index 100% rename from articles/tests/conftest.py rename to src/articles/tests/conftest.py diff --git a/articles/tests/unit/__init__.py b/src/articles/tests/integration/__init__.py similarity index 100% rename from articles/tests/unit/__init__.py rename to src/articles/tests/integration/__init__.py diff --git a/articles/tests/integration/test_tasks.py b/src/articles/tests/integration/test_tasks.py similarity index 100% rename from articles/tests/integration/test_tasks.py rename to src/articles/tests/integration/test_tasks.py diff --git a/scraper/tests/integration/__init__.py b/src/articles/tests/unit/__init__.py similarity index 100% rename from scraper/tests/integration/__init__.py rename to src/articles/tests/unit/__init__.py diff --git a/articles/tests/unit/test_models.py b/src/articles/tests/unit/test_models.py similarity index 100% rename from articles/tests/unit/test_models.py rename to src/articles/tests/unit/test_models.py diff --git a/articles/tests/unit/test_views.py b/src/articles/tests/unit/test_views.py similarity index 100% rename from articles/tests/unit/test_views.py rename to src/articles/tests/unit/test_views.py diff --git a/articles/urls.py b/src/articles/urls.py similarity index 100% rename from articles/urls.py rename to src/articles/urls.py diff --git a/articles/views.py b/src/articles/views.py similarity index 100% rename from articles/views.py rename to src/articles/views.py diff --git a/scraper/__init__.py b/src/scraper/__init__.py similarity index 100% rename from scraper/__init__.py rename to src/scraper/__init__.py diff --git a/scraper/headers.py b/src/scraper/headers.py similarity index 100% rename from scraper/headers.py rename to src/scraper/headers.py diff --git a/scraper/parser.py b/src/scraper/parser.py similarity index 100% rename from scraper/parser.py rename to src/scraper/parser.py diff --git a/scraper/spiders.py b/src/scraper/spiders.py similarity index 98% rename from scraper/spiders.py rename to src/scraper/spiders.py index ccb766f..46a664b 100644 --- a/scraper/spiders.py +++ b/src/scraper/spiders.py @@ -34,7 +34,7 @@ def __init__(self, starting_urls: list[str], sitemap: dict) -> None: self.articles: set[str] = set() async def connect(self, session: ClientSession, url: str) -> str | None: - headers = random.choice(self.headers) + headers = random.choice(self.headers) # nosec try: async with session.get(url, headers=headers) as response: diff --git a/scraper/tests/__init__.py b/src/scraper/tests/__init__.py similarity index 100% rename from scraper/tests/__init__.py rename to src/scraper/tests/__init__.py diff --git a/scraper/tests/conftest.py b/src/scraper/tests/conftest.py similarity index 100% rename from scraper/tests/conftest.py rename to src/scraper/tests/conftest.py diff --git a/scraper/tests/files/articles/aj/_start.html b/src/scraper/tests/files/articles/aj/_start.html similarity index 100% rename from scraper/tests/files/articles/aj/_start.html rename to src/scraper/tests/files/articles/aj/_start.html diff --git a/scraper/tests/files/articles/aj/asian_cup.html b/src/scraper/tests/files/articles/aj/asian_cup.html similarity index 100% rename from scraper/tests/files/articles/aj/asian_cup.html rename to src/scraper/tests/files/articles/aj/asian_cup.html diff --git a/scraper/tests/files/articles/aj/football.html b/src/scraper/tests/files/articles/aj/football.html similarity index 100% rename from scraper/tests/files/articles/aj/football.html rename to src/scraper/tests/files/articles/aj/football.html diff --git a/scraper/tests/files/articles/aj/footprints.html b/src/scraper/tests/files/articles/aj/footprints.html similarity index 100% rename from scraper/tests/files/articles/aj/footprints.html rename to src/scraper/tests/files/articles/aj/footprints.html diff --git a/scraper/tests/files/articles/aj/girl.html b/src/scraper/tests/files/articles/aj/girl.html similarity index 100% rename from scraper/tests/files/articles/aj/girl.html rename to src/scraper/tests/files/articles/aj/girl.html diff --git a/scraper/tests/files/articles/aj/indonesia.html b/src/scraper/tests/files/articles/aj/indonesia.html similarity index 100% rename from scraper/tests/files/articles/aj/indonesia.html rename to src/scraper/tests/files/articles/aj/indonesia.html diff --git a/scraper/tests/files/articles/aj/israel.html b/src/scraper/tests/files/articles/aj/israel.html similarity index 100% rename from scraper/tests/files/articles/aj/israel.html rename to src/scraper/tests/files/articles/aj/israel.html diff --git a/scraper/tests/files/articles/aj/israel2.html b/src/scraper/tests/files/articles/aj/israel2.html similarity index 100% rename from scraper/tests/files/articles/aj/israel2.html rename to src/scraper/tests/files/articles/aj/israel2.html diff --git a/scraper/tests/files/articles/aj/lunar.html b/src/scraper/tests/files/articles/aj/lunar.html similarity index 100% rename from scraper/tests/files/articles/aj/lunar.html rename to src/scraper/tests/files/articles/aj/lunar.html diff --git a/scraper/tests/files/articles/aj/resistance.html b/src/scraper/tests/files/articles/aj/resistance.html similarity index 100% rename from scraper/tests/files/articles/aj/resistance.html rename to src/scraper/tests/files/articles/aj/resistance.html diff --git a/scraper/tests/files/articles/aj/russia.html b/src/scraper/tests/files/articles/aj/russia.html similarity index 100% rename from scraper/tests/files/articles/aj/russia.html rename to src/scraper/tests/files/articles/aj/russia.html diff --git a/scraper/tests/files/articles/aj/student.html b/src/scraper/tests/files/articles/aj/student.html similarity index 100% rename from scraper/tests/files/articles/aj/student.html rename to src/scraper/tests/files/articles/aj/student.html diff --git a/scraper/tests/files/articles/aj/taiwan.html b/src/scraper/tests/files/articles/aj/taiwan.html similarity index 100% rename from scraper/tests/files/articles/aj/taiwan.html rename to src/scraper/tests/files/articles/aj/taiwan.html diff --git a/scraper/tests/unit/__init__.py b/src/scraper/tests/integration/__init__.py similarity index 100% rename from scraper/tests/unit/__init__.py rename to src/scraper/tests/integration/__init__.py diff --git a/scraper/tests/integration/test_spider.py b/src/scraper/tests/integration/test_spider.py similarity index 100% rename from scraper/tests/integration/test_spider.py rename to src/scraper/tests/integration/test_spider.py diff --git a/scraper/tests/mocks.py b/src/scraper/tests/mocks.py similarity index 100% rename from scraper/tests/mocks.py rename to src/scraper/tests/mocks.py diff --git a/src/scraper/tests/unit/__init__.py b/src/scraper/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scraper/tests/unit/test_parsers.py b/src/scraper/tests/unit/test_parsers.py similarity index 100% rename from scraper/tests/unit/test_parsers.py rename to src/scraper/tests/unit/test_parsers.py diff --git a/scraper/tests/unit/test_spider.py b/src/scraper/tests/unit/test_spider.py similarity index 100% rename from scraper/tests/unit/test_spider.py rename to src/scraper/tests/unit/test_spider.py diff --git a/scraper/tests/utils.py b/src/scraper/tests/utils.py similarity index 100% rename from scraper/tests/utils.py rename to src/scraper/tests/utils.py