diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..28bf22c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +static/* linguist-vendored +tests/* linguist-vendored +*.html linguist-detectable=false diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 343eb2d..76afdd7 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -19,7 +19,8 @@ jobs: pip install -r requirements/ci.txt - name: Run Bandit run: | - bandit -r articles/ -x tests + bandit -r src/articles/ -x tests + bandit -r src/scraper/ -x tests # # Tests # @@ -58,9 +59,9 @@ jobs: DJANGO_ENV: BASE SECURE_SSL_REDIRECT: False run: | - pytest articles/tests/unit/ - pytest articles/tests/integration/ - pytest scraper/tests/ + pytest src/articles/tests/unit/ + pytest src/articles/tests/integration/ + pytest src/scraper/tests/ # # Migrations diff --git a/articles/static/css/main.css b/articles/static/css/main.css deleted file mode 100644 index 9b2da56..0000000 --- a/articles/static/css/main.css +++ /dev/null @@ -1,137 +0,0 @@ -body { - font-family: sans-serif; - font-size: 14px; - color: #000; -} - -h1,h2,h3,h4,h5 { - font-family: "Ubuntu", sans-serif; -} - -a { - text-decoration: none; -} - -ul { - list-style-type: none; - margin: 0; - padding: 0; -} - -hr { - margin: 0; - padding: 4px 0; -} - - -/* ****** - * Navbar - * ******/ -.navbar { - padding: 0; - background: #333333; - overflow: hidden; - position: fixed; - width: 100%; -} - -.navbar-brand, -.navbar-brand:hover { - margin-right: 4rem; - padding: 5px 7px; - font-family: "Ubuntu", sans-serif; - background: #E83A14; - color: #FFF; -} - -.navbar-nav .nav-item .nav-link { - font-family: "Ubuntu", sans-serif; - color: #FFF; -} - -.search-field, -.search-field:focus { - height: 32px; - box-shadow: none; -} - -.search-button { - height: 32px; - line-height: 15px; - color: #FFF; - border-color: #FFF; -} -.search-button:focus, -.search-button:hover { - border-color: #E83A14; - background: #E83A14; - box-shadow: none!important; -} - -.navbar-toggler { - background: #FFF; -} -.navbar-toggler-icon { - width: 20px; - height: 20px; -} - -.undo-button { - margin-left: 1rem; -} -.fa-undo { - margin-top: 8px; - color: #fff; -} -.fa-undo:hover { - color: #E83A14; -} - -@media (max-width: 990px) { - .search-field { - margin-bottom: 10px; - } -} - -/* ******** - * Articles - * ********/ -.container-sources{ - padding: 4rem 0 2.5rem 0rem; -} - -.container-articles-outer { - max-width: 30rem; - padding: 1rem; -} - -.container-articles-inner { - padding: 0; - max-height: 18rem; -} - -/* scrolling */ -.container-scrolling { - -ms-overflow-style: none; /* for Internet Explorer, Edge */ - scrollbar-width: none; /* for Firefox */ - overflow-y: scroll; -} -.container-scrolling::-webkit-scrollbar { - display: none; /* for Chrome, Safari, Opera */ -} - -.source-name { - font-weight: 600; - font-size: 19px; - color: #E83A14; -} - -.article-link { - color: #000; -} - -.article-headline { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} diff --git a/nous_aggregator/__init__.py b/config/__init__.py similarity index 100% rename from nous_aggregator/__init__.py rename to config/__init__.py diff --git a/nous_aggregator/asgi.py b/config/asgi.py similarity index 75% rename from nous_aggregator/asgi.py rename to config/asgi.py index 751c449..ca3c31e 100644 --- a/nous_aggregator/asgi.py +++ b/config/asgi.py @@ -1,5 +1,5 @@ """ -ASGI config for nous_aggregator project. +ASGI config for nous-aggregator project. It exposes the ASGI callable as a module-level variable named ``application``. @@ -12,6 +12,6 @@ from django.core.asgi import get_asgi_application from django.core.handlers.asgi import ASGIHandler -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') application: ASGIHandler = get_asgi_application() diff --git a/nous_aggregator/celery.py b/config/celery.py similarity index 53% rename from nous_aggregator/celery.py rename to config/celery.py index fd5d514..cfd95e5 100644 --- a/nous_aggregator/celery.py +++ b/config/celery.py @@ -2,9 +2,9 @@ from celery import Celery -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "nous_aggregator.settings.local") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") -app = Celery("nous_aggregator") +app = Celery("config") app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/config/scraper.py b/config/scraper.py new file mode 100644 index 0000000..0113853 --- /dev/null +++ b/config/scraper.py @@ -0,0 +1,21 @@ +tasks = { + "magazines": { + "en": { + "schedule": 120, + "titles": [ + "Al Jazeera", + "Associated Press", + "Christian Science Monitor", + "Consortium News", + "Current Affairs", + "New York Times", + "NPR", + "Reuters", + "The Atlantic", + "The Intercept", + "UPI", + "Wall Street Journal", + ] + }, + }, +} diff --git a/nous_aggregator/settings/__init__.py b/config/settings/__init__.py similarity index 89% rename from nous_aggregator/settings/__init__.py rename to config/settings/__init__.py index df4bafe..a06216a 100644 --- a/nous_aggregator/settings/__init__.py +++ b/config/settings/__init__.py @@ -10,9 +10,11 @@ DJANGO_ENV = config('DJANGO_ENV', default="") match DJANGO_ENV: + case "": + from .base import * case "BASE": from .base import * case "LOCAL": from .local import * - case "": + case "PRODUCTION": from .production import * diff --git a/nous_aggregator/settings/base.py b/config/settings/base.py similarity index 90% rename from nous_aggregator/settings/base.py rename to config/settings/base.py index e83a2e4..09ec0e5 100644 --- a/nous_aggregator/settings/base.py +++ b/config/settings/base.py @@ -1,14 +1,17 @@ import os +import sys from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Dict from decouple import Csv, config -from scraper import tasks as scraper_tasks +from .. import scraper -# Build paths inside the project like this: BASE_DIR / 'subdir'. -# (modified because settings files are nested one level deeper) -BASE_DIR: Path = Path(__file__).resolve().parent.parent.parent +BASE_DIR = Path(__file__).resolve().parent.parent.parent + +# Add "src" to Python path +PROJECT_DIR = os.path.join(BASE_DIR, "src") +sys.path.insert(0, PROJECT_DIR) SECRET_KEY = config("SECRET_KEY", default="") @@ -62,7 +65,7 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = "nous_aggregator.urls" +ROOT_URLCONF = "config.urls" INTERNAL_IPS = [ "127.0.0.1", @@ -84,7 +87,7 @@ }, ] -WSGI_APPLICATION = "nous_aggregator.wsgi.application" +WSGI_APPLICATION = "config.wsgi.application" # Logging @@ -169,9 +172,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.0/howto/static-files/ - -STATIC_URL = "static/" -STATIC_ROOT: str = os.path.join(BASE_DIR, 'static') +STATIC_URL: str = "static/" +STATIC_ROOT: str = f"{PROJECT_DIR}/staticfiles" # Default primary key field type @@ -188,7 +190,7 @@ CELERY_BEAT_SCHEDULE = { "get_articles_en": { "task": "articles.tasks.get_articles", - "schedule": scraper_tasks.magazines["en"]["schedule"], + "schedule": scraper.tasks["magazines"]["en"]["schedule"], "kwargs": { "language": "en", } diff --git a/nous_aggregator/settings/local.py b/config/settings/local.py similarity index 100% rename from nous_aggregator/settings/local.py rename to config/settings/local.py diff --git a/nous_aggregator/settings/production.py b/config/settings/production.py similarity index 100% rename from nous_aggregator/settings/production.py rename to config/settings/production.py diff --git a/nous_aggregator/urls.py b/config/urls.py similarity index 91% rename from nous_aggregator/urls.py rename to config/urls.py index 2523e9d..3e82492 100644 --- a/nous_aggregator/urls.py +++ b/config/urls.py @@ -1,4 +1,4 @@ -"""nous_aggregator URL Configuration +"""nous-aggregator URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/4.0/topics/http/urls/ @@ -15,7 +15,7 @@ """ from typing import List -from django.conf import settings +from decouple import config from django.contrib import admin from django.urls import include, path from django.urls.resolvers import URLResolver @@ -28,7 +28,7 @@ ] -if settings.DEBUG is True: +if config("DJANGO_ENV") == "LOCAL": urlpatterns += [ path('__debug__/', include('debug_toolbar.urls')), ] diff --git a/nous_aggregator/wsgi.py b/config/wsgi.py similarity index 75% rename from nous_aggregator/wsgi.py rename to config/wsgi.py index 3c4105a..9dd7450 100644 --- a/nous_aggregator/wsgi.py +++ b/config/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for nous_aggregator project. +WSGI config for nous-aggregator project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -12,6 +12,6 @@ from django.core.handlers.wsgi import WSGIHandler from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') application: WSGIHandler = get_wsgi_application() diff --git a/heroku.yml b/heroku.yml index 85e1991..56cca5b 100644 --- a/heroku.yml +++ b/heroku.yml @@ -12,7 +12,7 @@ release: image: web run: - web: gunicorn nous_aggregator.wsgi + web: gunicorn config.wsgi worker: command: - python manage.py scrape diff --git a/manage.py b/manage.py index 5b2728f..17ccd1b 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,7 @@ def main() -> None: """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nous_aggregator.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/pyproject.toml b/pyproject.toml index b335971..1d2880c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,18 @@ +# mypy [tool.mypy] +python_version = 3.11 plugins = ["mypy_django_plugin.main"] -python_executable="home/pi-sigma/.virtualenvs/nous/bin/python" +ignore_missing_imports=true [tool.django-stubs] -django_settings_module = "nous_aggregator.settings.local" +django_settings_module = "config.settings" + +# pytest +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "config.settings" + +# pytype +[too.pytype] +python_version = 3.11 +inputs = ["src/articles/management", "src/articles/scraper", "src/articles/*.py"] +disable = "import-error" diff --git a/requirements/dev.in b/requirements/dev.in index c95c0db..079367f 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -7,7 +7,7 @@ black flake8 isort -# Debug tooling +# Development & debug tooling django-debug-toolbar django-extensions ipython diff --git a/requirements/dev.txt b/requirements/dev.txt index 1187e92..4391cc9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -28,6 +28,8 @@ asgiref==3.7.2 # -c requirements/ci.txt # -r requirements/ci.txt # django +asttokens==2.4.1 + # via stack-data attrs==23.2.0 # via # -c requirements/ci.txt @@ -88,6 +90,8 @@ cssselect==1.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # pyquery +decorator==5.1.1 + # via ipython dj-database-url==2.1.0 # via # -c requirements/ci.txt @@ -115,6 +119,8 @@ django-stubs==4.2.4 # via -r requirements/dev.in django-stubs-ext==4.2.2 # via django-stubs +executing==2.0.1 + # via stack-data flake8==6.1.0 # via -r requirements/dev.in frozenlist==1.4.1 @@ -154,8 +160,12 @@ iniconfig==2.0.0 # -c requirements/ci.txt # -r requirements/ci.txt # pytest +ipython==8.22.2 + # via -r requirements/dev.in isort==5.12.0 # via -r requirements/dev.in +jedi==0.19.1 + # via ipython jinja2==3.1.3 # via pytype kombu==5.3.5 @@ -181,6 +191,8 @@ markdown-it-py==3.0.0 # rich markupsafe==2.1.3 # via jinja2 +matplotlib-inline==0.1.6 + # via ipython mccabe==0.7.0 # via flake8 mdurl==0.1.2 @@ -217,6 +229,8 @@ packaging==23.2 # build # gunicorn # pytest +parso==0.8.3 + # via jedi pathspec==0.11.2 # via black pbr==5.11.1 @@ -224,6 +238,10 @@ pbr==5.11.1 # -c requirements/ci.txt # -r requirements/ci.txt # stevedore +pdbr==0.8.8 + # via -r requirements/dev.in +pexpect==4.9.0 + # via ipython pip-tools==7.4.1 # via -r requirements/dev.in platformdirs==3.10.0 @@ -238,6 +256,7 @@ prompt-toolkit==3.0.43 # -c requirements/ci.txt # -r requirements/ci.txt # click-repl + # ipython psycopg2==2.9.9 # via # -c requirements/ci.txt @@ -247,6 +266,10 @@ psycopg2-binary==2.9.7 # -c requirements/ci.txt # -r requirements/ci.txt # django-on-heroku +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data pycnite==2023.10.11 # via pytype pycodestyle==2.11.0 @@ -264,6 +287,7 @@ pygments==2.16.1 # via # -c requirements/ci.txt # -r requirements/ci.txt + # ipython # rich pyparsing==3.1.1 # via pydot @@ -332,6 +356,7 @@ rich==13.5.3 # -c requirements/ci.txt # -r requirements/ci.txt # bandit + # pdbr sentry-sdk==1.39.2 # via # -c requirements/ci.txt @@ -340,6 +365,7 @@ six==1.16.0 # via # -c requirements/ci.txt # -r requirements/ci.txt + # asttokens # langdetect # python-dateutil smmap==5.0.1 @@ -353,6 +379,8 @@ sqlparse==0.4.4 # -r requirements/ci.txt # django # django-debug-toolbar +stack-data==0.6.3 + # via ipython stevedore==5.1.0 # via # -c requirements/ci.txt @@ -367,6 +395,10 @@ tqdm==4.66.1 # -c requirements/ci.txt # -r requirements/ci.txt # pyppeteer +traitlets==5.14.2 + # via + # ipython + # matplotlib-inline types-pytz==2023.3.0.1 # via django-stubs types-pyyaml==6.0.12.11 diff --git a/requirements/pyppeteer_deps.txt b/requirements/pyppeteer_deps.txt deleted file mode 100644 index 10cce9d..0000000 --- a/requirements/pyppeteer_deps.txt +++ /dev/null @@ -1,38 +0,0 @@ -gconf-service -libasound2 -libatk1.0-0 -libc6 -libcairo2 -libcups2 -libdbus-1-3 -libexpat1 -libfontconfig1 -libgcc1 -libgconf-2-4 -libgdk-pixbuf2.0-0 -libglib2.0-0 -libgtk-3-0 -libnspr4 -libpango-1.0-0 -libpangocairo-1.0-0 -libstdc++6 -libx11-6 -libx11-xcb1 -libxcb1 -libxcomposite1 -libxcursor1 -libxdamage1 -libxext6 -libxfixes3 -libxi6 -libxrandr2 -libxrender1 -libxss1 -libxtst6 -ca-certificates -fonts-liberation -libappindicator1 -libnss3 -lsb-release -xdg-utils -wget diff --git a/scraper/tasks.py b/scraper/tasks.py deleted file mode 100644 index c13fd7d..0000000 --- a/scraper/tasks.py +++ /dev/null @@ -1,19 +0,0 @@ -magazines = { - "en": { - "schedule": 120, - "titles": [ - "Al Jazeera", - "Associated Press", - "Christian Science Monitor", - "Consortium News", - "Current Affairs", - "New York Times", - "NPR", - "Reuters", - "The Atlantic", - "The Intercept", - "UPI", - "Wall Street Journal", - ] - } -} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index db6fe90..0000000 --- a/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[pytype] -python_version = 3.10 -inputs = - articles/management - articles/scraper - articles/*.py - -disable = - import-error - -[mypy] -python_version = 3.10 -files = articles/*.py, articles/scraper/*.py, articles/management/commands/*.py -plugins = - mypy_django_plugin.main -ignore_missing_imports=true - -# mypy: stubs -[mypy.plugins.django-stubs] -django_settings_module = "nous_aggregator.settings" - -# mypy: ignore imports -[mypy-apscheduler.*] -ignore_missing_imports = True -[mypy-django_apscheduler.*] -ignore_missing_imports = True - -[flake8] -max-line-length = 120 -exclude = tests/*, articles/migrations/*, settings/*, sitemaps.py, headers.py -max-complexity = 10 -ignore = W391, E116 - -[tool:pytest] -DJANGO_SETTINGS_MODULE = nous_aggregator.settings diff --git a/articles/__init__.py b/src/articles/__init__.py similarity index 100% rename from articles/__init__.py rename to src/articles/__init__.py diff --git a/articles/admin.py b/src/articles/admin.py similarity index 100% rename from articles/admin.py rename to src/articles/admin.py diff --git a/articles/apps.py b/src/articles/apps.py similarity index 100% rename from articles/apps.py rename to src/articles/apps.py diff --git a/articles/constants.py b/src/articles/constants.py similarity index 100% rename from articles/constants.py rename to src/articles/constants.py diff --git a/articles/migrations/0001_initial.py b/src/articles/migrations/0001_initial.py similarity index 100% rename from articles/migrations/0001_initial.py rename to src/articles/migrations/0001_initial.py diff --git a/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py b/src/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py similarity index 100% rename from articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py rename to src/articles/migrations/0002_alter_source_options_alter_article_headline_and_more.py diff --git a/articles/migrations/__init__.py b/src/articles/migrations/__init__.py similarity index 100% rename from articles/migrations/__init__.py rename to src/articles/migrations/__init__.py diff --git a/articles/models.py b/src/articles/models.py similarity index 100% rename from articles/models.py rename to src/articles/models.py diff --git a/articles/tasks.py b/src/articles/tasks.py similarity index 95% rename from articles/tasks.py rename to src/articles/tasks.py index f37f874..22b068f 100644 --- a/articles/tasks.py +++ b/src/articles/tasks.py @@ -6,7 +6,7 @@ from django.utils import timezone import scraper -from scraper.tasks import magazines +from config.scraper import tasks as scraper_tasks from .models import Article, Source @@ -58,7 +58,7 @@ def get_articles_for_source(source_title: str) -> None: @shared_task def get_articles(language: str): task_group = group( - get_articles_for_source.s(source_title=title) for title in magazines[language]["titles"] + get_articles_for_source.s(source_title=title) for title in scraper_tasks["magazines"][language]["titles"] ) promise = task_group.apply_async() if promise.ready(): diff --git a/articles/templates/articles/base.html b/src/articles/templates/articles/base.html similarity index 100% rename from articles/templates/articles/base.html rename to src/articles/templates/articles/base.html diff --git a/articles/templates/articles/index.html b/src/articles/templates/articles/index.html similarity index 100% rename from articles/templates/articles/index.html rename to src/articles/templates/articles/index.html diff --git a/articles/templates/articles/navbar.html b/src/articles/templates/articles/navbar.html similarity index 100% rename from articles/templates/articles/navbar.html rename to src/articles/templates/articles/navbar.html diff --git a/articles/templates/articles/search_results.html b/src/articles/templates/articles/search_results.html similarity index 100% rename from articles/templates/articles/search_results.html rename to src/articles/templates/articles/search_results.html diff --git a/articles/tests/__init__.py b/src/articles/tests/__init__.py similarity index 100% rename from articles/tests/__init__.py rename to src/articles/tests/__init__.py diff --git a/articles/tests/conftest.py b/src/articles/tests/conftest.py similarity index 100% rename from articles/tests/conftest.py rename to src/articles/tests/conftest.py diff --git a/articles/tests/integration/__init__.py b/src/articles/tests/integration/__init__.py similarity index 100% rename from articles/tests/integration/__init__.py rename to src/articles/tests/integration/__init__.py diff --git a/articles/tests/integration/test_tasks.py b/src/articles/tests/integration/test_tasks.py similarity index 100% rename from articles/tests/integration/test_tasks.py rename to src/articles/tests/integration/test_tasks.py diff --git a/articles/tests/unit/__init__.py b/src/articles/tests/unit/__init__.py similarity index 100% rename from articles/tests/unit/__init__.py rename to src/articles/tests/unit/__init__.py diff --git a/articles/tests/unit/test_models.py b/src/articles/tests/unit/test_models.py similarity index 100% rename from articles/tests/unit/test_models.py rename to src/articles/tests/unit/test_models.py diff --git a/articles/tests/unit/test_views.py b/src/articles/tests/unit/test_views.py similarity index 100% rename from articles/tests/unit/test_views.py rename to src/articles/tests/unit/test_views.py diff --git a/articles/urls.py b/src/articles/urls.py similarity index 100% rename from articles/urls.py rename to src/articles/urls.py diff --git a/articles/views.py b/src/articles/views.py similarity index 100% rename from articles/views.py rename to src/articles/views.py diff --git a/scraper/__init__.py b/src/scraper/__init__.py similarity index 100% rename from scraper/__init__.py rename to src/scraper/__init__.py diff --git a/scraper/headers.py b/src/scraper/headers.py similarity index 100% rename from scraper/headers.py rename to src/scraper/headers.py diff --git a/scraper/parser.py b/src/scraper/parser.py similarity index 100% rename from scraper/parser.py rename to src/scraper/parser.py diff --git a/scraper/spiders.py b/src/scraper/spiders.py similarity index 98% rename from scraper/spiders.py rename to src/scraper/spiders.py index ccb766f..46a664b 100644 --- a/scraper/spiders.py +++ b/src/scraper/spiders.py @@ -34,7 +34,7 @@ def __init__(self, starting_urls: list[str], sitemap: dict) -> None: self.articles: set[str] = set() async def connect(self, session: ClientSession, url: str) -> str | None: - headers = random.choice(self.headers) + headers = random.choice(self.headers) # nosec try: async with session.get(url, headers=headers) as response: diff --git a/scraper/tests/__init__.py b/src/scraper/tests/__init__.py similarity index 100% rename from scraper/tests/__init__.py rename to src/scraper/tests/__init__.py diff --git a/scraper/tests/conftest.py b/src/scraper/tests/conftest.py similarity index 100% rename from scraper/tests/conftest.py rename to src/scraper/tests/conftest.py diff --git a/scraper/tests/files/articles/aj/_start.html b/src/scraper/tests/files/articles/aj/_start.html similarity index 100% rename from scraper/tests/files/articles/aj/_start.html rename to src/scraper/tests/files/articles/aj/_start.html diff --git a/scraper/tests/files/articles/aj/asian_cup.html b/src/scraper/tests/files/articles/aj/asian_cup.html similarity index 100% rename from scraper/tests/files/articles/aj/asian_cup.html rename to src/scraper/tests/files/articles/aj/asian_cup.html diff --git a/scraper/tests/files/articles/aj/football.html b/src/scraper/tests/files/articles/aj/football.html similarity index 100% rename from scraper/tests/files/articles/aj/football.html rename to src/scraper/tests/files/articles/aj/football.html diff --git a/scraper/tests/files/articles/aj/footprints.html b/src/scraper/tests/files/articles/aj/footprints.html similarity index 100% rename from scraper/tests/files/articles/aj/footprints.html rename to src/scraper/tests/files/articles/aj/footprints.html diff --git a/scraper/tests/files/articles/aj/girl.html b/src/scraper/tests/files/articles/aj/girl.html similarity index 100% rename from scraper/tests/files/articles/aj/girl.html rename to src/scraper/tests/files/articles/aj/girl.html diff --git a/scraper/tests/files/articles/aj/indonesia.html b/src/scraper/tests/files/articles/aj/indonesia.html similarity index 100% rename from scraper/tests/files/articles/aj/indonesia.html rename to src/scraper/tests/files/articles/aj/indonesia.html diff --git a/scraper/tests/files/articles/aj/israel.html b/src/scraper/tests/files/articles/aj/israel.html similarity index 100% rename from scraper/tests/files/articles/aj/israel.html rename to src/scraper/tests/files/articles/aj/israel.html diff --git a/scraper/tests/files/articles/aj/israel2.html b/src/scraper/tests/files/articles/aj/israel2.html similarity index 100% rename from scraper/tests/files/articles/aj/israel2.html rename to src/scraper/tests/files/articles/aj/israel2.html diff --git a/scraper/tests/files/articles/aj/lunar.html b/src/scraper/tests/files/articles/aj/lunar.html similarity index 100% rename from scraper/tests/files/articles/aj/lunar.html rename to src/scraper/tests/files/articles/aj/lunar.html diff --git a/scraper/tests/files/articles/aj/resistance.html b/src/scraper/tests/files/articles/aj/resistance.html similarity index 100% rename from scraper/tests/files/articles/aj/resistance.html rename to src/scraper/tests/files/articles/aj/resistance.html diff --git a/scraper/tests/files/articles/aj/russia.html b/src/scraper/tests/files/articles/aj/russia.html similarity index 100% rename from scraper/tests/files/articles/aj/russia.html rename to src/scraper/tests/files/articles/aj/russia.html diff --git a/scraper/tests/files/articles/aj/student.html b/src/scraper/tests/files/articles/aj/student.html similarity index 100% rename from scraper/tests/files/articles/aj/student.html rename to src/scraper/tests/files/articles/aj/student.html diff --git a/scraper/tests/files/articles/aj/taiwan.html b/src/scraper/tests/files/articles/aj/taiwan.html similarity index 100% rename from scraper/tests/files/articles/aj/taiwan.html rename to src/scraper/tests/files/articles/aj/taiwan.html diff --git a/scraper/tests/integration/__init__.py b/src/scraper/tests/integration/__init__.py similarity index 100% rename from scraper/tests/integration/__init__.py rename to src/scraper/tests/integration/__init__.py diff --git a/scraper/tests/integration/test_spider.py b/src/scraper/tests/integration/test_spider.py similarity index 100% rename from scraper/tests/integration/test_spider.py rename to src/scraper/tests/integration/test_spider.py diff --git a/scraper/tests/mocks.py b/src/scraper/tests/mocks.py similarity index 100% rename from scraper/tests/mocks.py rename to src/scraper/tests/mocks.py diff --git a/scraper/tests/unit/__init__.py b/src/scraper/tests/unit/__init__.py similarity index 100% rename from scraper/tests/unit/__init__.py rename to src/scraper/tests/unit/__init__.py diff --git a/scraper/tests/unit/test_parsers.py b/src/scraper/tests/unit/test_parsers.py similarity index 100% rename from scraper/tests/unit/test_parsers.py rename to src/scraper/tests/unit/test_parsers.py diff --git a/scraper/tests/unit/test_spider.py b/src/scraper/tests/unit/test_spider.py similarity index 100% rename from scraper/tests/unit/test_spider.py rename to src/scraper/tests/unit/test_spider.py diff --git a/scraper/tests/utils.py b/src/scraper/tests/utils.py similarity index 100% rename from scraper/tests/utils.py rename to src/scraper/tests/utils.py