Use git to crawl dependencies

api/Dockerfile

@@ -1,14 +1,17 @@
-FROM python:3.11
+FROM python:3.12
 
 WORKDIR /app
 
 RUN wget -q https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -P /usr/local/bin \
  && chmod +x /usr/local/bin/wait-for-it.sh \
  && mkdir -p /app \
+ && mkdir /projects \
  && useradd -u 901 -r outdated --create-home \
  # all project specific folders need to be accessible by newly created user but also for unknown users (when UID is set manually). Such users are in group root.
  && chown -R outdated:root /home/outdated \
+ && chown -R outdated:root /projects \
  && chmod -R 770 /home/outdated \
+ && chmod -R 770 /projects \
  && apt-get update && apt-get install -y --no-install-recommends \
  # needed for psycopg2
  libpq-dev \
@@ -19,6 +22,9 @@ ENV HOME=/home/outdated
 
 ENV PYTHONUNBUFFERED=1
 ENV DJANGO_SETTINGS_MODULE outdated.settings
+
+# prevent git from asking for credentials on wrong urls
+ENV GIT_ASKPASS true
 ENV APP_HOME=/app
 
 COPY pyproject.toml poetry.lock $APP_HOME/

api/outdated/conftest.py

@@ -1,11 +1,17 @@
+from __future__ import annotations
+
+from collections.abc import Callable
 from functools import partial
+from unittest.mock import MagicMock
 
 import pytest
 from pytest_factoryboy import register
+from pytest_mock import MockerFixture
 from rest_framework.test import APIClient
 
 from .oidc_auth.models import OIDCUser
 from .outdated import factories
+from .tracking import Tracker
 from .user.factories import UserFactory
 
 register(factories.DependencyFactory)
@@ -59,11 +65,14 @@ def client(db, settings, get_claims):
  return client
 
 
-@pytest.fixture(scope="module")
-def vcr_config():
- return {
- # Replace the Authorization header with a dummy value
- "filter_headers": [("Authorization", "DUMMY")],
- "ignore_localhost": True,
- "ignore_hosts": ["https://outdated.local"],
- }
+@pytest.fixture
+def tracker_mock(mocker: MockerFixture) -> Callable[[str], MagicMock]:
+ def _tracker_mock(target: str) -> MagicMock:
+ return mocker.patch.object(Tracker, target)
+
+ return _tracker_mock
+
+
+@pytest.fixture
+def tracker_init_mock(mocker: MockerFixture) -> MagicMock:
+ return mocker.patch.object(Tracker, "__init__", return_value=None)

api/outdated/models.py

@@ -1,5 +1,7 @@
+from subprocess import run
 from uuid import uuid4
 
+from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
 from django.db import models
 
@@ -57,11 +59,24 @@ def pre_save(self, model_instance, add):
  return super().pre_save(model_instance, add)
 
 
+def validate_repo_exists(value: str) -> None:
+ """Validate the existance of a remote git repository."""
+ result = run(
+ ["git", "ls-remote", "https://" + value], # noqa: S603,S607
+ capture_output=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ msg = "Repository does not exist."
+ raise ValidationError(msg, params={"value": value})
+
+
 class RepositoryURLField(models.CharField):
  default_validators = [
  RegexValidator(
  regex=r"^([-_\w]+\.[-._\w]+)\/([-_\w]+)\/([-_\w]+)$",
  message="Invalid repository url",
  ),
+ validate_repo_exists,
  ]
  description = "Field for git repository URLs."

api/outdated/outdated/management/commands/syncproject.py

@@ -1,11 +1,11 @@
 from django.core.management.base import BaseCommand
 
 from outdated.outdated.models import Project
-from outdated.outdated.synchroniser import Synchroniser
+from outdated.tracking import Tracker
 
 
 class Command(BaseCommand):
- help = "Syncs the given project with its remote counterpart."
+ help = "Syncs the given project with its repository."
 
  def add_arguments(self, parser):
  parser.add_argument("project_name", type=str)
@@ -15,7 +15,7 @@ def handle(self, *_, **options):
  try:
  project = Project.objects.get(name__iexact=project_name)
  self.stdout.write(f"Syncing project {project}")
- Synchroniser(project).sync()
+ Tracker(project).sync()
  self.stdout.write(f"Finished syncing {project}")
  except Project.DoesNotExist:
  self.stdout.write(f"Project {project_name} not found")

api/outdated/outdated/management/commands/syncprojects.py

@@ -1,15 +1,12 @@
-from asyncio import gather
+from django.core.management import BaseCommand
 
-from outdated.commands import AsyncCommand
 from outdated.outdated.models import Project
-from outdated.outdated.synchroniser import Synchroniser
+from outdated.tracking import Tracker
 
 
-class Command(AsyncCommand):
- help = "Syncs all projects with their remote counterparts."
+class Command(BaseCommand):
+ help = "Syncs all projects with their remote repos."
 
- async def _handle(self, *args, **options):
- projects = Project.objects.all()
- project_tasks = [Synchroniser(project).a_sync() async for project in projects]
- await gather(*project_tasks)
+ def handle(self, *args, **options):
+ [Tracker(project).sync() for project in Project.objects.all()]
  self.stdout.write("Finished syncing all projects")

api/outdated/outdated/models.py

@@ -94,7 +94,6 @@ def version(self):
 
 class Project(UUIDModel):
  name = models.CharField(max_length=100, db_index=True)
-
  versioned_dependencies = models.ManyToManyField(Version, blank=True)
  repo = RepositoryURLField(max_length=100)
 
@@ -116,6 +115,22 @@ def status(self) -> str:
  first = self.versioned_dependencies.first()
  return first.release_version.status if first else STATUS_OPTIONS["undefined"]
 
+ @property
+ def repo_domain(self):
+ return self.repo.split("/")[0].lower()
+
+ @property
+ def repo_namespace(self):
+ return self.repo.split("/")[-1].lower()
+
+ @property
+ def repo_name(self):
+ return self.repo.split("/")[1].lower()
+
+ @property
+ def clone_path(self):
+ return f"{self.repo_domain}/{self.repo_namespace}/{self.repo_name}"
+
  def __str__(self):
  return self.name
 

api/outdated/outdated/serializers.py

@@ -1,6 +1,7 @@
 from rest_framework_json_api import serializers
 
-from . import models
+from outdated.outdated import models
+from outdated.tracking import Tracker
 
 
 class DependencySerializer(serializers.ModelSerializer):
@@ -53,6 +54,21 @@ class ProjectSerializer(serializers.ModelSerializer):
  "maintainers": "outdated.outdated.serializers.MaintainerSerializer",
  }
 
+ def create(self, validated_data):
+ project = models.Project.objects.create(**validated_data)
+ Tracker(project).setup()
+ project.refresh_from_db()
+ return project
+
+ def update(self, instance: models.Project, validated_data: dict) -> models.Project:
+ old_instance = models.Project(repo=instance.repo)
+ super().update(instance, validated_data)
+ if instance.clone_path != old_instance.clone_path:
+ Tracker(old_instance).delete()
+ Tracker(instance).setup()
+ instance.refresh_from_db()
+ return instance
+
  class Meta:
  model = models.Project
  fields = (