ATV-152 Scheduled document deletion command

Added delete_after field to Document
Created management command to delete expired documents
Add tests
Updated test snapshots
Add new field to serializers

documents/management/commands/delete_expired_documents.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+
+from documents.tasks import delete_expired_documents
+
+
+class Command(BaseCommand):
+ help = "Delete documents and related objects and files that have reached their deletion date"
+
+ def handle(self, *args, **kwargs):
+ delete_expired_documents()

documents/migrations/0014_document_delete_after.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.19 on 2023-08-31 10:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("documents", "0013_add_document_language_and_content_schema"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="document",
+ name="delete_after",
+ field=models.DateField(
+ help_text="Date which after the document and related attachments are permanently deleted",
+ null=True,
+ ),
+ ),
+ ]

documents/models.py

@@ -261,6 +261,12 @@ class Document(UUIDModel, TimestampedModel):
  content_schema_url = models.URLField(
  null=True, blank=True, help_text=_("Link to content schema if available")
  )
+ delete_after = models.DateField(
+ null=True,
+ help_text=_(
+ "Date which after the document and related attachments are permanently deleted"
+ ),
+ )
 
  class Meta:
  verbose_name = _("document")

documents/serializers/document.py

@@ -92,6 +92,7 @@ class Meta:
  "type",
  "human_readable_type",
  "deletable",
+ "delete_after",
  "attachment_count",
  "attachments",
  )
@@ -130,6 +131,7 @@ class Meta:
  "status_display_values",
  "status_timestamp",
  "status_histories",
+ "delete_after",
  "document_language",
  "content_schema_url",
  )
@@ -177,6 +179,7 @@ class Meta:
  "draft",
  "locked_after",
  "deletable",
+ "delete_after",
  "document_language",
  "content_schema_url",
  "attachments",
@@ -244,6 +247,7 @@ class Meta:
  "draft",
  "locked_after",
  "deletable",
+ "delete_after",
  "document_language",
  "content_schema_url",
  "attachments",

documents/tasks.py

@@ -0,0 +1,20 @@
+import logging
+
+from django.utils import timezone
+
+from documents.models import Document
+
+logger = logging.getLogger(__name__)
+
+
+def delete_expired_documents():
+ """Delete expired documents and all related objects and files"""
+ documents_to_delete_qs = Document.objects.filter(delete_after__lt=timezone.now())
+
+ total, by_type_dict = documents_to_delete_qs.delete()
+ if total != 0:
+ logger.info(
+ f"Deleted {total} objects: {', '.join([f'{i[1]} {i[0]}' for i in by_type_dict.items()])}."
+ )
+ else:
+ logger.info("Nothing to delete.")

documents/tests/snapshots/snap_test_api_create_document.py

@@ -19,6 +19,7 @@
  "content_schema_url": "https://schema.fi",
  "created_at": "2021-06-30T12:00:00+03:00",
  "deletable": False,
+ "delete_after": None,
  "document_language": "en",
  "draft": False,
  "locked_after": None,
@@ -61,6 +62,7 @@
  "content_schema_url": "https://schema.fi",
  "created_at": "2021-06-30T12:00:00+03:00",
  "deletable": False,
+ "delete_after": "2022-12-12",
  "document_language": "en",
  "draft": False,
  "locked_after": None,

documents/tests/snapshots/snap_test_api_patch_document.py

@@ -19,6 +19,7 @@
  "content_schema_url": None,
  "created_at": "2021-06-30T12:00:00+03:00",
  "deletable": True,
+ "delete_after": None,
  "document_language": None,
  "draft": False,
  "id": "2d2b7a36-a306-4e35-990f-13aea04263ff",
@@ -52,6 +53,7 @@
  "content_schema_url": "https://schema.fi",
  "created_at": "2021-06-30T12:00:00+03:00",
  "deletable": False,
+ "delete_after": None,
  "document_language": "en",
  "draft": True,
  "id": "2d2b7a36-a306-4e35-990f-13aea04263ff",
@@ -86,6 +88,7 @@
  "content_schema_url": "https://schema.fi",
  "created_at": "2021-06-30T12:00:00+03:00",
  "deletable": False,
+ "delete_after": None,
  "document_language": "en",
  "draft": False,
  "id": "2d2b7a36-a306-4e35-990f-13aea04263ff",

documents/tests/snapshots/snap_test_api_retrieve_document.py

@@ -13,12 +13,13 @@
  "content_schema_url": None,
  "created_at": "2020-06-01T03:00:00+03:00",
  "deletable": True,
+ "delete_after": None,
  "document_language": None,
  "draft": False,
  "id": "485af718-d9d1-46b9-ad7b-33ea054126e3",
  "locked_after": None,
  "metadata": {},
- "service": "service 205",
+ "service": "service 207",
  "status": {
  "timestamp": "2020-06-01T03:00:00+03:00",
  "value": "testing",
@@ -40,12 +41,13 @@
  "content_schema_url": None,
  "created_at": "2020-06-01T03:00:00+03:00",
  "deletable": True,
+ "delete_after": None,
  "document_language": None,
  "draft": False,
  "id": "485af718-d9d1-46b9-ad7b-33ea054126e3",
  "locked_after": None,
  "metadata": {},
- "service": "service 208",
+ "service": "service 210",
  "status": {
  "timestamp": "2020-06-01T03:00:00+03:00",
  "value": "testing",
@@ -67,12 +69,13 @@
  "content_schema_url": None,
  "created_at": "2020-06-01T03:00:00+03:00",
  "deletable": True,
+ "delete_after": None,
  "document_language": None,
  "draft": False,
  "id": "485af718-d9d1-46b9-ad7b-33ea054126e3",
  "locked_after": None,
  "metadata": {},
- "service": "service 207",
+ "service": "service 209",
  "status": {
  "timestamp": "2020-06-01T03:00:00+03:00",
  "value": "testing",

documents/tests/test_api_create_document.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 import uuid
 from unittest import mock
@@ -101,8 +102,11 @@ def test_create_authenticated_document(user, service, snapshot):
  """Normal user creates a document which is attached to his/her account."""
  api_client = get_user_service_client(user, service)
 
+ delete_after_date = "2022-12-12"
  response = api_client.post(
- reverse("documents-list"), VALID_DOCUMENT_DATA, format="multipart"
+ reverse("documents-list"),
+ {**VALID_DOCUMENT_DATA, "delete_after": delete_after_date},
+ format="multipart",
  )
 
  assert response.status_code == status.HTTP_201_CREATED
@@ -111,6 +115,7 @@ def test_create_authenticated_document(user, service, snapshot):
  document = Document.objects.first()
  assert document.user == user
  assert document.service == service
+ assert document.delete_after == datetime.date(day=12, month=12, year=2022)
 
  body = response.json()
  assert uuid.UUID(body.pop("id")) == document.id

documents/tests/test_api_patch_document.py

@@ -611,6 +611,22 @@ def test_update_document_not_found(superuser_api_client):
  )
 
 
+def test_update_delete_after_user(service, user):
+ api_client = get_user_service_client(user, service)
+
+ response = api_client.post(
+ reverse("documents-list"), VALID_DOCUMENT_DATA, format="multipart"
+ )
+ assert response.status_code == status.HTTP_201_CREATED
+ assert StatusHistory.objects.count() == 1
+ document_id = response.json().get("id")
+ response = api_client.patch(
+ reverse("documents-detail", args=[document_id]), {"delete_after": "2024-12-12"}
+ )
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
 @pytest.mark.parametrize("attachments", [0, 1, 2])
 @pytest.mark.parametrize(
  "ip_address", ["213.255.180.34", "2345:0425:2CA1::0567:5673:23b5"]

documents/tests/test_commands.py

@@ -1,8 +1,14 @@
+from datetime import timezone
 from pathlib import Path
 from uuid import uuid4
 
+from dateutil.relativedelta import relativedelta
+from dateutil.utils import today
 from django.core.management import call_command
 
+from documents.models import Activity, Attachment, Document, StatusHistory
+from documents.tests.factories import AttachmentFactory, DocumentFactory
+
 
 def test_call_remove_outdated_files():
  call_command("remove_outdated_files")
@@ -32,3 +38,30 @@ def test_call_remove_extra_directories(settings):
  assert path_to_remove.exists()
  call_command("remove_outdated_files")
  assert not path_to_remove.exists()
+
+
+def test_delete_expired_documents(service):
+ document1 = DocumentFactory(
+ service=service, delete_after=today(timezone.utc) - relativedelta(days=1)
+ )
+ document2 = DocumentFactory(
+ service=service, delete_after=today(timezone.utc) + relativedelta(days=2)
+ )
+
+ AttachmentFactory(document=document1)
+ AttachmentFactory(document=document2)
+
+ status_history1 = StatusHistory.objects.create(document=document1)
+ status_history2 = StatusHistory.objects.create(document=document2)
+
+ Activity.objects.create(status=status_history1)
+ Activity.objects.create(status=status_history2)
+
+ assert Document.objects.count() == 2
+ assert Attachment.objects.count() == 2
+ assert Activity.objects.count() == 2
+ call_command("delete_expired_documents")
+
+ assert Document.objects.count() == 1
+ assert Attachment.objects.count() == 1
+ assert StatusHistory.objects.count() == 1