feat(attachments, storage): u#3727 t#3961 physical attachment deletion system

python/apps/taiga/src/taiga/__main__.py

@@ -33,6 +33,7 @@
 from taiga.base.django.commands import call_django_command
 from taiga.base.i18n.commands import cli as i18n_cli
 from taiga.base.sampledata.commands import cli as sampledata_cli
+from taiga.commons.storage.commands import cli as storage_cli
 from taiga.emails.commands import cli as emails_cli
 from taiga.tasksqueue.commands import cli as tasksqueue_cli
 from taiga.tasksqueue.commands import init as init_tasksqueue
@@ -70,6 +71,7 @@ def main(
 cli.add_typer(i18n_cli, name="i18n")
 cli.add_typer(sampledata_cli, name="sampledata")
 cli.add_typer(tasksqueue_cli, name="tasksqueue")
+cli.add_typer(storage_cli, name="storage")
 cli.add_typer(tokens_cli, name="tokens")
 cli.add_typer(users_cli, name="users")
 

python/apps/taiga/src/taiga/attachments/admin.py

@@ -48,7 +48,8 @@ class AttachmentAdmin(admin.ModelAdmin[Attachment]):
     list_display = (
         "b64id",
         "name",
-        "content_object_link" "created_by",
+        "content_object_link",
+        "created_by",
     )
     readonly_fields = (
         "id",

python/apps/taiga/src/taiga/attachments/apps.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC
+
+from taiga.base.django import apps
+
+
+class AttachmentConfig(apps.AppConfig):
+    name = "taiga.attachments"
+    label = "attachments"
+    verbose_name = "Attachments"
+
+    def ready(self) -> None:
+        from taiga.attachments.signals import mark_attachment_file_to_delete
+        from taiga.base.db.models import signals
+
+        signals.post_delete.connect(
+            mark_attachment_file_to_delete,
+            sender="attachments.Attachment",
+            dispatch_uid="mark_attachment_file_to_delete",
+        )

python/apps/taiga/src/taiga/attachments/migrations/0001_initial.py

@@ -5,10 +5,9 @@
 #
 # Copyright (c) 2023-present Kaleidos INC
 
-# Generated by Django 4.2.3 on 2023-08-21 15:14
+# Generated by Django 4.2.3 on 2023-09-01 19:15
 
 import django.db.models.deletion
-import taiga.attachments.models
 import taiga.base.db.models
 import taiga.base.utils.datetime
 from django.conf import settings
@@ -19,6 +18,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
+        ("storage", "0001_initial"),
         ("contenttypes", "0002_remove_content_type_name"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
@@ -45,14 +45,6 @@ class Migration(migrations.Migration):
                         verbose_name="created at",
                     ),
                 ),
-                (
-                    "file",
-                    models.FileField(
-                        max_length=500,
-                        upload_to=taiga.attachments.models.get_attachment_file_path,
-                        verbose_name="file",
-                    ),
-                ),
                 ("name", models.TextField(verbose_name="file name")),
                 ("content_type", models.TextField(verbose_name="file content type")),
                 ("size", models.IntegerField(verbose_name="file size (bytes)")),
@@ -67,6 +59,15 @@ class Migration(migrations.Migration):
                         verbose_name="created by",
                     ),
                 ),
+                (
+                    "file",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.RESTRICT,
+                        related_name="attachments",
+                        to="storage.storagedobject",
+                        verbose_name="file",
+                    ),
+                ),
                 (
                     "object_content_type",
                     models.ForeignKey(

python/apps/taiga/src/taiga/attachments/models.py

@@ -5,36 +5,18 @@
 #
 # Copyright (c) 2023-present Kaleidos INC
 
-from os import path
-from typing import cast
 
 from taiga.base.db import models
 from taiga.base.db.mixins import CreatedMetaInfoMixin
-from taiga.base.utils.files import get_obfuscated_file_path
-
-
-def get_attachment_file_path(instance: "Attachment", filename: str) -> str:
-    content_object = cast(models.BaseModel, instance.content_object)
-    label = content_object._meta.app_label
-    model_name = content_object._meta.model_name or content_object.__class__.__name__
-
-    base_path = path.join(
-        "attachments",
-        f"{label.lower()}_{model_name.lower()}",
-        content_object.b64id,
-    )
-    return get_obfuscated_file_path(instance, filename, base_path)
 
 
 class Attachment(models.BaseModel, CreatedMetaInfoMixin):
-    # TODO: We need to remove file on delete content_object. It may depend on the real life that
-    #       the files have beyond their content object (especially with history or activity timelines).
-    #       (Some inspiration https://github.com/un1t/django-cleanup)
-    file = models.FileField(
-        upload_to=get_attachment_file_path,
-        max_length=500,
+    file = models.ForeignKey(
+        "storage.StoragedObject",
         null=False,
         blank=False,
+        on_delete=models.RESTRICT,
+        related_name="attachments",
         verbose_name="file",
     )
     name = models.TextField(

python/apps/taiga/src/taiga/attachments/repositories.py

@@ -12,14 +12,15 @@
 from taiga.attachments.models import Attachment
 from taiga.base.db.models import BaseModel, Model, QuerySet, get_contenttype_for_model
 from taiga.base.utils.files import get_size, uploadfile_to_file
+from taiga.commons.storage import repositories as storage_repositories
 from taiga.users.models import User
 
 ##########################################################
 # filters and querysets
 ##########################################################
 
 
-DEFAULT_QUERYSET = Attachment.objects.all()
+DEFAULT_QUERYSET = Attachment.objects.select_related("file").all()
 
 
 class AttachmentFilters(TypedDict, total=False):
@@ -77,11 +78,13 @@ async def create_attachment(
     created_by: User,
     object: Model,
 ) -> Attachment:
+    storaged_object = await storage_repositories.create_storaged_object(uploadfile_to_file(file))
+
     return await Attachment.objects.acreate(
+        file=storaged_object,
         name=file.filename or "unknown",
         size=get_size(file.file),
         content_type=file.content_type or "unknown",
-        file=uploadfile_to_file(file),
         content_object=object,
         created_by=created_by,
     )

python/apps/taiga/src/taiga/attachments/serializers.py

@@ -8,7 +8,7 @@
 from datetime import datetime
 
 from taiga.base.serializers import UUIDB64, BaseModel
-from taiga.base.serializers.fields import FileField
+from taiga.commons.storage.serializers import StoragedObjectFileField
 
 
 class AttachmentSerializer(BaseModel):
@@ -17,7 +17,7 @@ class AttachmentSerializer(BaseModel):
     content_type: str
     size: int
     created_at: datetime
-    file: FileField
+    file: StoragedObjectFileField
 
     class Config:
         orm_mode = True

python/apps/taiga/src/taiga/attachments/signals.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC
+
+from typing import Any
+
+from taiga.attachments.models import Attachment
+from taiga.base.db.models import Model
+from taiga.commons.storage import repositories as storage_repositories
+
+
+def mark_attachment_file_to_delete(sender: Model, instance: Attachment, **kwargs: Any) -> None:
+    """
+    Mark the store object (with the file) of the attachment as deleted.
+    """
+    storage_repositories.mark_storaged_object_as_deleted(storaged_object=instance.file)

python/apps/taiga/src/taiga/base/db/exceptions.py

@@ -6,6 +6,7 @@
 # Copyright (c) 2023-present Kaleidos INC
 
 
+from django.db.models.deletion import RestrictedError  # noqa
 from django.db.utils import IntegrityError, ProgrammingError  # noqa
 
 

python/apps/taiga/src/taiga/base/db/mixins.py

@@ -54,14 +54,20 @@ class Meta:
         abstract = True
 
 
-class DeletedMetaInfoMixin(models.Model):
+class DeletedByMetaInfoMixin(models.Model):
     deleted_by = models.ForeignKey(
         "users.User",
         null=True,
         blank=True,
         on_delete=models.SET_NULL,
         verbose_name="deleted by",
     )
+
+    class Meta:
+        abstract = True
+
+
+class DeletedAtMetaInfoMixin(models.Model):
     deleted_at = models.DateTimeField(
         null=True,
         blank=True,
@@ -72,6 +78,11 @@ class Meta:
         abstract = True
 
 
+class DeletedMetaInfoMixin(DeletedByMetaInfoMixin, DeletedAtMetaInfoMixin):
+    class Meta:
+        abstract = True
+
+
 #######################################################
 # Title
 #######################################################

python/apps/taiga/src/taiga/base/db/models/__init__.py

@@ -23,7 +23,7 @@
 from django.contrib.postgres.lookups import Unaccent  # noqa
 from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector  # noqa
 from django.db.models import Model, UUIDField
-from django.db.models.functions import Coalesce, Lower, StrIndex  # noqa
+from django.db.models.functions import Coalesce, Lower, StrIndex, TruncDate  # noqa
 from taiga.base.db.models.fields import *  # noqa
 from taiga.base.utils.uuid import encode_uuid_to_b64str
 from taiga.conf import settings

python/apps/taiga/src/taiga/base/django/apps.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC
+
+from django.apps import AppConfig  # noqa

python/apps/taiga/src/taiga/base/django/settings.py

@@ -38,6 +38,7 @@
     "django.contrib.staticfiles",
     # taiga
     "taiga.base.db",
+    "taiga.commons.storage",
     "taiga.emails",
     "taiga.mediafiles",
     "taiga.projects.invitations",

python/apps/taiga/src/taiga/commons/storage/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC

python/apps/taiga/src/taiga/commons/storage/admin.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC
+
+from taiga.base.db import admin
+from taiga.commons.storage.models import StoragedObject
+
+
+@admin.register(StoragedObject)
+class StoragedObjectAdmin(admin.ModelAdmin[StoragedObject]):
+    list_display = (
+        "id",
+        "file",
+        "created_at",
+        "deleted_at",
+    )
+    readonly_fields = (
+        "created_at",
+        "deleted_at",
+    )
+    search_fields = ("file",)
+    list_filter = ("created_at", "deleted_at")
+    ordering = ("-created_at",)

python/apps/taiga/src/taiga/commons/storage/commands.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2023-present Kaleidos INC
+
+from datetime import timedelta
+
+import typer
+from taiga.base.utils import pprint
+from taiga.base.utils.concurrency import run_async_as_sync
+from taiga.base.utils.datetime import aware_utcnow
+from taiga.commons.storage import services as storage_services
+from taiga.conf import settings
+
+cli = typer.Typer(
+    name="Taiga Storage commands",
+    help="Manage the starege system of Taiga.",
+    add_completion=True,
+)
+
+
+@cli.command(help="Clean deleted storaged object. Remove entries from DB and files from storage")
+def clean_storaged_objects(
+    days_to_store_deleted_storaged_object: int = typer.Option(
+        settings.STORAGE.DAYS_TO_STORE_DELETED_STORAGED_OBJECTS,
+        "--days",
+        "-d",
+        help="Number of days to store deleted storaged objects",
+    ),
+) -> None:
+    total_deleted = run_async_as_sync(
+        storage_services.clean_deleted_storaged_objects(
+            before=aware_utcnow() - timedelta(days=days_to_store_deleted_storaged_object)
+        )
+    )
+
+    color = "red" if total_deleted else "white"
+    pprint.print(f"Deleted [bold][{color}]{total_deleted}[/{color}][/bold] storaged objects.")