diff --git a/.github/sql-fixtures/fixtures.sql b/.github/sql-fixtures/fixtures.sql index f683c391d..a35e68c0e 100644 Binary files a/.github/sql-fixtures/fixtures.sql and b/.github/sql-fixtures/fixtures.sql differ diff --git a/python/apps/taiga/src/taiga/__main__.py b/python/apps/taiga/src/taiga/__main__.py index f7bdb91f3..8830d3e70 100644 --- a/python/apps/taiga/src/taiga/__main__.py +++ b/python/apps/taiga/src/taiga/__main__.py @@ -33,6 +33,7 @@ from taiga.base.django.commands import call_django_command from taiga.base.i18n.commands import cli as i18n_cli from taiga.base.sampledata.commands import cli as sampledata_cli +from taiga.commons.storage.commands import cli as storage_cli from taiga.emails.commands import cli as emails_cli from taiga.tasksqueue.commands import cli as tasksqueue_cli from taiga.tasksqueue.commands import init as init_tasksqueue @@ -70,6 +71,7 @@ def main( cli.add_typer(i18n_cli, name="i18n") cli.add_typer(sampledata_cli, name="sampledata") cli.add_typer(tasksqueue_cli, name="tasksqueue") +cli.add_typer(storage_cli, name="storage") cli.add_typer(tokens_cli, name="tokens") cli.add_typer(users_cli, name="users") diff --git a/python/apps/taiga/src/taiga/attachments/admin.py b/python/apps/taiga/src/taiga/attachments/admin.py index 67aad2959..6d7d8aaf3 100644 --- a/python/apps/taiga/src/taiga/attachments/admin.py +++ b/python/apps/taiga/src/taiga/attachments/admin.py @@ -48,7 +48,8 @@ class AttachmentAdmin(admin.ModelAdmin[Attachment]): list_display = ( "b64id", "name", - "content_object_link" "created_by", + "content_object_link", + "created_by", ) readonly_fields = ( "id", diff --git a/python/apps/taiga/src/taiga/attachments/apps.py b/python/apps/taiga/src/taiga/attachments/apps.py new file mode 100644 index 000000000..4f7a41102 --- /dev/null +++ b/python/apps/taiga/src/taiga/attachments/apps.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from taiga.base.django import apps + + +class AttachmentConfig(apps.AppConfig): + name = "taiga.attachments" + label = "attachments" + verbose_name = "Attachments" + + def ready(self) -> None: + from taiga.attachments.signals import mark_attachment_file_to_delete + from taiga.base.db.models import signals + + signals.post_delete.connect( + mark_attachment_file_to_delete, + sender="attachments.Attachment", + dispatch_uid="mark_attachment_file_to_delete", + ) diff --git a/python/apps/taiga/src/taiga/attachments/migrations/0001_initial.py b/python/apps/taiga/src/taiga/attachments/migrations/0001_initial.py index 7844801d2..c16e84159 100644 --- a/python/apps/taiga/src/taiga/attachments/migrations/0001_initial.py +++ b/python/apps/taiga/src/taiga/attachments/migrations/0001_initial.py @@ -5,10 +5,9 @@ # # Copyright (c) 2023-present Kaleidos INC -# Generated by Django 4.2.3 on 2023-08-21 15:14 +# Generated by Django 4.2.3 on 2023-09-01 19:15 import django.db.models.deletion -import taiga.attachments.models import taiga.base.db.models import taiga.base.utils.datetime from django.conf import settings @@ -19,6 +18,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ("storage", "0001_initial"), ("contenttypes", "0002_remove_content_type_name"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] @@ -45,14 +45,6 @@ class Migration(migrations.Migration): verbose_name="created at", ), ), - ( - "file", - models.FileField( - max_length=500, - upload_to=taiga.attachments.models.get_attachment_file_path, - verbose_name="file", - ), - ), ("name", models.TextField(verbose_name="file name")), ("content_type", models.TextField(verbose_name="file content type")), ("size", models.IntegerField(verbose_name="file size (bytes)")), @@ -67,6 +59,15 @@ class Migration(migrations.Migration): verbose_name="created by", ), ), + ( + "file", + models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + related_name="attachments", + to="storage.storagedobject", + verbose_name="file", + ), + ), ( "object_content_type", models.ForeignKey( diff --git a/python/apps/taiga/src/taiga/attachments/models.py b/python/apps/taiga/src/taiga/attachments/models.py index 8ee0f62f3..cba7e1c4a 100644 --- a/python/apps/taiga/src/taiga/attachments/models.py +++ b/python/apps/taiga/src/taiga/attachments/models.py @@ -5,36 +5,18 @@ # # Copyright (c) 2023-present Kaleidos INC -from os import path -from typing import cast from taiga.base.db import models from taiga.base.db.mixins import CreatedMetaInfoMixin -from taiga.base.utils.files import get_obfuscated_file_path - - -def get_attachment_file_path(instance: "Attachment", filename: str) -> str: - content_object = cast(models.BaseModel, instance.content_object) - label = content_object._meta.app_label - model_name = content_object._meta.model_name or content_object.__class__.__name__ - - base_path = path.join( - "attachments", - f"{label.lower()}_{model_name.lower()}", - content_object.b64id, - ) - return get_obfuscated_file_path(instance, filename, base_path) class Attachment(models.BaseModel, CreatedMetaInfoMixin): - # TODO: We need to remove file on delete content_object. It may depend on the real life that - # the files have beyond their content object (especially with history or activity timelines). - # (Some inspiration https://github.com/un1t/django-cleanup) - file = models.FileField( - upload_to=get_attachment_file_path, - max_length=500, + file = models.ForeignKey( + "storage.StoragedObject", null=False, blank=False, + on_delete=models.RESTRICT, + related_name="attachments", verbose_name="file", ) name = models.TextField( diff --git a/python/apps/taiga/src/taiga/attachments/repositories.py b/python/apps/taiga/src/taiga/attachments/repositories.py index bd0150935..d9288264e 100644 --- a/python/apps/taiga/src/taiga/attachments/repositories.py +++ b/python/apps/taiga/src/taiga/attachments/repositories.py @@ -12,6 +12,7 @@ from taiga.attachments.models import Attachment from taiga.base.db.models import BaseModel, Model, QuerySet, get_contenttype_for_model from taiga.base.utils.files import get_size, uploadfile_to_file +from taiga.commons.storage import repositories as storage_repositories from taiga.users.models import User ########################################################## @@ -19,7 +20,7 @@ ########################################################## -DEFAULT_QUERYSET = Attachment.objects.all() +DEFAULT_QUERYSET = Attachment.objects.select_related("file").all() class AttachmentFilters(TypedDict, total=False): @@ -77,11 +78,13 @@ async def create_attachment( created_by: User, object: Model, ) -> Attachment: + storaged_object = await storage_repositories.create_storaged_object(uploadfile_to_file(file)) + return await Attachment.objects.acreate( + file=storaged_object, name=file.filename or "unknown", size=get_size(file.file), content_type=file.content_type or "unknown", - file=uploadfile_to_file(file), content_object=object, created_by=created_by, ) diff --git a/python/apps/taiga/src/taiga/attachments/serializers.py b/python/apps/taiga/src/taiga/attachments/serializers.py index 1f44ff847..1387e114b 100644 --- a/python/apps/taiga/src/taiga/attachments/serializers.py +++ b/python/apps/taiga/src/taiga/attachments/serializers.py @@ -8,7 +8,7 @@ from datetime import datetime from taiga.base.serializers import UUIDB64, BaseModel -from taiga.base.serializers.fields import FileField +from taiga.commons.storage.serializers import StoragedObjectFileField class AttachmentSerializer(BaseModel): @@ -17,7 +17,7 @@ class AttachmentSerializer(BaseModel): content_type: str size: int created_at: datetime - file: FileField + file: StoragedObjectFileField class Config: orm_mode = True diff --git a/python/apps/taiga/src/taiga/attachments/signals.py b/python/apps/taiga/src/taiga/attachments/signals.py new file mode 100644 index 000000000..391100262 --- /dev/null +++ b/python/apps/taiga/src/taiga/attachments/signals.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from typing import Any + +from taiga.attachments.models import Attachment +from taiga.base.db.models import Model +from taiga.commons.storage import repositories as storage_repositories + + +def mark_attachment_file_to_delete(sender: Model, instance: Attachment, **kwargs: Any) -> None: + """ + Mark the store object (with the file) of the attachment as deleted. + """ + storage_repositories.mark_storaged_object_as_deleted(storaged_object=instance.file) diff --git a/python/apps/taiga/src/taiga/base/db/exceptions.py b/python/apps/taiga/src/taiga/base/db/exceptions.py index 0d2580598..1bc36e2e6 100644 --- a/python/apps/taiga/src/taiga/base/db/exceptions.py +++ b/python/apps/taiga/src/taiga/base/db/exceptions.py @@ -6,6 +6,7 @@ # Copyright (c) 2023-present Kaleidos INC +from django.db.models.deletion import RestrictedError # noqa from django.db.utils import IntegrityError, ProgrammingError # noqa diff --git a/python/apps/taiga/src/taiga/base/db/mixins.py b/python/apps/taiga/src/taiga/base/db/mixins.py index 9c9c1d54a..4ecd41c1b 100644 --- a/python/apps/taiga/src/taiga/base/db/mixins.py +++ b/python/apps/taiga/src/taiga/base/db/mixins.py @@ -54,7 +54,7 @@ class Meta: abstract = True -class DeletedMetaInfoMixin(models.Model): +class DeletedByMetaInfoMixin(models.Model): deleted_by = models.ForeignKey( "users.User", null=True, @@ -62,6 +62,12 @@ class DeletedMetaInfoMixin(models.Model): on_delete=models.SET_NULL, verbose_name="deleted by", ) + + class Meta: + abstract = True + + +class DeletedAtMetaInfoMixin(models.Model): deleted_at = models.DateTimeField( null=True, blank=True, @@ -72,6 +78,11 @@ class Meta: abstract = True +class DeletedMetaInfoMixin(DeletedByMetaInfoMixin, DeletedAtMetaInfoMixin): + class Meta: + abstract = True + + ####################################################### # Title ####################################################### diff --git a/python/apps/taiga/src/taiga/base/db/models/__init__.py b/python/apps/taiga/src/taiga/base/db/models/__init__.py index 25ffbea0a..e507a9de4 100644 --- a/python/apps/taiga/src/taiga/base/db/models/__init__.py +++ b/python/apps/taiga/src/taiga/base/db/models/__init__.py @@ -23,7 +23,7 @@ from django.contrib.postgres.lookups import Unaccent # noqa from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector # noqa from django.db.models import Model, UUIDField -from django.db.models.functions import Coalesce, Lower, StrIndex # noqa +from django.db.models.functions import Coalesce, Lower, StrIndex, TruncDate # noqa from taiga.base.db.models.fields import * # noqa from taiga.base.utils.uuid import encode_uuid_to_b64str from taiga.conf import settings diff --git a/python/apps/taiga/src/taiga/base/django/apps.py b/python/apps/taiga/src/taiga/base/django/apps.py new file mode 100644 index 000000000..7b1995781 --- /dev/null +++ b/python/apps/taiga/src/taiga/base/django/apps.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from django.apps import AppConfig # noqa diff --git a/python/apps/taiga/src/taiga/base/django/settings.py b/python/apps/taiga/src/taiga/base/django/settings.py index 02a7e83b8..3fb45be47 100644 --- a/python/apps/taiga/src/taiga/base/django/settings.py +++ b/python/apps/taiga/src/taiga/base/django/settings.py @@ -38,6 +38,7 @@ "django.contrib.staticfiles", # taiga "taiga.base.db", + "taiga.commons.storage", "taiga.emails", "taiga.mediafiles", "taiga.projects.invitations", diff --git a/python/apps/taiga/tests/samples/occ/__init__.py b/python/apps/taiga/src/taiga/commons/storage/__init__.py similarity index 100% rename from python/apps/taiga/tests/samples/occ/__init__.py rename to python/apps/taiga/src/taiga/commons/storage/__init__.py diff --git a/python/apps/taiga/src/taiga/commons/storage/admin.py b/python/apps/taiga/src/taiga/commons/storage/admin.py new file mode 100644 index 000000000..147ddabc8 --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/admin.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from taiga.base.db import admin +from taiga.commons.storage.models import StoragedObject + + +@admin.register(StoragedObject) +class StoragedObjectAdmin(admin.ModelAdmin[StoragedObject]): + list_display = ( + "file", + "created_at", + "deleted_at", + ) + readonly_fields = ( + "created_at", + "deleted_at", + ) + search_fields = ("file",) + list_filter = ("created_at", "deleted_at") + ordering = ("-created_at",) diff --git a/python/apps/taiga/src/taiga/commons/storage/commands.py b/python/apps/taiga/src/taiga/commons/storage/commands.py new file mode 100644 index 000000000..910554ac8 --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/commands.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import timedelta + +import typer +from taiga.base.utils import pprint +from taiga.base.utils.concurrency import run_async_as_sync +from taiga.base.utils.datetime import aware_utcnow +from taiga.commons.storage import services as storage_services +from taiga.conf import settings + +cli = typer.Typer( + name="Taiga Storage commands", + help="Manage the starege system of Taiga.", + add_completion=True, +) + + +@cli.command(help="Clean deleted storaged object. Remove entries from DB and files from storage") +def clean_storaged_objects( + days_to_store_deleted_storaged_object: int = typer.Option( + settings.STORAGE.DAYS_TO_STORE_DELETED_STORAGED_OBJECTS, + "--days", + "-d", + help="Number of days to store deleted storaged objects", + ), +) -> None: + total_deleted = run_async_as_sync( + storage_services.clean_deleted_storaged_objects( + before=aware_utcnow() - timedelta(days=days_to_store_deleted_storaged_object) + ) + ) + + color = "red" if total_deleted else "white" + pprint.print(f"Deleted [bold][{color}]{total_deleted}[/{color}][/bold] storaged objects.") diff --git a/python/apps/taiga/src/taiga/commons/storage/migrations/0001_initial.py b/python/apps/taiga/src/taiga/commons/storage/migrations/0001_initial.py new file mode 100644 index 000000000..70b6685d3 --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/migrations/0001_initial.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +# Generated by Django 4.2.3 on 2023-09-01 19:14 + +import django.db.models.functions.datetime +import taiga.base.db.models +import taiga.base.utils.datetime +import taiga.commons.storage.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="StoragedObject", + fields=[ + ( + "id", + models.UUIDField( + blank=True, + default=taiga.base.db.models.uuid_generator, + editable=False, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + default=taiga.base.utils.datetime.aware_utcnow, + verbose_name="created at", + ), + ), + ( + "deleted_at", + models.DateTimeField(blank=True, null=True, verbose_name="deleted at"), + ), + ( + "file", + models.FileField( + max_length=500, + upload_to=taiga.commons.storage.models.get_storaged_object_file_patch, + verbose_name="file", + ), + ), + ], + options={ + "verbose_name": "storaged_objects", + "verbose_name_plural": "storaged_objectss", + "ordering": ["-created_at"], + "indexes": [ + models.Index( + django.db.models.functions.datetime.TruncDate("created_at"), + models.F("created_at"), + name="created_at_date_idx", + ), + models.Index( + django.db.models.functions.datetime.TruncDate("deleted_at"), + models.F("deleted_at"), + name="deleted_at_date_idx", + ), + ], + }, + ), + ] diff --git a/python/apps/taiga/src/taiga/commons/storage/migrations/__init__.py b/python/apps/taiga/src/taiga/commons/storage/migrations/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/migrations/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/src/taiga/commons/storage/models.py b/python/apps/taiga/src/taiga/commons/storage/models.py new file mode 100644 index 000000000..185318b37 --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/models.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from taiga.base.db import models +from taiga.base.db.mixins import CreatedAtMetaInfoMixin, DeletedAtMetaInfoMixin +from taiga.base.utils.files import get_obfuscated_file_path + + +def get_storaged_object_file_patch(instance: "StoragedObject", filename: str) -> str: + return get_obfuscated_file_path(instance, filename, "storagedobjets") + + +class StoragedObject(models.BaseModel, CreatedAtMetaInfoMixin, DeletedAtMetaInfoMixin): + file = models.FileField( + upload_to=get_storaged_object_file_patch, + max_length=500, + null=False, + blank=False, + verbose_name="file", + ) + + class Meta: + verbose_name = "storaged_objects" + verbose_name_plural = "storaged_objectss" + indexes = [ + models.Index(models.TruncDate("created_at"), "created_at", name="created_at_date_idx"), + models.Index(models.TruncDate("deleted_at"), "deleted_at", name="deleted_at_date_idx"), + ] + ordering = [ + "-created_at", + ] + + def __str__(self) -> str: + return self.file.name + + def __repr__(self) -> str: + return f"" diff --git a/python/apps/taiga/src/taiga/commons/storage/repositories.py b/python/apps/taiga/src/taiga/commons/storage/repositories.py new file mode 100644 index 000000000..244ae5a1f --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/repositories.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import datetime +from typing import TypedDict +from uuid import UUID + +from taiga.base.db.exceptions import RestrictedError +from taiga.base.db.models import QuerySet +from taiga.base.utils.datetime import aware_utcnow +from taiga.base.utils.files import File +from taiga.commons.storage.models import StoragedObject + +########################################################## +# filters and querysets +########################################################## + + +DEFAULT_QUERYSET = StoragedObject.objects.all() + + +class StoragedObjectFilters(TypedDict, total=False): + id: UUID + deleted_before: datetime + + +async def _apply_filters_to_queryset( + qs: QuerySet[StoragedObject], + filters: StoragedObjectFilters = {}, +) -> QuerySet[StoragedObject]: + filter_data = dict(filters.copy()) + + if "deleted_before" in filter_data: + deleted_before = filter_data.pop("deleted_before") + filter_data["deleted_at__lt"] = deleted_before + + return qs.filter(**filter_data) + + +########################################################## +# create storaged object +########################################################## + + +async def create_storaged_object( + file: File, +) -> StoragedObject: + return await StoragedObject.objects.acreate(file=file) + + +########################################################## +# list storaged object +########################################################## + + +async def list_storaged_objects(filters: StoragedObjectFilters = {}) -> list[StoragedObject]: + qs = await _apply_filters_to_queryset(qs=DEFAULT_QUERYSET, filters=filters) + return [so async for so in qs] + + +########################################################## +# delete storaged object +######################################################## + + +async def delete_storaged_object( + storaged_object: StoragedObject, +) -> bool: + try: + await storaged_object.adelete() + storaged_object.file.delete(save=False) + return True + except RestrictedError: + # This happens when you try to delete a StoragedObject that is being used by someone + # (using ForeignKey with on_delete=PROTECT). + + # TODO: log this + return False + + +def mark_storaged_object_as_deleted( + storaged_object: StoragedObject, +) -> None: + storaged_object.deleted_at = aware_utcnow() + storaged_object.save(update_fields=["deleted_at"]) diff --git a/python/apps/taiga/src/taiga/commons/storage/serializers.py b/python/apps/taiga/src/taiga/commons/storage/serializers.py new file mode 100644 index 000000000..bcba3451a --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/serializers.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from pydantic import AnyHttpUrl, AnyUrl, BaseConfig +from pydantic.fields import ModelField +from taiga.commons.storage.models import StoragedObject + + +class StoragedObjectFileField(AnyHttpUrl): + @classmethod + def validate(cls, value: StoragedObject, field: ModelField, config: BaseConfig) -> AnyUrl: + return value.file.url diff --git a/python/apps/taiga/src/taiga/commons/storage/services.py b/python/apps/taiga/src/taiga/commons/storage/services.py new file mode 100644 index 000000000..75e744aee --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/services.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import datetime + +from taiga.commons.storage import repositories as storage_repositories + + +async def clean_deleted_storaged_objects(before: datetime) -> int: + storaged_objects = await storage_repositories.list_storaged_objects(filters={"deleted_before": before}) + deleted = 0 + for storaged_object in storaged_objects: + if storage_repositories.delete_storaged_object(storaged_object=storaged_object): + deleted += 1 + + return deleted diff --git a/python/apps/taiga/src/taiga/commons/storage/tasks.py b/python/apps/taiga/src/taiga/commons/storage/tasks.py new file mode 100644 index 000000000..21b96f5fc --- /dev/null +++ b/python/apps/taiga/src/taiga/commons/storage/tasks.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import timedelta + +from taiga.base.utils.datetime import aware_utcnow +from taiga.commons.storage import services as storage_services +from taiga.conf import settings +from taiga.tasksqueue.manager import manager as tqmanager + + +@tqmanager.periodic(cron=settings.STORAGE.CLEAN_DELETED_STORAGE_OBJECTS_CRON) # type: ignore +@tqmanager.task +async def clean_deleted_storaged_objects(timestamp: int) -> int: + if not settings.STORAGE.DAYS_TO_STORE_DELETED_STORAGED_OBJECTS: + return 0 + + return await storage_services.clean_deleted_storaged_objects( + before=aware_utcnow() - timedelta(days=settings.STORAGE.DAYS_TO_STORE_DELETED_STORAGED_OBJECTS) + ) diff --git a/python/apps/taiga/src/taiga/conf/__init__.py b/python/apps/taiga/src/taiga/conf/__init__.py index e37cef96c..dc89eaba3 100644 --- a/python/apps/taiga/src/taiga/conf/__init__.py +++ b/python/apps/taiga/src/taiga/conf/__init__.py @@ -17,6 +17,7 @@ from taiga.conf.events import EventsSettings from taiga.conf.images import ImageSettings from taiga.conf.logs import LOGGING_CONFIG +from taiga.conf.storage import StorageSettings from taiga.conf.tasksqueue import TaskQueueSettings from taiga.conf.tokens import TokensSettings @@ -98,6 +99,7 @@ class Settings(BaseSettings): EMAIL: EmailSettings = EmailSettings() EVENTS: EventsSettings = EventsSettings() IMAGES: ImageSettings = ImageSettings() + STORAGE: StorageSettings = StorageSettings() TASKQUEUE: TaskQueueSettings = TaskQueueSettings() TOKENS: TokensSettings = TokensSettings() diff --git a/python/apps/taiga/src/taiga/conf/storage.py b/python/apps/taiga/src/taiga/conf/storage.py new file mode 100644 index 000000000..53c96cd83 --- /dev/null +++ b/python/apps/taiga/src/taiga/conf/storage.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from pydantic import BaseSettings + + +class StorageSettings(BaseSettings): + CLEAN_DELETED_STORAGE_OBJECTS_CRON: str = "0 4 * * *" # default: once a day, at 4:00 AM + DAYS_TO_STORE_DELETED_STORAGED_OBJECTS: int | None = 30 # 90 day diff --git a/python/apps/taiga/src/taiga/conf/tasksqueue.py b/python/apps/taiga/src/taiga/conf/tasksqueue.py index 7602c24a9..4c1cac184 100644 --- a/python/apps/taiga/src/taiga/conf/tasksqueue.py +++ b/python/apps/taiga/src/taiga/conf/tasksqueue.py @@ -11,8 +11,9 @@ class TaskQueueSettings(BaseSettings): # We must include all the modules that define tasks. TASKS_MODULES_PATHS: set[str] = { + "taiga.commons.storage.tasks", "taiga.emails.tasks", + "taiga.projects.projects.tasks", "taiga.tokens.tasks", "taiga.users.tasks", - "taiga.projects.projects.tasks", } diff --git a/python/apps/taiga/tests/conf_django.py b/python/apps/taiga/tests/conf_django.py index 462b47cb7..1fffa0d54 100644 --- a/python/apps/taiga/tests/conf_django.py +++ b/python/apps/taiga/tests/conf_django.py @@ -8,7 +8,8 @@ import os -from taiga.base.django.settings import * # noqa, pylint: disable=unused-wildcard-import +from taiga.base.django.settings import * # noqa +from taiga.base.django.settings import INSTALLED_APPS DEBUG = True @@ -16,9 +17,12 @@ STATIC_ROOT = "/tmp/taiga/static" -INSTALLED_APPS += [ # noqa: F405 - "tests.samples.occ", -] +INSTALLED_APPS = [ + # NOTE: We should add sample apps first because pytest-django uses an strange strategy to load models to the db and + # it causes integrity errors with foreign keys and many to many fields + "tests.samples.test_occ", + "tests.samples.test_storage", +] + INSTALLED_APPS # This is only for GitHubActions diff --git a/python/apps/taiga/tests/conftest.py b/python/apps/taiga/tests/conftest.py index cd0801e65..1a8e72fa3 100644 --- a/python/apps/taiga/tests/conftest.py +++ b/python/apps/taiga/tests/conftest.py @@ -55,9 +55,11 @@ def django_db_setup(django_db_setup, django_db_blocker): @pytest_asyncio.fixture(scope="session", autouse=True) async def connect_events_manage_on_startup(): - from taiga.events import connect_events_manager + from taiga.events import connect_events_manager, disconnect_events_manager await connect_events_manager() + yield + await disconnect_events_manager() # @@ -65,7 +67,6 @@ async def connect_events_manage_on_startup(): # def pytest_addoption(parser): parser.addoption("--slow_only", action="store_true", default=False, help="run slow tests only") - parser.addoption("--fast_only", action="store_true", default=False, help="exclude slow tests") diff --git a/python/apps/taiga/tests/integration/taiga/attachments/test_signals.py b/python/apps/taiga/tests/integration/taiga/attachments/test_signals.py new file mode 100644 index 000000000..adb874cef --- /dev/null +++ b/python/apps/taiga/tests/integration/taiga/attachments/test_signals.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +import pytest +from taiga.attachments import repositories as attachments_repositories +from taiga.attachments.models import Attachment +from taiga.attachments.signals import mark_attachment_file_to_delete +from taiga.workspaces.workspaces import repositories as workspaces_repositories +from tests.utils import db as db_utils +from tests.utils import factories as f +from tests.utils import signals as signals_utils + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_mark_attachment_file_to_delete_is_connected(): + assert mark_attachment_file_to_delete in signals_utils.get_receivers_for_model("post_delete", Attachment) + + +async def test_mark_attachment_file_to_delete_when_delete_first_level_related_model(): + story = await f.create_story() + attachment = await f.create_attachment(content_object=story) + file = attachment.file + + assert file.deleted_at is None + + assert await attachments_repositories.delete_attachments(filters={"id": attachment.id}) + await db_utils.refresh_model_from_db(file) + + assert file.deleted_at + + +async def test_mark_attachment_file_to_delete_when_delete_n_level_related_object(): + workspace = await f.create_workspace() + project1 = await f.create_project(workspace=workspace) + project2 = await f.create_project(workspace=workspace) + story1 = await f.create_story(project=project1) + story2 = await f.create_story(project=project2) + attachment11 = await f.create_attachment(content_object=story1) + attachment12 = await f.create_attachment(content_object=story1) + attachment21 = await f.create_attachment(content_object=story2) + + file11 = attachment11.file + file12 = attachment12.file + file21 = attachment21.file + + assert file11.deleted_at is None + assert file12.deleted_at is None + assert file21.deleted_at is None + + assert await workspaces_repositories.delete_workspaces(filters={"id": workspace.id}) + + await db_utils.refresh_model_from_db(file11) + await db_utils.refresh_model_from_db(file12) + await db_utils.refresh_model_from_db(file21) + + assert file11.deleted_at + assert file12.deleted_at + assert file21.deleted_at diff --git a/python/apps/taiga/tests/integration/taiga/base/occ/test_repositories.py b/python/apps/taiga/tests/integration/taiga/base/occ/test_repositories.py index 07a542819..70eee8365 100644 --- a/python/apps/taiga/tests/integration/taiga/base/occ/test_repositories.py +++ b/python/apps/taiga/tests/integration/taiga/base/occ/test_repositories.py @@ -7,7 +7,7 @@ import pytest from taiga.base.occ import repositories -from tests.samples.occ.models import SampleOCCItem +from tests.samples.test_occ.models import SampleOCCItem pytestmark = pytest.mark.django_db(transaction=True) diff --git a/python/apps/taiga/tests/integration/taiga/commons/storage/__init__.py b/python/apps/taiga/tests/integration/taiga/commons/storage/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/tests/integration/taiga/commons/storage/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/tests/integration/taiga/commons/storage/test_repositories.py b/python/apps/taiga/tests/integration/taiga/commons/storage/test_repositories.py new file mode 100644 index 000000000..13b41c76c --- /dev/null +++ b/python/apps/taiga/tests/integration/taiga/commons/storage/test_repositories.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import timedelta +from uuid import uuid1 + +import pytest +from asgiref.sync import sync_to_async +from taiga.base.utils.datetime import aware_utcnow +from taiga.commons.storage import repositories +from tests.samples.test_storage.models import SampleAttachment +from tests.utils import factories as f + +pytestmark = pytest.mark.django_db(transaction=True) + + +############################################################# +# create_storaged_objects +############################################################# + + +async def test_create_storaged_object(): + file = f.build_image_file(name="test") + + storaged_object = await repositories.create_storaged_object( + file=file, + ) + + assert storaged_object.id + assert len(await repositories.list_storaged_objects(filters={"id": storaged_object.id})) == 1 + + +########################################################## +# list_storaged_objects +########################################################## + + +async def test_list_storage_objects(): + storaged_object1 = await f.create_storaged_object() + storaged_object2 = await f.create_storaged_object(deleted_at=aware_utcnow() - timedelta(days=3)) + + assert await repositories.list_storaged_objects() == [ + storaged_object2, + storaged_object1, + ] + + +async def test_list_storage_objects_filters_by_id(): + storaged_object1 = await f.create_storaged_object() + await f.create_storaged_object(deleted_at=aware_utcnow() - timedelta(days=3)) + + assert await repositories.list_storaged_objects(filters={"id": uuid1()}) == [] + assert await repositories.list_storaged_objects(filters={"id": storaged_object1.id}) == [storaged_object1] + + +async def test_list_storage_objects_filters_by_deleted_datetime(): + await f.create_storaged_object() + storaged_object2 = await f.create_storaged_object(deleted_at=aware_utcnow() - timedelta(days=3)) + + assert ( + await repositories.list_storaged_objects(filters={"deleted_before": aware_utcnow() - timedelta(days=4)}) == [] + ) + assert await repositories.list_storaged_objects(filters={"deleted_before": aware_utcnow() - timedelta(days=2)}) == [ + storaged_object2 + ] + + +########################################################## +# delete_storaged_object +########################################################## + + +async def test_delete_storaged_object(): + storaged_object = await f.create_storaged_object() + file_path = storaged_object.file.path + storage = storaged_object.file.storage + + assert len(await repositories.list_storaged_objects()) == 1 + assert storage.exists(file_path) + + await repositories.delete_storaged_object(storaged_object=storaged_object) + + assert len(await repositories.list_storaged_objects()) == 0 + assert not storage.exists(file_path) + + +async def test_delete_storaged_object_that_has_been_used(): + storaged_object = await f.create_storaged_object() + file_path = storaged_object.file.path + storage = storaged_object.file.storage + + await SampleAttachment.objects.acreate(storaged_object=storaged_object) + + assert len(await repositories.list_storaged_objects()) == 1 + assert storage.exists(file_path) + + assert not await repositories.delete_storaged_object(storaged_object=storaged_object) + + assert len(await repositories.list_storaged_objects()) == 1 + assert storage.exists(file_path) + + +########################################################## +# mark_storaged_object_as_deleted +########################################################## + + +async def test_mark_storaged_object_as_deleted(): + storaged_object = await f.create_storaged_object() + + assert not storaged_object.deleted_at + await sync_to_async(repositories.mark_storaged_object_as_deleted)(storaged_object=storaged_object) + assert storaged_object.deleted_at diff --git a/python/apps/taiga/tests/samples/test_occ/__init__.py b/python/apps/taiga/tests/samples/test_occ/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/tests/samples/test_occ/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/tests/samples/occ/models.py b/python/apps/taiga/tests/samples/test_occ/models.py similarity index 100% rename from python/apps/taiga/tests/samples/occ/models.py rename to python/apps/taiga/tests/samples/test_occ/models.py diff --git a/python/apps/taiga/tests/samples/test_storage/__init__.py b/python/apps/taiga/tests/samples/test_storage/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/tests/samples/test_storage/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/tests/samples/test_storage/models.py b/python/apps/taiga/tests/samples/test_storage/models.py new file mode 100644 index 000000000..556bc17bd --- /dev/null +++ b/python/apps/taiga/tests/samples/test_storage/models.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from taiga.base.db import models + + +class SampleAttachment(models.Model): + storaged_object = models.ForeignKey( + "storage.StoragedObject", + null=False, + blank=False, + on_delete=models.RESTRICT, + related_name="+", + ) diff --git a/python/apps/taiga/tests/unit/taiga/commons/__init__.py b/python/apps/taiga/tests/unit/taiga/commons/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/tests/unit/taiga/commons/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/tests/unit/taiga/commons/storage/__init__.py b/python/apps/taiga/tests/unit/taiga/commons/storage/__init__.py new file mode 100644 index 000000000..87d9a5256 --- /dev/null +++ b/python/apps/taiga/tests/unit/taiga/commons/storage/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC diff --git a/python/apps/taiga/tests/unit/taiga/commons/storage/test_services.py b/python/apps/taiga/tests/unit/taiga/commons/storage/test_services.py new file mode 100644 index 000000000..b7a107e91 --- /dev/null +++ b/python/apps/taiga/tests/unit/taiga/commons/storage/test_services.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from datetime import timedelta +from unittest.mock import call, patch + +from taiga.base.utils.datetime import aware_utcnow +from taiga.commons.storage import services +from tests.utils import factories as f + + +async def test_clean_deleted_storaged_objects(): + storaged_objects = [ + f.build_storaged_object(), + f.build_storaged_object(), + ] + before_datetime = aware_utcnow() - timedelta(days=1) + + with patch("taiga.commons.storage.services.storage_repositories", autospec=True) as fake_storage_repositories: + fake_storage_repositories.list_storaged_objects.return_value = storaged_objects + + assert await services.clean_deleted_storaged_objects(before=before_datetime) == 2 + + fake_storage_repositories.list_storaged_objects.assert_awaited_once_with( + filters={"deleted_before": before_datetime} + ) + + fake_storage_repositories.delete_storaged_object.assert_has_awaits( + [ + call(storaged_objects[0]), + call(storaged_objects[1]), + ], + ) diff --git a/python/apps/taiga/tests/utils/factories/__init__.py b/python/apps/taiga/tests/utils/factories/__init__.py index 7165edcf3..85296ce32 100644 --- a/python/apps/taiga/tests/utils/factories/__init__.py +++ b/python/apps/taiga/tests/utils/factories/__init__.py @@ -34,6 +34,7 @@ create_project_role, create_simple_project, ) +from .storage import StoragedObjectFactory, build_storaged_object, create_storaged_object # noqa from .stories import ( # noqa StoryAssignmentFactory, StoryFactory, diff --git a/python/apps/taiga/tests/utils/factories/attachments.py b/python/apps/taiga/tests/utils/factories/attachments.py index d975c6482..c1b3c6723 100644 --- a/python/apps/taiga/tests/utils/factories/attachments.py +++ b/python/apps/taiga/tests/utils/factories/attachments.py @@ -11,8 +11,8 @@ class AttachmentFactory(Factory): + file = factory.SubFactory("tests.utils.factories.StoragedObjectFactory") name = factory.Sequence(lambda n: f"test-file-{n}.png") - file = factory.django.ImageField(format="PNG") content_type = "image/png" size = 145 diff --git a/python/apps/taiga/tests/utils/factories/storage.py b/python/apps/taiga/tests/utils/factories/storage.py new file mode 100644 index 000000000..a68ec1170 --- /dev/null +++ b/python/apps/taiga/tests/utils/factories/storage.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from asgiref.sync import sync_to_async + +from .base import Factory, factory + + +class StoragedObjectFactory(Factory): + file = factory.django.ImageField(format="PNG") + + class Meta: + model = "storage.StoragedObject" + + +@sync_to_async +def create_storaged_object(**kwargs): + return StoragedObjectFactory.create(**kwargs) + + +def build_storaged_object(**kwargs): + return StoragedObjectFactory.build(**kwargs) diff --git a/python/apps/taiga/tests/utils/signals.py b/python/apps/taiga/tests/utils/signals.py new file mode 100644 index 000000000..631d9d856 --- /dev/null +++ b/python/apps/taiga/tests/utils/signals.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2023-present Kaleidos INC + +from taiga.base.db.models import Model, signals + + +def get_receivers_for_model(listener_name: str, sender: Model): + """ + Returns a list of all receivers functions for a given listener name ("post_save", "post_delete"...) + and sender (a model class). + """ + dispatcher = getattr(signals, listener_name) + return dispatcher._live_receivers(sender)