From f601d39140a410bed5406f1573d756c2f45e36db Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 6 Jun 2024 23:56:21 +0545 Subject: [PATCH 01/17] Add LeadreviewAttachment Model --- .../entry_images_v2/migrate.py | 4 +- apps/commons/receivers.py | 4 +- apps/deepl_integration/handlers.py | 59 ++++++++++++++----- apps/deepl_integration/serializers.py | 22 +++++-- apps/entry/serializers.py | 6 +- apps/entry/tests/test_apis.py | 6 +- apps/lead/admin.py | 8 +-- apps/lead/dataloaders.py | 15 ++++- apps/lead/enums.py | 1 + apps/lead/factories.py | 6 +- .../migrations/0050_auto_20240606_0608.py | 29 +++++++++ apps/lead/models.py | 15 ++++- apps/lead/schema.py | 22 ++++++- apps/lead/serializers.py | 16 ++--- apps/lead/tests/test_apis.py | 19 ++++-- apps/lead/tests/test_mutations.py | 4 +- apps/lead/views.py | 4 +- schema.graphql | 15 +++++ 18 files changed, 198 insertions(+), 57 deletions(-) create mode 100644 apps/lead/migrations/0050_auto_20240606_0608.py diff --git a/apps/bulk_data_migration/entry_images_v2/migrate.py b/apps/bulk_data_migration/entry_images_v2/migrate.py index 8a456902a5..56db783f35 100644 --- a/apps/bulk_data_migration/entry_images_v2/migrate.py +++ b/apps/bulk_data_migration/entry_images_v2/migrate.py @@ -3,7 +3,7 @@ from utils.common import parse_number -from lead.models import LeadPreviewImage +from lead.models import LeadPreviewAttachment from entry.models import Entry from gallery.models import File @@ -34,7 +34,7 @@ def _get_file_from_s3_url(entry, string): return # NOTE: For lead-preview generate gallery files if file_path.startswith('lead-preview/'): - lead_preview = LeadPreviewImage.objects.filter(file=file_path).first() + lead_preview = LeadPreviewAttachment.objects.filter(file=file_path).first() if lead_preview and lead_preview.file and lead_preview.file.storage.exists(lead_preview.file.name): return lead_preview.clone_as_deep_file(entry.created_by) return diff --git a/apps/commons/receivers.py b/apps/commons/receivers.py index 0ee99c8411..088ce696d3 100644 --- a/apps/commons/receivers.py +++ b/apps/commons/receivers.py @@ -5,14 +5,14 @@ from lead.models import ( LeadPreview, - LeadPreviewImage, + LeadPreviewAttachment, ) from unified_connector.models import ConnectorLeadPreviewImage # Lead @receiver(models.signals.post_delete, sender=LeadPreview) -@receiver(models.signals.post_delete, sender=LeadPreviewImage) +@receiver(models.signals.post_delete, sender=LeadPreviewAttachment) # Unified Connector @receiver(models.signals.post_delete, sender=ConnectorLeadPreviewImage) def cleanup_file_on_instance_delete(sender, instance, **kwargs): diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 031a5f3d63..5cb5ee4223 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -37,7 +37,7 @@ from lead.models import ( Lead, LeadPreview, - LeadPreviewImage, + LeadPreviewAttachment, ) from lead.typings import NlpExtractorDocument from entry.models import Entry @@ -636,13 +636,14 @@ def trigger_lead_extract(cls, lead, task_instance=None): def save_data( lead: Lead, text_source_uri: str, - images_uri: List[str], + images_uri: List[dict], + table_uri: List[dict], word_count: int, page_count: int, text_extraction_id: str, ): LeadPreview.objects.filter(lead=lead).delete() - LeadPreviewImage.objects.filter(lead=lead).delete() + LeadPreviewAttachment.objects.filter(lead=lead).delete() # and create new one LeadPreview.objects.create( lead=lead, @@ -651,18 +652,44 @@ def save_data( page_count=page_count, text_extraction_id=text_extraction_id, ) - # Save extracted images as LeadPreviewImage instances + # Save extracted images as LeadPreviewAttachment instances # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? - image_base_path = f'{lead.pk}' - for image_uri in images_uri: - lead_image = LeadPreviewImage(lead=lead) - image_obj = RequestHelper(url=image_uri, ignore_error=True).get_file() - if image_obj: - lead_image.file.save( - os.path.join(image_base_path, os.path.basename(urlparse(image_uri).path)), - image_obj + + attachement_base_path = f'{lead.pk}' + images = [dict(item) for item in images_uri] + for image_uri in images: + for image in image_uri['images']: + lead_attachement = LeadPreviewAttachment(lead=lead) + image_obj = RequestHelper(url=image, ignore_error=True).get_file() + if image_obj: + lead_attachement.file.save( + os.path.join(attachement_base_path, os.path.basename(urlparse(image).path)), + image_obj + ) + lead_attachement.page_number = image_uri['page_number'] + lead_attachement.type = LeadPreviewAttachment.AttachementFileType.IMAGE + lead_attachement.file_preview = lead_attachement.file + + lead_attachement.save() + + table_path = [dict(item) for item in table_uri] + for table in table_path: + lead_attachement = LeadPreviewAttachment(lead=lead) + table_img = RequestHelper(url=table['image_link'], ignore_error=True).get_file() + table_attahcment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() + if table_img: + lead_attachement.file_preview.save( + os.path.join(attachement_base_path, os.path.basename(urlparse(table['image_link']).path)), + table_img ) - lead_image.save() + lead_attachement.page_number = table['page_number'] + lead_attachement.type = LeadPreviewAttachment.AttachementFileType.XLSX + lead_attachement.file.save( + os.path.join(attachement_base_path, os.path.basename(urlparse(table['content_link']).path)), + table_attahcment + ) + lead_attachement.save() + lead.update_extraction_status(Lead.ExtractionStatus.SUCCESS) return lead @@ -674,7 +701,7 @@ def save_lead_data_using_connector_lead( if connector_lead.extraction_status != ConnectorLead.ExtractionStatus.SUCCESS: return False LeadPreview.objects.filter(lead=lead).delete() - LeadPreviewImage.objects.filter(lead=lead).delete() + LeadPreviewAttachment.objects.filter(lead=lead).delete() # and create new one LeadPreview.objects.create( lead=lead, @@ -683,10 +710,10 @@ def save_lead_data_using_connector_lead( page_count=connector_lead.page_count, text_extraction_id=connector_lead.text_extraction_id, ) - # Save extracted images as LeadPreviewImage instances + # Save extracted images as LeadPreviewAttachment instances # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? for connector_lead_preview_image in connector_lead.preview_images.all(): - lead_image = LeadPreviewImage(lead=lead) + lead_image = LeadPreviewAttachment(lead=lead) lead_image.file.save( connector_lead_preview_image.image.name, connector_lead_preview_image.image, diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index c4d9f98d15..203701e542 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -63,6 +63,21 @@ class Status(models.IntegerChoices): status = serializers.ChoiceField(choices=Status.choices) +class ImagePathSerializer(serializers.Serializer): + page_number = serializers.IntegerField(required=True) + images = serializers.ListField( + child=serializers.CharField(allow_blank=True), + default=[] + ) + + +class TablePathSerializer(serializers.Serializer): + page_number = serializers.IntegerField(required=True) + order = serializers.IntegerField(required=True) + image_link = serializers.URLField(required=True) + content_link = serializers.URLField(required=True) + + # -- Lead class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ @@ -70,10 +85,8 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ url = serializers.CharField(required=False) # Data fields - images_path = serializers.ListField( - child=serializers.CharField(allow_blank=True), - required=False, default=[], - ) + images_path = serializers.ListSerializer(child=ImagePathSerializer(required=False)) + tables_path = serializers.ListSerializer(child=TablePathSerializer(required=False)) text_path = serializers.CharField(required=False, allow_null=True) total_words_count = serializers.IntegerField(required=False, default=0, allow_null=True) total_pages = serializers.IntegerField(required=False, default=0, allow_null=True) @@ -106,6 +119,7 @@ def create(self, data): lead, data['text_path'], data.get('images_path', [])[:10], # TODO: Support for more images, too much image will error. + data.get('tables_path', []), data.get('total_words_count'), data.get('total_pages'), data.get('text_extraction_id'), diff --git a/apps/entry/serializers.py b/apps/entry/serializers.py index 8bef07f866..1340b53897 100644 --- a/apps/entry/serializers.py +++ b/apps/entry/serializers.py @@ -17,7 +17,7 @@ from gallery.serializers import FileSerializer, SimpleFileSerializer from project.models import Project from lead.serializers import LeadSerializer -from lead.models import Lead, LeadPreviewImage +from lead.models import Lead, LeadPreviewAttachment from analysis_framework.serializers import AnalysisFrameworkSerializer from geo.models import GeoArea, Region from geo.serializers import SimpleRegionSerializer @@ -211,7 +211,7 @@ class EntrySerializer(RemoveNullFieldsMixin, lead_image = serializers.PrimaryKeyRelatedField( required=False, write_only=True, - queryset=LeadPreviewImage.objects.all() + queryset=LeadPreviewAttachment.objects.all() ) # NOTE: Provided by annotate `annotate_comment_count` verified_by_count = serializers.IntegerField(read_only=True) @@ -594,7 +594,7 @@ class EntryGqSerializer(ProjectPropertySerializerMixin, TempClientIdMixin, UserR lead_image = serializers.PrimaryKeyRelatedField( required=False, write_only=True, - queryset=LeadPreviewImage.objects.all(), + queryset=LeadPreviewAttachment.objects.all(), help_text=( 'This is used to add images from Lead Preview Images.' ' This will be changed into gallery image and supplied back in image field.' diff --git a/apps/entry/tests/test_apis.py b/apps/entry/tests/test_apis.py index a090be0f69..be3b591caf 100644 --- a/apps/entry/tests/test_apis.py +++ b/apps/entry/tests/test_apis.py @@ -5,7 +5,7 @@ from deep.tests import TestCase from project.models import Project from user.models import User -from lead.models import Lead, LeadPreviewImage +from lead.models import Lead, LeadPreviewAttachment from organization.models import Organization, OrganizationType from analysis_framework.models import ( AnalysisFramework, Widget, Filter @@ -724,7 +724,7 @@ def test_entry_image_validation(self): self.authenticate() # Using lead image (same lead) - data['lead_image'] = self.create(LeadPreviewImage, lead=lead, file=image.file).pk + data['lead_image'] = self.create(LeadPreviewAttachment, lead=lead, file=image.file).pk response = self.client.post(url, data) self.assert_201(response) assert 'image' in response.data @@ -732,7 +732,7 @@ def test_entry_image_validation(self): data.pop('lead_image') # Using lead image (different lead) - data['lead_image'] = self.create(LeadPreviewImage, lead=self.create_lead(), file=image.file).pk + data['lead_image'] = self.create(LeadPreviewAttachment, lead=self.create_lead(), file=image.file).pk response = self.client.post(url, data) self.assert_400(response) data.pop('lead_image') diff --git a/apps/lead/admin.py b/apps/lead/admin.py index 0052268ff3..ef6249ce34 100644 --- a/apps/lead/admin.py +++ b/apps/lead/admin.py @@ -7,7 +7,7 @@ from .tasks import extract_from_lead from .models import ( Lead, LeadGroup, - LeadPreview, LeadPreviewImage, + LeadPreview, LeadPreviewAttachment, EMMEntity, ) @@ -16,8 +16,8 @@ class LeadPreviewInline(admin.StackedInline): model = LeadPreview -class LeadPreviewImageInline(admin.TabularInline): - model = LeadPreviewImage +class LeadPreviewAttachmentInline(admin.TabularInline): + model = LeadPreviewAttachment extra = 0 @@ -42,7 +42,7 @@ def trigger_lead_extract(modeladmin, request, queryset): @admin.register(Lead) class LeadAdmin(VersionAdmin): - inlines = [LeadPreviewInline, LeadPreviewImageInline] + inlines = [LeadPreviewInline, LeadPreviewAttachmentInline] search_fields = ['title'] list_filter = ( AutocompleteFilterFactory('Project', 'project'), diff --git a/apps/lead/dataloaders.py b/apps/lead/dataloaders.py index 91a6e43c8f..3695171020 100644 --- a/apps/lead/dataloaders.py +++ b/apps/lead/dataloaders.py @@ -11,7 +11,7 @@ from organization.dataloaders import OrganizationLoader -from .models import Lead, LeadPreview, LeadGroup +from .models import Lead, LeadPreview, LeadGroup, LeadPreviewAttachment from assisted_tagging.models import DraftEntry from assessment_registry.models import AssessmentRegistry @@ -26,6 +26,15 @@ def batch_load_fn(self, keys): return Promise.resolve([_map.get(key) for key in keys]) +class LeadPreviewAttachmentLoader(DataLoaderWithContext): + def batch_load_fn(self, keys): + lead_preview_attachment_qs = LeadPreviewAttachment.objects.filter(lead__in=keys) + lead_preview_attachments = defaultdict(list) + for lead_preview_attachment in lead_preview_attachment_qs: + lead_preview_attachments[lead_preview_attachment.lead_id].append(lead_preview_attachment) + return Promise.resolve([lead_preview_attachments.get(key) for key in keys]) + + class EntriesCountLoader(DataLoaderWithContext): def batch_load_fn(self, keys): active_af = self.context.active_project.analysis_framework @@ -137,6 +146,10 @@ class DataLoaders(WithContextMixin): def lead_preview(self): return LeadPreviewLoader(context=self.context) + @cached_property + def lead_preview_attachment(self): + return LeadPreviewAttachmentLoader(context=self.context) + @cached_property def entries_count(self): return EntriesCountLoader(context=self.context) diff --git a/apps/lead/enums.py b/apps/lead/enums.py index a767fe58de..39516159ed 100644 --- a/apps/lead/enums.py +++ b/apps/lead/enums.py @@ -16,6 +16,7 @@ Lead.AutoExtractionStatus, name='LeadAutoEntryExtractionTypeEnum' ) + enum_map = { get_enum_name_from_django_field(field): enum for field, enum in ( diff --git a/apps/lead/factories.py b/apps/lead/factories.py index 167974f145..2f60d667d8 100644 --- a/apps/lead/factories.py +++ b/apps/lead/factories.py @@ -11,7 +11,7 @@ LeadGroup, LeadEMMTrigger, LeadPreview, - LeadPreviewImage, + LeadPreviewAttachment, UserSavedLeadFilter, ) @@ -84,9 +84,9 @@ class Meta: model = LeadPreview -class LeadPreviewImageFactory(DjangoModelFactory): +class LeadPreviewAttachmentFactory(DjangoModelFactory): class Meta: - model = LeadPreviewImage + model = LeadPreviewAttachment class UserSavedLeadFilterFactory(DjangoModelFactory): diff --git a/apps/lead/migrations/0050_auto_20240606_0608.py b/apps/lead/migrations/0050_auto_20240606_0608.py new file mode 100644 index 0000000000..bf17c0883c --- /dev/null +++ b/apps/lead/migrations/0050_auto_20240606_0608.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.25 on 2024-06-06 06:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0049_auto_20231121_0926_squashed_0054_auto_20231218_0552'), + ] + + operations = [ + migrations.CreateModel( + name='LeadPreviewAttachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.IntegerField(default=0)), + ('page_number', models.IntegerField(default=0)), + ('type', models.CharField(choices=[('XLSX', 'XLSX'), ('image', 'Image')], max_length=20)), + ('file', models.FileField(upload_to='lead-preview/attachments/')), + ('file_preview', models.FileField(upload_to='lead-preview/attachments-preview/')), + ('lead', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to='lead.lead')), + ], + ), + migrations.DeleteModel( + name='LeadPreviewImage', + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index eb04b1fe5a..98403aa87f 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -372,14 +372,25 @@ def __str__(self): return 'Text extracted for {}'.format(self.lead) -class LeadPreviewImage(models.Model): +class LeadPreviewAttachment(models.Model): """ NOTE: File can be only used by gallery (when attached to a entry) """ + class AttachementFileType(models.TextChoices): + XLSX = 'XLSX', 'XLSX' + IMAGE = 'image', 'Image' + lead = models.ForeignKey( Lead, related_name='images', on_delete=models.CASCADE, ) - file = models.FileField(upload_to='lead-preview/') + order = models.IntegerField(default=0) + page_number = models.IntegerField(default=0) + type = models.CharField( + max_length=20, + choices=AttachementFileType.choices, + ) + file = models.FileField(upload_to='lead-preview/attachments/') + file_preview = models.FileField(upload_to='lead-preview/attachments-preview/') def __str__(self): return 'Image extracted for {}'.format(self.lead) diff --git a/apps/lead/schema.py b/apps/lead/schema.py index cf352b7031..519aee5b13 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -8,7 +8,7 @@ from utils.graphene.pagination import NoOrderingPageGraphqlPagination from utils.graphene.enums import EnumDescription -from utils.graphene.types import CustomDjangoListObjectType, ClientIdMixin +from utils.graphene.types import CustomDjangoListObjectType, ClientIdMixin, FileFieldType from utils.graphene.fields import DjangoPaginatedListObjectField from user.models import User @@ -33,6 +33,7 @@ LeadEMMTrigger, EMMEntity, UserSavedLeadFilter, + LeadPreviewAttachment, ) from .enums import ( LeadConfidentialityEnum, @@ -216,6 +217,21 @@ class Meta: ) +class LeadPreviewAttachmentsType(DjangoObjectType): + file = graphene.Field(FileFieldType) + file_preview = graphene.Field(FileFieldType) + + class Meta: + model = LeadPreviewAttachment + only_fields = ( + 'type', + 'page_number', + 'order', + 'file', + 'file_preview', + ) + + class LeadEmmTriggerType(DjangoObjectType): class Meta: model = LeadEMMTrigger @@ -347,6 +363,7 @@ class Meta: extraction_status = graphene.Field(LeadExtractionStatusEnum) lead_preview = graphene.Field(LeadPreviewType) + lead_preview_attachments = graphene.List(graphene.NonNull(LeadPreviewAttachmentsType)) source = graphene.Field(OrganizationType) authors = DjangoListField(OrganizationType) assignee = graphene.Field(UserType) @@ -412,6 +429,9 @@ def resolve_attachment(root, info, **kwargs): if root.attachment_id: return info.context.dl.deep_gallery.file.load(root.attachment_id) + def resolve_lead_preview_attachments(root, info, **kwargs): + return info.context.dl.lead.lead_preview_attachment.load(root.pk) + class DraftEntryCountByLead(graphene.ObjectType): undiscarded_draft_entry = graphene.Int(required=False) diff --git a/apps/lead/serializers.py b/apps/lead/serializers.py index f8879ae09e..0267122691 100644 --- a/apps/lead/serializers.py +++ b/apps/lead/serializers.py @@ -37,7 +37,7 @@ Lead, LeadEMMTrigger, LeadGroup, - LeadPreviewImage, + LeadPreviewAttachment, UserSavedLeadFilter, ) @@ -288,9 +288,11 @@ def update(self, instance, validated_data): return lead -class LeadPreviewImageSerializer(RemoveNullFieldsMixin, - DynamicFieldsMixin, - serializers.ModelSerializer): +class LeadPreviewAttachmentSerializer( + RemoveNullFieldsMixin, + DynamicFieldsMixin, + serializers.ModelSerializer +): """ Serializer for lead preview image """ @@ -298,7 +300,7 @@ class LeadPreviewImageSerializer(RemoveNullFieldsMixin, file = URLCachedFileField(read_only=True) class Meta: - model = LeadPreviewImage + model = LeadPreviewAttachment fields = ('id', 'file',) @@ -310,7 +312,7 @@ class LeadPreviewSerializer(RemoveNullFieldsMixin, text = serializers.CharField(source='leadpreview.text_extract', read_only=True) - images = LeadPreviewImageSerializer(many=True, read_only=True) + images = LeadPreviewAttachmentSerializer(many=True, read_only=True) classified_doc_id = serializers.IntegerField( source='leadpreview.classified_doc_id', read_only=True, @@ -634,7 +636,7 @@ def _get_clone_ready(obj, lead): new_lead.authors.set(authors) # Clone Many to one Fields - LeadPreviewImage.objects.bulk_create([ + LeadPreviewAttachment.objects.bulk_create([ _get_clone_ready(image, new_lead) for image in preview_images ]) LeadEMMTrigger.objects.bulk_create([ diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 681e717ca0..5a813e2537 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -37,7 +37,7 @@ from lead.models import ( Lead, LeadPreview, - LeadPreviewImage, + LeadPreviewAttachment, EMMEntity, LeadEMMTrigger, LeadGroup, @@ -811,7 +811,7 @@ def test_lead_copy(self): # Generating Foreign elements for lead1 self.create(LeadPreview, lead=lead1, text_extract=lead1_text_extract) - self.create(LeadPreviewImage, lead=lead1, file=lead1_preview_file) + self.create(LeadPreviewAttachment, lead=lead1, file=lead1_preview_file) emm_trigger = self.create( LeadEMMTrigger, lead=lead1, emm_keyword=emm_keyword, emm_risk_factor=emm_risk_factor, count=emm_count) lead1.emm_entities.set([self.create(EMMEntity, name=emm_entity_name)]) @@ -1797,7 +1797,16 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f data = { 'client_id': LeadExtractionHandler.get_client_id(self.lead), - 'images_path': ['http://random.com/image1.jpeg', 'http://random.com/image1.jpeg'], + 'images_path': [ + { + 'page_number': 1, + 'images': [ + 'http://random.com/image1.jpeg', + 'http://random.com/image1.jpeg' + ], + } + ], + 'tables_path': [], 'text_path': 'http://random.com/extracted_file.txt', 'url': 'http://random.com/pdf_file.pdf', 'total_words_count': 300, @@ -1812,7 +1821,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f self.lead.refresh_from_db() self.assertEqual(self.lead.extraction_status, Lead.ExtractionStatus.FAILED) self.assertEqual(LeadPreview.objects.filter(lead=self.lead).count(), 0) - self.assertEqual(LeadPreviewImage.objects.filter(lead=self.lead).count(), 0) + self.assertEqual(LeadPreviewAttachment.objects.filter(lead=self.lead).count(), 0) data['status'] = DeeplServerBaseCallbackSerializer.Status.SUCCESS.value # After callback [Success] @@ -1826,7 +1835,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f self.assertEqual(lead_preview.text_extract, 'Extracted text') self.assertEqual(lead_preview.word_count, 300) self.assertEqual(lead_preview.page_count, 4) - self.assertEqual(LeadPreviewImage.objects.filter(lead=self.lead).count(), 2) + self.assertEqual(LeadPreviewAttachment.objects.filter(lead=self.lead).count(), 2) index_lead_func.assert_called_once_with(self.lead.id) diff --git a/apps/lead/tests/test_mutations.py b/apps/lead/tests/test_mutations.py index 0a894205d5..ce85db556e 100644 --- a/apps/lead/tests/test_mutations.py +++ b/apps/lead/tests/test_mutations.py @@ -13,7 +13,7 @@ LeadGroupFactory, LeadEMMTriggerFactory, LeadPreviewFactory, - LeadPreviewImageFactory, + LeadPreviewAttachmentFactory, ) @@ -517,7 +517,7 @@ def test_lead_copy_mutation(self): # Generating Foreign elements for wa_lead1 wa_lead1_preview = LeadPreviewFactory.create(lead=wa_lead1, text_extract='This is a random text extarct') - wa_lead1_image_preview = LeadPreviewImageFactory.create(lead=wa_lead1, file='test-file-123') + wa_lead1_image_preview = LeadPreviewAttachmentFactory.create(lead=wa_lead1, file='test-file-123') LeadEMMTriggerFactory.create( lead=wa_lead1, emm_keyword='emm1', diff --git a/apps/lead/views.py b/apps/lead/views.py index b922e32770..d56e8428b8 100644 --- a/apps/lead/views.py +++ b/apps/lead/views.py @@ -48,7 +48,7 @@ Lead, EMMEntity, LeadEMMTrigger, - LeadPreviewImage, + LeadPreviewAttachment, ) from .serializers import ( raise_or_return_existing_lead, @@ -812,7 +812,7 @@ def _get_clone_ready(obj, lead): lead.authors.set(authors) # Clone Many to one Fields - LeadPreviewImage.objects.bulk_create([ + LeadPreviewAttachment.objects.bulk_create([ _get_clone_ready(image, lead) for image in preview_images ]) LeadEMMTrigger.objects.bulk_create([ diff --git a/schema.graphql b/schema.graphql index a48ab2fb2c..b437efa53c 100644 --- a/schema.graphql +++ b/schema.graphql @@ -4556,6 +4556,7 @@ type LeadDetailType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType + leadPreviewAttachments: [LeadPreviewAttachmentsType!] source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] @@ -4687,6 +4688,19 @@ enum LeadOrderingEnum { DESC_ENTRIES_COUNT } +enum LeadPreviewAttachmentType { + XLSX + IMAGE +} + +type LeadPreviewAttachmentsType { + order: Int! + pageNumber: Int! + type: LeadPreviewAttachmentType! + file: FileFieldType + filePreview: FileFieldType +} + type LeadPreviewType { textExtract: String! thumbnail: String @@ -4747,6 +4761,7 @@ type LeadType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType + leadPreviewAttachments: [LeadPreviewAttachmentsType!] source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] From bcb2fcfdb38795496d5c9e3ff979d742f8d18128 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 7 Jun 2024 13:56:06 +0545 Subject: [PATCH 02/17] PR fixes --- apps/deepl_integration/handlers.py | 6 ++--- apps/deepl_integration/serializers.py | 6 ++--- apps/lead/enums.py | 6 ++++- .../0051_alter_leadpreviewattachment_type.py | 18 +++++++++++++ .../0052_alter_leadpreviewattachment_type.py | 18 +++++++++++++ apps/lead/models.py | 8 +++--- apps/lead/schema.py | 11 ++++---- schema.graphql | 25 ++++++++++++------- 8 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 apps/lead/migrations/0051_alter_leadpreviewattachment_type.py create mode 100644 apps/lead/migrations/0052_alter_leadpreviewattachment_type.py diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 5cb5ee4223..e035044099 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -656,8 +656,7 @@ def save_data( # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? attachement_base_path = f'{lead.pk}' - images = [dict(item) for item in images_uri] - for image_uri in images: + for image_uri in images_uri: for image in image_uri['images']: lead_attachement = LeadPreviewAttachment(lead=lead) image_obj = RequestHelper(url=image, ignore_error=True).get_file() @@ -672,8 +671,7 @@ def save_data( lead_attachement.save() - table_path = [dict(item) for item in table_uri] - for table in table_path: + for table in table_uri: lead_attachement = LeadPreviewAttachment(lead=lead) table_img = RequestHelper(url=table['image_link'], ignore_error=True).get_file() table_attahcment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 203701e542..afaddc9315 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -1,4 +1,4 @@ -from typing import Type +from typing import Type, List, Dict import logging from rest_framework import serializers @@ -67,7 +67,7 @@ class ImagePathSerializer(serializers.Serializer): page_number = serializers.IntegerField(required=True) images = serializers.ListField( child=serializers.CharField(allow_blank=True), - default=[] + default=[], ) @@ -111,7 +111,7 @@ def validate(self, data): raise serializers.ValidationError(errors) return data - def create(self, data): + def create(self, data: List[Dict]): success = data['status'] == self.Status.SUCCESS lead = data['object'] # Added from validate if success: diff --git a/apps/lead/enums.py b/apps/lead/enums.py index 39516159ed..07e9f5d448 100644 --- a/apps/lead/enums.py +++ b/apps/lead/enums.py @@ -5,7 +5,7 @@ get_enum_name_from_django_field, ) -from .models import Lead +from .models import Lead, LeadPreviewAttachment LeadConfidentialityEnum = convert_enum_to_graphene_enum(Lead.Confidentiality, name='LeadConfidentialityEnum') LeadStatusEnum = convert_enum_to_graphene_enum(Lead.Status, name='LeadStatusEnum') @@ -15,6 +15,9 @@ LeadAutoEntryExtractionTypeEnum = convert_enum_to_graphene_enum( Lead.AutoExtractionStatus, name='LeadAutoEntryExtractionTypeEnum' ) +LeadPreviewAttachmentTypeEnum = convert_enum_to_graphene_enum( + LeadPreviewAttachment.AttachementFileType, name='LeadPreviewAttachmentTypeEnum' +) enum_map = { @@ -26,6 +29,7 @@ (Lead.source_type, LeadSourceTypeEnum), (Lead.extraction_status, LeadExtractionStatusEnum), (Lead.auto_entry_extraction_status, LeadAutoEntryExtractionTypeEnum), + (LeadPreviewAttachment.type, LeadPreviewAttachmentTypeEnum), ) } diff --git a/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py new file mode 100644 index 0000000000..bfa87741c4 --- /dev/null +++ b/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-07 06:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0050_auto_20240606_0608'), + ] + + operations = [ + migrations.AlterField( + model_name='leadpreviewattachment', + name='type', + field=models.SmallIntegerField(choices=[('1', 'XLSX'), ('2', 'Image')]), + ), + ] diff --git a/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py new file mode 100644 index 0000000000..0185485a73 --- /dev/null +++ b/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-07 08:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0051_alter_leadpreviewattachment_type'), + ] + + operations = [ + migrations.AlterField( + model_name='leadpreviewattachment', + name='type', + field=models.PositiveSmallIntegerField(choices=[('1', 'XLSX'), ('2', 'Image')], default='1'), + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index 98403aa87f..f941e2b4f4 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -377,17 +377,17 @@ class LeadPreviewAttachment(models.Model): NOTE: File can be only used by gallery (when attached to a entry) """ class AttachementFileType(models.TextChoices): - XLSX = 'XLSX', 'XLSX' - IMAGE = 'image', 'Image' + XLSX = 1, 'XLSX' + IMAGE = 2, 'Image' lead = models.ForeignKey( Lead, related_name='images', on_delete=models.CASCADE, ) order = models.IntegerField(default=0) page_number = models.IntegerField(default=0) - type = models.CharField( - max_length=20, + type = models.PositiveSmallIntegerField( choices=AttachementFileType.choices, + default=AttachementFileType.XLSX ) file = models.FileField(upload_to='lead-preview/attachments/') file_preview = models.FileField(upload_to='lead-preview/attachments-preview/') diff --git a/apps/lead/schema.py b/apps/lead/schema.py index 519aee5b13..24c3107373 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -37,6 +37,7 @@ ) from .enums import ( LeadConfidentialityEnum, + LeadPreviewAttachmentTypeEnum, LeadStatusEnum, LeadPriorityEnum, LeadSourceTypeEnum, @@ -217,18 +218,16 @@ class Meta: ) -class LeadPreviewAttachmentsType(DjangoObjectType): +class LeadPreviewAttachmentType(DjangoObjectType): file = graphene.Field(FileFieldType) file_preview = graphene.Field(FileFieldType) + type = graphene.Field(LeadPreviewAttachmentTypeEnum) class Meta: model = LeadPreviewAttachment only_fields = ( - 'type', 'page_number', 'order', - 'file', - 'file_preview', ) @@ -363,7 +362,7 @@ class Meta: extraction_status = graphene.Field(LeadExtractionStatusEnum) lead_preview = graphene.Field(LeadPreviewType) - lead_preview_attachments = graphene.List(graphene.NonNull(LeadPreviewAttachmentsType)) + lead_preview_attachment = graphene.List(graphene.NonNull(LeadPreviewAttachmentType), required=True) source = graphene.Field(OrganizationType) authors = DjangoListField(OrganizationType) assignee = graphene.Field(UserType) @@ -429,7 +428,7 @@ def resolve_attachment(root, info, **kwargs): if root.attachment_id: return info.context.dl.deep_gallery.file.load(root.attachment_id) - def resolve_lead_preview_attachments(root, info, **kwargs): + def resolve_lead_preview_attachment(root, info, **kwargs): return info.context.dl.lead.lead_preview_attachment.load(root.pk) diff --git a/schema.graphql b/schema.graphql index b437efa53c..edf262a359 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1576,6 +1576,7 @@ type AppEnumCollection { LeadSourceType: [AppEnumCollectionLeadSourceType!] LeadExtractionStatus: [AppEnumCollectionLeadExtractionStatus!] LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] + LeadPreviewAttachmentType: [AppEnumCollectionLeadPreviewAttachmentType!] EntryEntryType: [AppEnumCollectionEntryEntryType!] ExportFormat: [AppEnumCollectionExportFormat!] ExportStatus: [AppEnumCollectionExportStatus!] @@ -2024,6 +2025,12 @@ type AppEnumCollectionLeadExtractionStatus { description: String } +type AppEnumCollectionLeadPreviewAttachmentType { + enum: LeadPreviewAttachmentTypeEnum! + label: String! + description: String +} + type AppEnumCollectionLeadPriority { enum: LeadPriorityEnum! label: String! @@ -4556,7 +4563,7 @@ type LeadDetailType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType - leadPreviewAttachments: [LeadPreviewAttachmentsType!] + leadPreviewAttachment: [LeadPreviewAttachmentType!]! source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] @@ -4688,17 +4695,17 @@ enum LeadOrderingEnum { DESC_ENTRIES_COUNT } -enum LeadPreviewAttachmentType { - XLSX - IMAGE -} - -type LeadPreviewAttachmentsType { +type LeadPreviewAttachmentType { order: Int! pageNumber: Int! - type: LeadPreviewAttachmentType! file: FileFieldType filePreview: FileFieldType + type: LeadPreviewAttachmentTypeEnum +} + +enum LeadPreviewAttachmentTypeEnum { + XLSX + IMAGE } type LeadPreviewType { @@ -4761,7 +4768,7 @@ type LeadType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType - leadPreviewAttachments: [LeadPreviewAttachmentsType!] + leadPreviewAttachment: [LeadPreviewAttachmentType!]! source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] From 88fffeaf2bcb10e735a3bf406636fc412a823e34 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Sun, 9 Jun 2024 16:30:32 +0545 Subject: [PATCH 03/17] Add mutation of EntryAttachment --- apps/entry/admin.py | 3 ++ .../migrations/0038_auto_20240609_0904.py | 29 +++++++++++++++ .../migrations/0039_auto_20240609_0933.py | 23 ++++++++++++ ...0_alter_entryattachment_entry_file_type.py | 18 ++++++++++ apps/entry/models.py | 14 ++++++++ apps/entry/schema.py | 14 ++++++-- apps/entry/serializers.py | 26 +++++++++----- apps/lead/filter_set.py | 11 +++++- .../0053_alter_leadpreviewattachment_type.py | 18 ++++++++++ apps/lead/models.py | 2 +- apps/lead/schema.py | 24 ++++++++++--- apps/lead/tests/test_apis.py | 16 +++++++-- schema.graphql | 35 +++++++++++++++---- 13 files changed, 206 insertions(+), 27 deletions(-) create mode 100644 apps/entry/migrations/0038_auto_20240609_0904.py create mode 100644 apps/entry/migrations/0039_auto_20240609_0933.py create mode 100644 apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py create mode 100644 apps/lead/migrations/0053_alter_leadpreviewattachment_type.py diff --git a/apps/entry/admin.py b/apps/entry/admin.py index 13512843aa..c481e39a57 100644 --- a/apps/entry/admin.py +++ b/apps/entry/admin.py @@ -9,6 +9,7 @@ from entry.models import ( Entry, Attribute, + EntryAttachment, FilterData, ExportData, EntryComment, @@ -86,4 +87,6 @@ class ProjectEntryLabelAdmin(VersionAdmin): list_display = ('__str__', 'color') +admin.site.register(EntryAttachment) + reversion.register(LeadEntryGroup) diff --git a/apps/entry/migrations/0038_auto_20240609_0904.py b/apps/entry/migrations/0038_auto_20240609_0904.py new file mode 100644 index 0000000000..14029c166b --- /dev/null +++ b/apps/entry/migrations/0038_auto_20240609_0904.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.25 on 2024-06-09 09:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('entry', '0037_merge_20220401_0527'), + ] + + operations = [ + migrations.AlterField( + model_name='entry', + name='entry_type', + field=models.CharField(choices=[('excerpt', 'Excerpt'), ('image', 'Image'), ('attachment', 'Attachment'), ('dataSeries', 'Data Series')], default='excerpt', max_length=10), + ), + migrations.CreateModel( + name='EntryAttachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_file_type', models.PositiveSmallIntegerField(choices=[('1', 'XLSX')], default='1')), + ('file', models.FileField(upload_to='entry/attachment/')), + ('file_preview', models.FileField(upload_to='entry/attachment-preview')), + ('entry', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='entry.entry')), + ], + ), + ] diff --git a/apps/entry/migrations/0039_auto_20240609_0933.py b/apps/entry/migrations/0039_auto_20240609_0933.py new file mode 100644 index 0000000000..53a8985094 --- /dev/null +++ b/apps/entry/migrations/0039_auto_20240609_0933.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-06-09 09:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('entry', '0038_auto_20240609_0904'), + ] + + operations = [ + migrations.RemoveField( + model_name='entryattachment', + name='entry', + ), + migrations.AddField( + model_name='entry', + name='entry_attachment', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='entry.entryattachment'), + ), + ] diff --git a/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py b/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py new file mode 100644 index 0000000000..ad7aeccb31 --- /dev/null +++ b/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-09 10:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('entry', '0039_auto_20240609_0933'), + ] + + operations = [ + migrations.AlterField( + model_name='entryattachment', + name='entry_file_type', + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX')], default=1), + ), + ] diff --git a/apps/entry/models.py b/apps/entry/models.py index 1771b2798a..c7b98da209 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -21,6 +21,18 @@ from assisted_tagging.models import DraftEntry +class EntryAttachment(models.Model): + class EntryFileType(models.IntegerChoices): + XLSX = 1, 'XLSX' + + entry_file_type = models.PositiveSmallIntegerField( + choices=EntryFileType.choices, + default=EntryFileType.XLSX + ) + file = models.FileField(upload_to='entry/attachment/') + file_preview = models.FileField(upload_to='entry/attachment-preview') + + class Entry(UserResource, ProjectEntityMixin): """ Entry belonging to a lead @@ -32,6 +44,7 @@ class Entry(UserResource, ProjectEntityMixin): class TagType(models.TextChoices): EXCERPT = 'excerpt', 'Excerpt', IMAGE = 'image', 'Image', + ATTACHMENT = 'attachment', 'Attachment', DATA_SERIES = 'dataSeries', 'Data Series' # NOTE: data saved as tabular_field id lead = models.ForeignKey(Lead, on_delete=models.CASCADE) @@ -47,6 +60,7 @@ class TagType(models.TextChoices): image = models.ForeignKey(File, on_delete=models.SET_NULL, null=True, blank=True) image_raw = models.TextField(blank=True) tabular_field = models.ForeignKey('tabular.Field', on_delete=models.CASCADE, null=True, blank=True) + entry_attachment = models.OneToOneField(EntryAttachment, on_delete=models.CASCADE, null=True, blank=True) dropped_excerpt = models.TextField(blank=True) # NOTE: Original Exceprt. Modified version is stored in excerpt excerpt_modified = models.BooleanField(default=False) diff --git a/apps/entry/schema.py b/apps/entry/schema.py index f9a369aa9b..41b9f309a5 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -6,7 +6,7 @@ from utils.common import has_prefetched from utils.graphene.enums import EnumDescription -from utils.graphene.types import CustomDjangoListObjectType, ClientIdMixin +from utils.graphene.types import CustomDjangoListObjectType, ClientIdMixin, FileFieldType from utils.graphene.fields import DjangoPaginatedListObjectField, DjangoListField from user_resource.schema import UserResourceMixin from deep.permissions import ProjectPermissions as PP @@ -20,6 +20,7 @@ from .models import ( Entry, Attribute, + EntryAttachment, ) from .enums import EntryTagTypeEnum from .filter_set import EntryGQFilterSet @@ -84,6 +85,14 @@ def resolve_geo_selected_options(root, info, **_): ) +class EntryAttachmentType(DjangoObjectType): + file = graphene.Field(FileFieldType) + file_preview = graphene.Field(FileFieldType) + + class Meta: + model = EntryAttachment + + class EntryType(UserResourceMixin, ClientIdMixin, DjangoObjectType): class Meta: model = Entry @@ -92,7 +101,7 @@ class Meta: 'lead', 'project', 'analysis_framework', 'information_date', 'order', 'excerpt', 'dropped_excerpt', 'image', 'tabular_field', 'highlight_hidden', 'controlled', 'controlled_changed_by', - 'client_id', + 'client_id', 'entry_attachment' ) entry_type = graphene.Field(EntryTagTypeEnum, required=True) @@ -103,6 +112,7 @@ class Meta: verified_by_count = graphene.Int(required=True) review_comments_count = graphene.Int(required=True) draft_entry = graphene.ID(source="draft_entry_id") + entry_attachment = graphene.Field(EntryAttachmentType, required=False) # project_labels TODO: # tabular_field TODO: diff --git a/apps/entry/serializers.py b/apps/entry/serializers.py index 1340b53897..99d53721da 100644 --- a/apps/entry/serializers.py +++ b/apps/entry/serializers.py @@ -29,6 +29,7 @@ from .models import ( Attribute, Entry, + EntryAttachment, EntryComment, EntryCommentText, ExportData, @@ -591,12 +592,12 @@ class EntryGqSerializer(ProjectPropertySerializerMixin, TempClientIdMixin, UserR ' This will be changed into gallery image and supplied back in image field.' ) ) - lead_image = serializers.PrimaryKeyRelatedField( + lead_attachment = serializers.PrimaryKeyRelatedField( required=False, write_only=True, queryset=LeadPreviewAttachment.objects.all(), help_text=( - 'This is used to add images from Lead Preview Images.' + 'This is used to add attachment from Lead Preview Attachment.' ' This will be changed into gallery image and supplied back in image field.' ) ) @@ -611,7 +612,7 @@ class Meta: 'entry_type', 'image', 'image_raw', - 'lead_image', + 'lead_attachment', 'tabular_field', 'excerpt', 'dropped_excerpt', @@ -643,7 +644,7 @@ def validate(self, data): request = self.context['request'] image = data.get('image') image_raw = data.pop('image_raw', None) - lead_image = data.pop('lead_image', None) + lead_attachment = data.pop('lead_attachment', None) # ---------------- Lead lead = data['lead'] @@ -685,12 +686,21 @@ def validate(self, data): 'image': f'You don\'t have permission to attach image: {image}', }) # If lead image is provided make sure lead are same - elif lead_image: - if lead_image.lead != lead: + elif lead_attachment: + if lead_attachment.lead != lead: raise serializers.ValidationError({ - 'lead_image': f'You don\'t have permission to attach lead image: {lead_image}', + 'lead_attachment': f'You don\'t have permission to attach lead attachment: {lead_attachment}', }) - data['image'] = lead_image.clone_as_deep_file(request.user) + + if lead_attachment.type == LeadPreviewAttachment.AttachementFileType.XLSX: + data['entry_attachment'] = EntryAttachment.objects.create( + file=lead_attachment.file, + file_preview=lead_attachment.file_preview + ) + data['entry_type'] = Entry.TagType.ATTACHMENT + else: + data['image'] = lead_attachment.clone_as_deep_file(request.user) + data['entry_type'] = Entry.TagType.IMAGE elif image_raw: generated_image = base64_to_deep_image(image_raw, lead, request.user) if isinstance(generated_image, File): diff --git a/apps/lead/filter_set.py b/apps/lead/filter_set.py index 28eeac509b..49bb47b94d 100644 --- a/apps/lead/filter_set.py +++ b/apps/lead/filter_set.py @@ -22,7 +22,7 @@ from entry.filter_set import EntryGQFilterSet, EntriesFilterDataInputType, EntriesFilterDataType from user_resource.filters import UserResourceGqlFilterSet -from .models import Lead, LeadGroup, LeadDuplicates +from .models import Lead, LeadGroup, LeadDuplicates, LeadPreviewAttachment from .enums import ( LeadConfidentialityEnum, LeadStatusEnum, @@ -30,6 +30,7 @@ LeadSourceTypeEnum, LeadOrderingEnum, LeadExtractionStatusEnum, + LeadPreviewAttachmentTypeEnum, ) @@ -552,6 +553,14 @@ def filter_title(self, qs, name, value): return qs.filter(title__icontains=value).distinct() +class LeadPreviewAttachmentGQFilterSet(UserResourceGqlFilterSet): + type = MultipleInputFilter(LeadPreviewAttachmentTypeEnum, field_name='type') + + class Meta: + model = LeadPreviewAttachment + fields = ['lead', 'page_number'] + + LeadsFilterDataType, LeadsFilterDataInputType = generate_type_for_filter_set( LeadGQFilterSet, 'lead.schema.LeadListType', diff --git a/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py new file mode 100644 index 0000000000..a47c92bd98 --- /dev/null +++ b/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-09 10:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0052_alter_leadpreviewattachment_type'), + ] + + operations = [ + migrations.AlterField( + model_name='leadpreviewattachment', + name='type', + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1), + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index f941e2b4f4..21f1221ab6 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -376,7 +376,7 @@ class LeadPreviewAttachment(models.Model): """ NOTE: File can be only used by gallery (when attached to a entry) """ - class AttachementFileType(models.TextChoices): + class AttachementFileType(models.IntegerChoices): XLSX = 1, 'XLSX' IMAGE = 2, 'Image' diff --git a/apps/lead/schema.py b/apps/lead/schema.py index 24c3107373..c0a83ec5f4 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -47,6 +47,7 @@ from .filter_set import ( LeadGQFilterSet, LeadGroupGQFilterSet, + LeadPreviewAttachmentGQFilterSet, ) @@ -219,15 +220,16 @@ class Meta: class LeadPreviewAttachmentType(DjangoObjectType): - file = graphene.Field(FileFieldType) - file_preview = graphene.Field(FileFieldType) - type = graphene.Field(LeadPreviewAttachmentTypeEnum) + file = graphene.Field(FileFieldType, required=True) + file_preview = graphene.Field(FileFieldType, required=True) + type = graphene.Field(LeadPreviewAttachmentTypeEnum, required=True) class Meta: model = LeadPreviewAttachment only_fields = ( - 'page_number', + 'id', 'order', + 'page_number', ) @@ -362,7 +364,6 @@ class Meta: extraction_status = graphene.Field(LeadExtractionStatusEnum) lead_preview = graphene.Field(LeadPreviewType) - lead_preview_attachment = graphene.List(graphene.NonNull(LeadPreviewAttachmentType), required=True) source = graphene.Field(OrganizationType) authors = DjangoListField(OrganizationType) assignee = graphene.Field(UserType) @@ -472,6 +473,12 @@ class Meta: filterset_class = LeadGQFilterSet +class LeadPreviewAttachmentListType(CustomDjangoListObjectType): + class Meta: + model = LeadPreviewAttachment + filterset_class = LeadPreviewAttachmentGQFilterSet + + class Query: lead = DjangoObjectField(LeadDetailType) leads = DjangoPaginatedListObjectField( @@ -499,6 +506,13 @@ class Query: page_size_query_param='pageSize' ) ) + lead_preview_attachments = DjangoPaginatedListObjectField( + LeadPreviewAttachmentListType, + pagination=PageGraphqlPagination( + page_size_query_param='pageSize', + ) + ) + # TODO: Add Pagination emm_keywords = graphene.List(graphene.NonNull(EmmKeyWordType)) emm_risk_factors = graphene.List(graphene.NonNull(EmmKeyRiskFactorType)) diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 5a813e2537..6a98639dbb 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1802,11 +1802,18 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f 'page_number': 1, 'images': [ 'http://random.com/image1.jpeg', - 'http://random.com/image1.jpeg' + 'http://random.com/image2.jpeg' ], } ], - 'tables_path': [], + 'tables_path': [ + { + "page_number": 1, + "order": 0, + "image_link": "http://random.com/timetable.png", + "content_link": "http://random.com/table_timetable.xlsx" + } + ], 'text_path': 'http://random.com/extracted_file.txt', 'url': 'http://random.com/pdf_file.pdf', 'total_words_count': 300, @@ -1835,7 +1842,10 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f self.assertEqual(lead_preview.text_extract, 'Extracted text') self.assertEqual(lead_preview.word_count, 300) self.assertEqual(lead_preview.page_count, 4) - self.assertEqual(LeadPreviewAttachment.objects.filter(lead=self.lead).count(), 2) + self.assertEqual(LeadPreviewAttachment.objects.filter(lead=self.lead).count(), 3) + self.assertEqual(LeadPreviewAttachment.objects.filter( + lead=self.lead, type=LeadPreviewAttachment.AttachementFileType.IMAGE).count(), 2 + ) index_lead_func.assert_called_once_with(self.lead.id) diff --git a/schema.graphql b/schema.graphql index edf262a359..ac549102b9 100644 --- a/schema.graphql +++ b/schema.graphql @@ -3270,7 +3270,7 @@ input BulkEntryInputType { entryType: EntryTagTypeEnum image: ID imageRaw: String - leadImage: ID + leadAttachment: ID tabularField: ID excerpt: String droppedExcerpt: String @@ -3927,6 +3927,18 @@ type EntriesFilterDataType { filterableData: [EntryFilterDataType!] } +enum EntryAttachmentEntryFileType { + A_1 +} + +type EntryAttachmentType { + id: ID! + entryFileType: EntryAttachmentEntryFileType! + file: FileFieldType + filePreview: FileFieldType + entry: EntryType +} + input EntryFilterDataInputType { filterKey: ID! value: String @@ -3970,7 +3982,7 @@ input EntryInputType { entryType: EntryTagTypeEnum image: ID imageRaw: String - leadImage: ID + leadAttachment: ID tabularField: ID excerpt: String droppedExcerpt: String @@ -4052,6 +4064,7 @@ enum EntryReviewCommentTypeEnum { enum EntryTagTypeEnum { EXCERPT IMAGE + ATTACHMENT DATA_SERIES } @@ -4065,6 +4078,7 @@ type EntryType { informationDate: Date excerpt: String! image: GalleryFileType + entryAttachment: EntryAttachmentType droppedExcerpt: String! highlightHidden: Boolean! controlled: Boolean @@ -4563,7 +4577,6 @@ type LeadDetailType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType - leadPreviewAttachment: [LeadPreviewAttachmentType!]! source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] @@ -4695,12 +4708,20 @@ enum LeadOrderingEnum { DESC_ENTRIES_COUNT } +type LeadPreviewAttachmentListType { + results: [LeadPreviewAttachmentType!] + totalCount: Int + page: Int + pageSize: Int +} + type LeadPreviewAttachmentType { + id: ID! order: Int! pageNumber: Int! - file: FileFieldType - filePreview: FileFieldType - type: LeadPreviewAttachmentTypeEnum + file: FileFieldType! + filePreview: FileFieldType! + type: LeadPreviewAttachmentTypeEnum! } enum LeadPreviewAttachmentTypeEnum { @@ -4768,7 +4789,6 @@ type LeadType { statusDisplay: EnumDescription! extractionStatus: LeadExtractionStatusEnum leadPreview: LeadPreviewType - leadPreviewAttachment: [LeadPreviewAttachmentType!]! source: OrganizationType authors: [OrganizationType!] emmEntities: [EmmEntityType!] @@ -5268,6 +5288,7 @@ type ProjectDetailType { leadGroups(createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, page: Int = 1, ordering: String, pageSize: Int): LeadGroupListType emmEntities(name: String, page: Int = 1, ordering: String, pageSize: Int): EmmEntityListType leadEmmTriggers(lead: ID, emmKeyword: String, emmRiskFactor: String, count: Int, page: Int = 1, ordering: String, pageSize: Int): LeadEmmTriggerListType + leadPreviewAttachments(lead: ID, pageNumber: Int, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], type: [LeadPreviewAttachmentTypeEnum!], page: Int = 1, ordering: String, pageSize: Int): LeadPreviewAttachmentListType emmKeywords: [EmmKeyWordType!] emmRiskFactors: [EmmKeyRiskFactorType!] userSavedLeadFilter: UserSavedLeadFilterType From cf0b2730db3606f853e4840f2c45730d7a44eabc Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 11 Jun 2024 14:55:40 +0545 Subject: [PATCH 04/17] Add relation of leadattachment in entryattachment model Update testcases --- apps/ary/export/affected_groups_info.py | 2 +- apps/deepl_integration/handlers.py | 52 +++-- apps/deepl_integration/serializers.py | 10 +- apps/entry/admin.py | 7 +- apps/entry/enums.py | 7 +- apps/entry/factories.py | 23 +++ .../migrations/0041_auto_20240611_0810.py | 25 +++ apps/entry/models.py | 13 +- apps/entry/schema.py | 13 +- apps/entry/serializers.py | 20 +- .../tests/snapshots/snap_test_mutations.py | 194 ------------------ apps/entry/tests/test_mutations.py | 23 ++- apps/entry/tests/test_schemas.py | 29 ++- apps/entry/utils.py | 18 +- apps/export/entries/excel_exporter.py | 2 +- apps/geo/models.py | 2 +- apps/lead/enums.py | 2 +- apps/lead/filter_set.py | 13 +- apps/lead/models.py | 8 +- apps/lead/tests/test_apis.py | 2 +- apps/lead/tests/test_mutations.py | 4 +- schema.graphql | 28 ++- 22 files changed, 231 insertions(+), 266 deletions(-) create mode 100644 apps/entry/migrations/0041_auto_20240611_0810.py diff --git a/apps/ary/export/affected_groups_info.py b/apps/ary/export/affected_groups_info.py index e9ddca71a0..1c240bb918 100644 --- a/apps/ary/export/affected_groups_info.py +++ b/apps/ary/export/affected_groups_info.py @@ -9,7 +9,7 @@ def get_affected_groups_info(assessment): affected_group_type_dict = {choice.value: choice.label for choice in AssessmentRegistry.AffectedGroupType} affected_groups = [affected_group_type_dict.get(group) for group in assessment.affected_groups if group] max_level = max([len(v.split('/')) for k, v in AssessmentRegistry.AffectedGroupType.choices]) - levels = [f'Level {i+1}' for i in range(max_level)] + levels = [f'Level {i + 1}' for i in range(max_level)] affected_grp_list = [] for group in affected_groups: group = group.split("/") diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index e035044099..008ab39d64 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -655,38 +655,52 @@ def save_data( # Save extracted images as LeadPreviewAttachment instances # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? - attachement_base_path = f'{lead.pk}' + attachment_base_path = f'{lead.pk}' for image_uri in images_uri: for image in image_uri['images']: - lead_attachement = LeadPreviewAttachment(lead=lead) + lead_attachment = LeadPreviewAttachment(lead=lead) image_obj = RequestHelper(url=image, ignore_error=True).get_file() if image_obj: - lead_attachement.file.save( - os.path.join(attachement_base_path, os.path.basename(urlparse(image).path)), + lead_attachment.file.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(image).path + ) + ), image_obj ) - lead_attachement.page_number = image_uri['page_number'] - lead_attachement.type = LeadPreviewAttachment.AttachementFileType.IMAGE - lead_attachement.file_preview = lead_attachement.file - - lead_attachement.save() + lead_attachment.page_number = image_uri['page_number'] + lead_attachment.type = LeadPreviewAttachment.AttachmentFileType.IMAGE + lead_attachment.file_preview = lead_attachment.file + lead_attachment.save() for table in table_uri: - lead_attachement = LeadPreviewAttachment(lead=lead) + lead_attachment = LeadPreviewAttachment(lead=lead) table_img = RequestHelper(url=table['image_link'], ignore_error=True).get_file() - table_attahcment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() + table_attachment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() if table_img: - lead_attachement.file_preview.save( - os.path.join(attachement_base_path, os.path.basename(urlparse(table['image_link']).path)), + lead_attachment.file_preview.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(table['image_link']).path + ) + ), table_img ) - lead_attachement.page_number = table['page_number'] - lead_attachement.type = LeadPreviewAttachment.AttachementFileType.XLSX - lead_attachement.file.save( - os.path.join(attachement_base_path, os.path.basename(urlparse(table['content_link']).path)), - table_attahcment + lead_attachment.page_number = table['page_number'] + lead_attachment.type = LeadPreviewAttachment.AttachmentFileType.XLSX + lead_attachment.file.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(table['content_link']).path + ) + ), + table_attachment ) - lead_attachement.save() + lead_attachment.save() lead.update_extraction_status(Lead.ExtractionStatus.SUCCESS) return lead diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index afaddc9315..ffe69995fe 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -85,8 +85,14 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ url = serializers.CharField(required=False) # Data fields - images_path = serializers.ListSerializer(child=ImagePathSerializer(required=False)) - tables_path = serializers.ListSerializer(child=TablePathSerializer(required=False)) + images_path = serializers.ListSerializer( + child=ImagePathSerializer(required=True), + required=False + ) + tables_path = serializers.ListSerializer( + child=TablePathSerializer(required=True), + required=False + ) text_path = serializers.CharField(required=False, allow_null=True) total_words_count = serializers.IntegerField(required=False, default=0, allow_null=True) total_pages = serializers.IntegerField(required=False, default=0, allow_null=True) diff --git a/apps/entry/admin.py b/apps/entry/admin.py index c481e39a57..a1c92391e9 100644 --- a/apps/entry/admin.py +++ b/apps/entry/admin.py @@ -62,7 +62,7 @@ class EntryAdmin(VersionAdmin): ) autocomplete_fields = ( 'lead', 'project', 'created_by', 'modified_by', 'analysis_framework', 'tabular_field', - 'image', 'controlled_changed_by', 'verified_by', + 'image', 'controlled_changed_by', 'verified_by', 'entry_attachment', ) ordering = ('project', 'created_by', 'created_at') @@ -87,6 +87,9 @@ class ProjectEntryLabelAdmin(VersionAdmin): list_display = ('__str__', 'color') -admin.site.register(EntryAttachment) +@admin.register(EntryAttachment) +class EntryAttachmentAdmin(VersionAdmin): + search_fields = ['entry_file_type',] + reversion.register(LeadEntryGroup) diff --git a/apps/entry/enums.py b/apps/entry/enums.py index e98f268a8d..66d4e24653 100644 --- a/apps/entry/enums.py +++ b/apps/entry/enums.py @@ -3,13 +3,18 @@ get_enum_name_from_django_field, ) -from .models import Entry +from .models import Entry, EntryAttachment EntryTagTypeEnum = convert_enum_to_graphene_enum(Entry.TagType, name='EntryTagTypeEnum') +EntryAttachmentTypeEnum = convert_enum_to_graphene_enum( + EntryAttachment.EntryFileType, + name='EntryFileType' +) enum_map = { get_enum_name_from_django_field(field): enum for field, enum in ( (Entry.entry_type, EntryTagTypeEnum), + (EntryAttachment.entry_file_type, EntryAttachmentTypeEnum), ) } diff --git a/apps/entry/factories.py b/apps/entry/factories.py index 6f49b618dc..42b4c0c39a 100644 --- a/apps/entry/factories.py +++ b/apps/entry/factories.py @@ -3,11 +3,14 @@ from factory.django import DjangoModelFactory from gallery.factories import FileFactory +from django.core.files.base import ContentFile + from .models import ( Entry, Attribute, EntryComment, + EntryAttachment ) @@ -48,3 +51,23 @@ class Meta: class EntryCommentFactory(DjangoModelFactory): class Meta: model = EntryComment + + +class EntryAttachmentFactory(DjangoModelFactory): + class Meta: + model = EntryAttachment + + file = factory.LazyAttribute( + lambda _: ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), 'example.jpg' + ) + ) + file_preview = factory.LazyAttribute( + lambda _: ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), 'example.jpg' + ) + ) diff --git a/apps/entry/migrations/0041_auto_20240611_0810.py b/apps/entry/migrations/0041_auto_20240611_0810.py new file mode 100644 index 0000000000..b57e395c17 --- /dev/null +++ b/apps/entry/migrations/0041_auto_20240611_0810.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.25 on 2024-06-11 08:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0053_alter_leadpreviewattachment_type'), + ('entry', '0040_alter_entryattachment_entry_file_type'), + ] + + operations = [ + migrations.AddField( + model_name='entryattachment', + name='lead_attachment', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='lead.leadpreviewattachment'), + ), + migrations.AlterField( + model_name='entryattachment', + name='entry_file_type', + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1), + ), + ] diff --git a/apps/entry/models.py b/apps/entry/models.py index c7b98da209..4324e8af86 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -10,7 +10,7 @@ from gallery.models import File from user.models import User from user_resource.models import UserResource -from lead.models import Lead +from lead.models import Lead, LeadPreviewAttachment from notification.models import Assignment from analysis_framework.models import ( AnalysisFramework, @@ -21,17 +21,22 @@ from assisted_tagging.models import DraftEntry -class EntryAttachment(models.Model): +class EntryAttachment(models.Model): # The entry will make reference to it as entry_attachment. class EntryFileType(models.IntegerChoices): - XLSX = 1, 'XLSX' + XLSX = 1, 'XLSX', + IMAGE = 2, 'Image', + lead_attachment = models.ForeignKey(LeadPreviewAttachment, on_delete=models.SET_NULL, null=True) entry_file_type = models.PositiveSmallIntegerField( choices=EntryFileType.choices, - default=EntryFileType.XLSX + default=EntryFileType.XLSX, ) file = models.FileField(upload_to='entry/attachment/') file_preview = models.FileField(upload_to='entry/attachment-preview') + def __str__(self): + return f'{self.file}' + class Entry(UserResource, ProjectEntityMixin): """ diff --git a/apps/entry/schema.py b/apps/entry/schema.py index 41b9f309a5..b11e995194 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -22,7 +22,7 @@ Attribute, EntryAttachment, ) -from .enums import EntryTagTypeEnum +from .enums import EntryAttachmentTypeEnum, EntryTagTypeEnum from .filter_set import EntryGQFilterSet @@ -86,11 +86,16 @@ def resolve_geo_selected_options(root, info, **_): class EntryAttachmentType(DjangoObjectType): - file = graphene.Field(FileFieldType) - file_preview = graphene.Field(FileFieldType) + lead_attachment_id = graphene.ID(required=True) + file = graphene.Field(FileFieldType, required=True) + file_preview = graphene.Field(FileFieldType, required=True) + entry_file_type = graphene.Field(EntryAttachmentTypeEnum, required=True) class Meta: model = EntryAttachment + only_fields = ( + 'id', + ) class EntryType(UserResourceMixin, ClientIdMixin, DjangoObjectType): @@ -101,7 +106,7 @@ class Meta: 'lead', 'project', 'analysis_framework', 'information_date', 'order', 'excerpt', 'dropped_excerpt', 'image', 'tabular_field', 'highlight_hidden', 'controlled', 'controlled_changed_by', - 'client_id', 'entry_attachment' + 'client_id', ) entry_type = graphene.Field(EntryTagTypeEnum, required=True) diff --git a/apps/entry/serializers.py b/apps/entry/serializers.py index 99d53721da..19a174c789 100644 --- a/apps/entry/serializers.py +++ b/apps/entry/serializers.py @@ -29,7 +29,6 @@ from .models import ( Attribute, Entry, - EntryAttachment, EntryComment, EntryCommentText, ExportData, @@ -39,7 +38,7 @@ LeadEntryGroup, EntryGroupLabel, ) -from .utils import base64_to_deep_image +from .utils import base64_to_deep_image, leadattachment_to_entryattachment logger = logging.getLogger(__name__) @@ -598,7 +597,6 @@ class EntryGqSerializer(ProjectPropertySerializerMixin, TempClientIdMixin, UserR queryset=LeadPreviewAttachment.objects.all(), help_text=( 'This is used to add attachment from Lead Preview Attachment.' - ' This will be changed into gallery image and supplied back in image field.' ) ) @@ -636,6 +634,11 @@ def validate_lead(self, lead): raise serializers.ValidationError('Changing lead is not allowed') return lead + def validate_lead_attachment(self, lead_attachment): + if int(self.initial_data.get('lead')) != lead_attachment.lead.id: + raise serializers.ValidationError("Don't have access to this lead") + return lead_attachment + def validate(self, data): """ - Lead image is copied to deep gallery files @@ -687,20 +690,13 @@ def validate(self, data): }) # If lead image is provided make sure lead are same elif lead_attachment: + data.pop('excerpt', None) # removing excerpt when lead attachment is send if lead_attachment.lead != lead: raise serializers.ValidationError({ 'lead_attachment': f'You don\'t have permission to attach lead attachment: {lead_attachment}', }) - if lead_attachment.type == LeadPreviewAttachment.AttachementFileType.XLSX: - data['entry_attachment'] = EntryAttachment.objects.create( - file=lead_attachment.file, - file_preview=lead_attachment.file_preview - ) - data['entry_type'] = Entry.TagType.ATTACHMENT - else: - data['image'] = lead_attachment.clone_as_deep_file(request.user) - data['entry_type'] = Entry.TagType.IMAGE + data['entry_attachment'] = leadattachment_to_entryattachment(lead_attachment) elif image_raw: generated_image = base64_to_deep_image(image_raw, lead, request.user) if isinstance(generated_image, File): diff --git a/apps/entry/tests/snapshots/snap_test_mutations.py b/apps/entry/tests/snapshots/snap_test_mutations.py index aeaa3454ea..d560743bbe 100644 --- a/apps/entry/tests/snapshots/snap_test_mutations.py +++ b/apps/entry/tests/snapshots/snap_test_mutations.py @@ -7,129 +7,6 @@ snapshots = Snapshot() -snapshots['TestEntryMutation::test_entry_bulk error'] = { - 'data': { - 'project': { - 'entryBulk': { - 'deletedResult': [ - { - 'attributes': None, - 'clientId': '1', - 'droppedExcerpt': '', - 'entryType': 'IMAGE', - 'excerpt': 'fWdhjOkYRBMeyyMDHqJaRUhRIWrXPvhsBkDaUUqGWlGgOtOGMmjxWkIXHaMuFbhxZtpdpKffUFeWIXiiQEJkqHMBnIWUSmTtzQPx', - 'highlightHidden': False, - 'id': '1', - 'image': { - 'id': '4', - 'title': 'file-3' - }, - 'informationDate': None, - 'order': 1 - } - ], - 'errors': [ - [ - { - 'arrayErrors': None, - 'clientId': 'entry-old-101 (UPDATED)', - 'field': 'image', - 'messages': "You don't have permission to attach image: file-1", - 'objectErrors': None - } - ], - [ - { - 'arrayErrors': None, - 'clientId': 'entry-new-102', - 'field': 'image', - 'messages': "You don't have permission to attach image: file-1", - 'objectErrors': None - } - ] - ], - 'result': [ - None, - None - ] - } - } - } -} - -snapshots['TestEntryMutation::test_entry_bulk success'] = { - 'data': { - 'project': { - 'entryBulk': { - 'deletedResult': [ - ], - 'errors': [ - None, - None - ], - 'result': [ - { - 'attributes': [ - { - 'clientId': 'client-id-old-new-attribute-1', - 'data': { - }, - 'id': '2', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - }, - { - 'clientId': 'client-id-old-attribute-1', - 'data': { - }, - 'id': '1', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - } - ], - 'clientId': 'entry-old-101 (UPDATED)', - 'droppedExcerpt': 'This is a dropped text (UPDATED)', - 'entryType': 'EXCERPT', - 'excerpt': 'This is a text (UPDATED)', - 'highlightHidden': False, - 'id': '2', - 'image': { - 'id': '3', - 'title': 'file-2' - }, - 'informationDate': '2021-01-01', - 'order': 1 - }, - { - 'attributes': [ - { - 'clientId': 'client-id-new-attribute-1', - 'data': { - }, - 'id': '3', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - } - ], - 'clientId': 'entry-new-102', - 'droppedExcerpt': 'This is a dropped text (NEW)', - 'entryType': 'EXCERPT', - 'excerpt': 'This is a text (NEW)', - 'highlightHidden': False, - 'id': '3', - 'image': { - 'id': '3', - 'title': 'file-2' - }, - 'informationDate': '2021-01-01', - 'order': 1 - } - ] - } - } - } -} - snapshots['TestEntryMutation::test_entry_create error'] = { 'data': { 'project': { @@ -200,74 +77,3 @@ } } } - -snapshots['TestEntryMutation::test_entry_update error'] = { - 'data': { - 'project': { - 'entryUpdate': { - 'errors': [ - { - 'arrayErrors': None, - 'clientId': 'entry-101', - 'field': 'image', - 'messages': "You don't have permission to attach image: file-1", - 'objectErrors': None - } - ], - 'ok': False, - 'result': None - } - } - } -} - -snapshots['TestEntryMutation::test_entry_update success'] = { - 'data': { - 'project': { - 'entryUpdate': { - 'errors': None, - 'ok': True, - 'result': { - 'attributes': [ - { - 'clientId': 'client-id-attribute-3', - 'data': { - }, - 'id': '3', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - }, - { - 'clientId': 'client-id-attribute-1', - 'data': { - }, - 'id': '1', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - }, - { - 'clientId': 'client-id-attribute-2', - 'data': { - }, - 'id': '2', - 'widget': '2', - 'widgetType': 'TIME' - } - ], - 'clientId': 'entry-101', - 'droppedExcerpt': 'This is a dropped text', - 'entryType': 'EXCERPT', - 'excerpt': 'This is a text', - 'highlightHidden': False, - 'id': '1', - 'image': { - 'id': '3', - 'title': 'file-2' - }, - 'informationDate': '2021-01-01', - 'order': 1 - } - } - } - } -} diff --git a/apps/entry/tests/test_mutations.py b/apps/entry/tests/test_mutations.py index 56d703672d..cd557e1cd1 100644 --- a/apps/entry/tests/test_mutations.py +++ b/apps/entry/tests/test_mutations.py @@ -1,5 +1,6 @@ from django.utils import timezone +from lead.models import LeadPreviewAttachment from utils.graphene.tests import GraphQLSnapShotTestCase from entry.models import Entry @@ -7,7 +8,7 @@ from user.factories import UserFactory from entry.factories import EntryFactory, EntryAttributeFactory from project.factories import ProjectFactory -from lead.factories import LeadFactory +from lead.factories import LeadFactory, LeadPreviewAttachmentFactory from analysis_framework.factories import AnalysisFrameworkFactory, WidgetFactory from gallery.factories import FileFactory @@ -161,6 +162,17 @@ class TestEntryMutation(GraphQLSnapShotTestCase): data clientId } + entryAttachment { + entryFileType + file { + name + url + } + filePreview { + name + url + } + } } } } @@ -184,6 +196,10 @@ def setUp(self): # Files self.other_file = FileFactory.create() self.our_file = FileFactory.create(created_by=self.member_user) + self.leadattachment = LeadPreviewAttachmentFactory.create( + lead=self.lead, + type=LeadPreviewAttachment.AttachmentFileType.IMAGE + ) self.dummy_data = dict({}) def test_entry_create(self): @@ -238,6 +254,11 @@ def _query_check(**kwargs): response = _query_check() self.assertMatchSnapshot(response, 'success') + # Valid input with LeadPreviewAttachment id + minput['leadAttachment'] = self.leadattachment.id + response = _query_check() + self.assertMatchSnapshot(response, 'success') + def test_entry_update(self): """ This test makes sure only valid users can update entry diff --git a/apps/entry/tests/test_schemas.py b/apps/entry/tests/test_schemas.py index b4a49b2265..3ef85fcdf4 100644 --- a/apps/entry/tests/test_schemas.py +++ b/apps/entry/tests/test_schemas.py @@ -1,6 +1,6 @@ from lead.models import Lead -from entry.models import Entry +from entry.models import Entry, EntryAttachment from quality_assurance.models import EntryReviewComment from analysis_framework.models import Widget @@ -10,7 +10,7 @@ from geo.factories import RegionFactory, AdminLevelFactory, GeoAreaFactory from project.factories import ProjectFactory from lead.factories import LeadFactory -from entry.factories import EntryFactory, EntryAttributeFactory +from entry.factories import EntryFactory, EntryAttributeFactory, EntryAttachmentFactory from analysis_framework.factories import AnalysisFrameworkFactory, WidgetFactory from organization.factories import OrganizationFactory, OrganizationTypeFactory from assessment_registry.factories import AssessmentRegistryFactory @@ -61,6 +61,17 @@ def test_lead_entries_query(self): } } controlled + entryAttachment { + entryFileType + file { + name + url + } + filePreview { + name + url + } + } } } } @@ -69,6 +80,15 @@ def test_lead_entries_query(self): lead = LeadFactory.create(project=self.project) entry = EntryFactory.create(project=self.project, analysis_framework=self.af, lead=lead) + entry_attachment = EntryAttachmentFactory.create( + entry_file_type=EntryAttachment.EntryFileType.XLSX, + ) + EntryFactory.create( + project=self.project, + analysis_framework=self.af, + lead=lead, + entry_attachment=entry_attachment + ) def _query_check(**kwargs): return self.query_check(query, variables={'projectId': self.project.pk, 'leadId': lead.id}, **kwargs) @@ -79,8 +99,9 @@ def _query_check(**kwargs): self.force_login(self.user) content = _query_check() results = content['data']['project']['lead']['entries'] - self.assertEqual(len(content['data']['project']['lead']['entries']), 1, content) - self.assertIdEqual(results[0]['id'], entry.pk, results) + self.assertEqual(len(content['data']['project']['lead']['entries']), 2, content) + self.assertTrue(results, entry.id) + self.assertTrue(results, entry_attachment.id) def test_entries_query(self): # Includes permissions checks diff --git a/apps/entry/utils.py b/apps/entry/utils.py index c88dd1ffa5..a70078da50 100644 --- a/apps/entry/utils.py +++ b/apps/entry/utils.py @@ -1,5 +1,6 @@ -from entry.models import Attribute +from entry.models import Attribute, EntryAttachment from gallery.models import File +from lead.models import LeadPreviewAttachment from utils.image import decode_base64_if_possible from .widgets.utils import set_filter_data, set_export_data @@ -68,3 +69,18 @@ def base64_to_deep_image(image, lead, user): file.file.save(decoded_file.name, decoded_file) file.projects.add(lead.project) return file + + +def leadattachment_to_entryattachment(lead_attachment: LeadPreviewAttachment): + entry_attachment = EntryAttachment.objects.create( + lead_attachment_id=lead_attachment.id, + entry_file_type=lead_attachment.type # lead attachement type and entry attachment gave same enum + ) + if lead_attachment.type == LeadPreviewAttachment.AttachmentFileType.IMAGE: + entry_attachment.file.save(lead_attachment.file.name, lead_attachment.file) + entry_attachment.file_preview = entry_attachment.file + else: + entry_attachment.file_preview.save(lead_attachment.file_preview.name, lead_attachment.file) + entry_attachment.file_preview.save(lead_attachment.file_preview.name, lead_attachment.file_preview) + + return entry_attachment diff --git a/apps/export/entries/excel_exporter.py b/apps/export/entries/excel_exporter.py index 69d38a0d0c..ec8d0452a8 100644 --- a/apps/export/entries/excel_exporter.py +++ b/apps/export/entries/excel_exporter.py @@ -492,7 +492,7 @@ def add_entries(self, entries): for group_label in entry.entrygrouplabel_set.all(): key = (group_label.group.lead_id, group_label.group_id) entries_sheet_name = 'Grouped Entries' if self.decoupled else 'Entries' - link = f'#\'{entries_sheet_name}\'!A{i+2}' + link = f'#\'{entries_sheet_name}\'!A{i + 2}' self.group_label_matrix[key][group_label.label_id] = get_hyperlink(link, entry.excerpt[:50]) lead = entry.lead diff --git a/apps/geo/models.py b/apps/geo/models.py index f16efbdb91..d0be464d21 100644 --- a/apps/geo/models.py +++ b/apps/geo/models.py @@ -396,7 +396,7 @@ def get_sub_childrens(cls, value: List[Union[str, int]], level=1): if value: filters = models.Q(id__in=list(value)) for i in range(level - 1): - filters |= models.Q(**{f"{'parent__'*i}parent__in": value}) + filters |= models.Q(**{f"{'parent__' * i}parent__in": value}) return cls.objects.filter(filters) return cls.objects.none() diff --git a/apps/lead/enums.py b/apps/lead/enums.py index 07e9f5d448..8f96bc5053 100644 --- a/apps/lead/enums.py +++ b/apps/lead/enums.py @@ -16,7 +16,7 @@ Lead.AutoExtractionStatus, name='LeadAutoEntryExtractionTypeEnum' ) LeadPreviewAttachmentTypeEnum = convert_enum_to_graphene_enum( - LeadPreviewAttachment.AttachementFileType, name='LeadPreviewAttachmentTypeEnum' + LeadPreviewAttachment.AttachmentFileType, name='LeadPreviewAttachmentTypeEnum' ) diff --git a/apps/lead/filter_set.py b/apps/lead/filter_set.py index 49bb47b94d..6a44a04593 100644 --- a/apps/lead/filter_set.py +++ b/apps/lead/filter_set.py @@ -555,10 +555,21 @@ def filter_title(self, qs, name, value): class LeadPreviewAttachmentGQFilterSet(UserResourceGqlFilterSet): type = MultipleInputFilter(LeadPreviewAttachmentTypeEnum, field_name='type') + exclude_attachment_id = IDListFilter(method='filter_exclude_lead_attachment_id') class Meta: model = LeadPreviewAttachment - fields = ['lead', 'page_number'] + fields = [ + 'lead', + 'page_number', + 'exclude_attachment_id', + ] + + def filter_exclude_lead_attachment_id(self, qs, _, value): + if value: + qs = qs.exclude(id__in=value) + return qs + return qs LeadsFilterDataType, LeadsFilterDataInputType = generate_type_for_filter_set( diff --git a/apps/lead/models.py b/apps/lead/models.py index 21f1221ab6..dece49756b 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -376,7 +376,7 @@ class LeadPreviewAttachment(models.Model): """ NOTE: File can be only used by gallery (when attached to a entry) """ - class AttachementFileType(models.IntegerChoices): + class AttachmentFileType(models.IntegerChoices): XLSX = 1, 'XLSX' IMAGE = 2, 'Image' @@ -386,14 +386,14 @@ class AttachementFileType(models.IntegerChoices): order = models.IntegerField(default=0) page_number = models.IntegerField(default=0) type = models.PositiveSmallIntegerField( - choices=AttachementFileType.choices, - default=AttachementFileType.XLSX + choices=AttachmentFileType.choices, + default=AttachmentFileType.XLSX ) file = models.FileField(upload_to='lead-preview/attachments/') file_preview = models.FileField(upload_to='lead-preview/attachments-preview/') def __str__(self): - return 'Image extracted for {}'.format(self.lead) + return 'Image extracted for {}'.format(self.lead.pk) def clone_as_deep_file(self, user): """ diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 6a98639dbb..11e6aae21a 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1844,7 +1844,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f self.assertEqual(lead_preview.page_count, 4) self.assertEqual(LeadPreviewAttachment.objects.filter(lead=self.lead).count(), 3) self.assertEqual(LeadPreviewAttachment.objects.filter( - lead=self.lead, type=LeadPreviewAttachment.AttachementFileType.IMAGE).count(), 2 + lead=self.lead, type=LeadPreviewAttachment.AttachmentFileType.IMAGE).count(), 2 ) index_lead_func.assert_called_once_with(self.lead.id) diff --git a/apps/lead/tests/test_mutations.py b/apps/lead/tests/test_mutations.py index ce85db556e..68dcbcd40f 100644 --- a/apps/lead/tests/test_mutations.py +++ b/apps/lead/tests/test_mutations.py @@ -517,7 +517,7 @@ def test_lead_copy_mutation(self): # Generating Foreign elements for wa_lead1 wa_lead1_preview = LeadPreviewFactory.create(lead=wa_lead1, text_extract='This is a random text extarct') - wa_lead1_image_preview = LeadPreviewAttachmentFactory.create(lead=wa_lead1, file='test-file-123') + wa_lead1_attachment_preview = LeadPreviewAttachmentFactory.create(lead=wa_lead1, file='test-file-123') LeadEMMTriggerFactory.create( lead=wa_lead1, emm_keyword='emm1', @@ -611,7 +611,7 @@ def _query_check(source_project, **kwargs): self.assertEqual(copied_lead1.confidentiality, wa_lead1.confidentiality) # lets check for the foreign key field copy self.assertEqual(copied_lead1.leadpreview.text_extract, wa_lead1_preview.text_extract) - self.assertEqual(list(copied_lead1.images.values_list('file', flat=True)), [wa_lead1_image_preview.file.name]) + self.assertEqual(list(copied_lead1.images.values_list('file', flat=True)), [wa_lead1_attachment_preview.file.name]) self.assertEqual( list(copied_lead1.emm_triggers.values('emm_keyword', 'emm_risk_factor', 'count')), list(wa_lead1.emm_triggers.values('emm_keyword', 'emm_risk_factor', 'count')), diff --git a/schema.graphql b/schema.graphql index ac549102b9..954b4f846c 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1578,6 +1578,7 @@ type AppEnumCollection { LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] LeadPreviewAttachmentType: [AppEnumCollectionLeadPreviewAttachmentType!] EntryEntryType: [AppEnumCollectionEntryEntryType!] + EntryAttachmentEntryFileType: [AppEnumCollectionEntryAttachmentEntryFileType!] ExportFormat: [AppEnumCollectionExportFormat!] ExportStatus: [AppEnumCollectionExportStatus!] ExportType: [AppEnumCollectionExportType!] @@ -1923,6 +1924,12 @@ type AppEnumCollectionDraftEntryType { description: String } +type AppEnumCollectionEntryAttachmentEntryFileType { + enum: EntryFileType! + label: String! + description: String +} + type AppEnumCollectionEntryEntryType { enum: EntryTagTypeEnum! label: String! @@ -3927,16 +3934,17 @@ type EntriesFilterDataType { filterableData: [EntryFilterDataType!] } -enum EntryAttachmentEntryFileType { - A_1 -} - type EntryAttachmentType { id: ID! - entryFileType: EntryAttachmentEntryFileType! - file: FileFieldType - filePreview: FileFieldType - entry: EntryType + leadAttachmentId: ID! + file: FileFieldType! + filePreview: FileFieldType! + entryFileType: EntryFileType! +} + +enum EntryFileType { + XLSX + IMAGE } input EntryFilterDataInputType { @@ -4078,7 +4086,6 @@ type EntryType { informationDate: Date excerpt: String! image: GalleryFileType - entryAttachment: EntryAttachmentType droppedExcerpt: String! highlightHidden: Boolean! controlled: Boolean @@ -4095,6 +4102,7 @@ type EntryType { verifiedByCount: Int! reviewCommentsCount: Int! draftEntry: ID + entryAttachment: EntryAttachmentType } scalar EnumDescription @@ -5288,7 +5296,7 @@ type ProjectDetailType { leadGroups(createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, page: Int = 1, ordering: String, pageSize: Int): LeadGroupListType emmEntities(name: String, page: Int = 1, ordering: String, pageSize: Int): EmmEntityListType leadEmmTriggers(lead: ID, emmKeyword: String, emmRiskFactor: String, count: Int, page: Int = 1, ordering: String, pageSize: Int): LeadEmmTriggerListType - leadPreviewAttachments(lead: ID, pageNumber: Int, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], type: [LeadPreviewAttachmentTypeEnum!], page: Int = 1, ordering: String, pageSize: Int): LeadPreviewAttachmentListType + leadPreviewAttachments(lead: ID, pageNumber: Int, excludeAttachmentId: [ID!], createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], type: [LeadPreviewAttachmentTypeEnum!], page: Int = 1, ordering: String, pageSize: Int): LeadPreviewAttachmentListType emmKeywords: [EmmKeyWordType!] emmRiskFactors: [EmmKeyRiskFactorType!] userSavedLeadFilter: UserSavedLeadFilterType From 9ba9193bd7fc81fa38ab71a7e7fcc610dec6c6a8 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 13 Jun 2024 12:29:43 +0545 Subject: [PATCH 05/17] Add validation on entry tag type Update snapshots for entries --- apps/deepl_integration/handlers.py | 10 +- apps/entry/models.py | 13 +- apps/entry/serializers.py | 36 +- .../tests/snapshots/snap_test_mutations.py | 384 +++++++++++++++++- apps/entry/tests/test_mutations.py | 29 +- apps/entry/utils.py | 19 +- apps/lead/factories.py | 17 + apps/lead/filter_set.py | 6 +- schema.graphql | 2 +- 9 files changed, 466 insertions(+), 50 deletions(-) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 008ab39d64..4858875eaa 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -658,9 +658,9 @@ def save_data( attachment_base_path = f'{lead.pk}' for image_uri in images_uri: for image in image_uri['images']: - lead_attachment = LeadPreviewAttachment(lead=lead) image_obj = RequestHelper(url=image, ignore_error=True).get_file() if image_obj: + lead_attachment = LeadPreviewAttachment(lead=lead) lead_attachment.file.save( os.path.join( attachment_base_path, @@ -668,7 +668,7 @@ def save_data( urlparse(image).path ) ), - image_obj + image_obj, ) lead_attachment.page_number = image_uri['page_number'] lead_attachment.type = LeadPreviewAttachment.AttachmentFileType.IMAGE @@ -676,10 +676,10 @@ def save_data( lead_attachment.save() for table in table_uri: - lead_attachment = LeadPreviewAttachment(lead=lead) table_img = RequestHelper(url=table['image_link'], ignore_error=True).get_file() table_attachment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() if table_img: + lead_attachment = LeadPreviewAttachment(lead=lead) lead_attachment.file_preview.save( os.path.join( attachment_base_path, @@ -687,7 +687,7 @@ def save_data( urlparse(table['image_link']).path ) ), - table_img + table_img, ) lead_attachment.page_number = table['page_number'] lead_attachment.type = LeadPreviewAttachment.AttachmentFileType.XLSX @@ -698,7 +698,7 @@ def save_data( urlparse(table['content_link']).path ) ), - table_attachment + table_attachment, ) lead_attachment.save() diff --git a/apps/entry/models.py b/apps/entry/models.py index 4324e8af86..95d1894616 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -21,10 +21,11 @@ from assisted_tagging.models import DraftEntry -class EntryAttachment(models.Model): # The entry will make reference to it as entry_attachment. +class EntryAttachment(models.Model): + # The entry will make reference to it as entry_attachment. class EntryFileType(models.IntegerChoices): - XLSX = 1, 'XLSX', - IMAGE = 2, 'Image', + XLSX = 1, 'XLSX' + IMAGE = 2, 'Image' lead_attachment = models.ForeignKey(LeadPreviewAttachment, on_delete=models.SET_NULL, null=True) entry_file_type = models.PositiveSmallIntegerField( @@ -47,9 +48,9 @@ class Entry(UserResource, ProjectEntityMixin): """ class TagType(models.TextChoices): - EXCERPT = 'excerpt', 'Excerpt', - IMAGE = 'image', 'Image', - ATTACHMENT = 'attachment', 'Attachment', + EXCERPT = 'excerpt', 'Excerpt' + IMAGE = 'image', 'Image' + ATTACHMENT = 'attachment', 'Attachment' DATA_SERIES = 'dataSeries', 'Data Series' # NOTE: data saved as tabular_field id lead = models.ForeignKey(Lead, on_delete=models.CASCADE) diff --git a/apps/entry/serializers.py b/apps/entry/serializers.py index 19a174c789..4bbf445e8a 100644 --- a/apps/entry/serializers.py +++ b/apps/entry/serializers.py @@ -634,27 +634,32 @@ def validate_lead(self, lead): raise serializers.ValidationError('Changing lead is not allowed') return lead - def validate_lead_attachment(self, lead_attachment): - if int(self.initial_data.get('lead')) != lead_attachment.lead.id: - raise serializers.ValidationError("Don't have access to this lead") - return lead_attachment - def validate(self, data): """ - Lead image is copied to deep gallery files - Raw image (base64) are saved as deep gallery files """ request = self.context['request'] - image = data.get('image') image_raw = data.pop('image_raw', None) lead_attachment = data.pop('lead_attachment', None) # ---------------- Lead - lead = data['lead'] + lead = data.get('lead') if self.instance and lead != self.instance.lead: raise serializers.ValidationError({ 'lead': 'Changing lead is not allowed' }) + # validate entry tag type + entry_type = data.get('entry_type') + if entry_type and entry_type == Entry.TagType.ATTACHMENT and lead_attachment is None: + raise serializers.ValidationError({ + 'lead_attachment': f'LeadPreviewAttachment is required with entry_tag is {entry_type}' + }) + + elif entry_type and entry_type == Entry.TagType.EXCERPT and data.get('excerpt') is None: + raise serializers.ValidationError({ + 'excerpt': f'EXCERPT is required when entry_tag is {entry_type}' + }) # ----------------- Validate Draft entry if provided draft_entry = data.get('draft_entry') @@ -677,19 +682,7 @@ def validate(self, data): else: # For update, set entry's AF with active AF data['analysis_framework_id'] = active_af_id - # ---------------- Set/validate image properly - # If gallery file is provided make sure user owns the file - if image: - if ( - (self.instance and self.instance.image) != image and - not image.is_public and - image.created_by != request.user - ): - raise serializers.ValidationError({ - 'image': f'You don\'t have permission to attach image: {image}', - }) - # If lead image is provided make sure lead are same - elif lead_attachment: + if lead_attachment: data.pop('excerpt', None) # removing excerpt when lead attachment is send if lead_attachment.lead != lead: raise serializers.ValidationError({ @@ -697,7 +690,10 @@ def validate(self, data): }) data['entry_attachment'] = leadattachment_to_entryattachment(lead_attachment) + + # ---------------- Set/validate image properly elif image_raw: + data.pop('excerpt', None) generated_image = base64_to_deep_image(image_raw, lead, request.user) if isinstance(generated_image, File): data['image'] = generated_image diff --git a/apps/entry/tests/snapshots/snap_test_mutations.py b/apps/entry/tests/snapshots/snap_test_mutations.py index d560743bbe..1be5591c67 100644 --- a/apps/entry/tests/snapshots/snap_test_mutations.py +++ b/apps/entry/tests/snapshots/snap_test_mutations.py @@ -7,27 +7,172 @@ snapshots = Snapshot() -snapshots['TestEntryMutation::test_entry_create error'] = { +snapshots['TestEntryMutation::test_entry_bulk error'] = { 'data': { 'project': { - 'entryCreate': { + 'entryBulk': { + 'deletedResult': [ + { + 'attributes': None, + 'clientId': '1', + 'droppedExcerpt': '', + 'entryType': 'DATA_SERIES', + 'excerpt': 'fWdhjOkYRBMeyyMDHqJaRUhRIWrXPvhsBkDaUUqGWlGgOtOGMmjxWkIXHaMuFbhxZtpdpKffUFeWIXiiQEJkqHMBnIWUSmTtzQPx', + 'highlightHidden': False, + 'id': '1', + 'image': { + 'id': '4', + 'title': 'file-3' + }, + 'informationDate': None, + 'order': 1 + } + ], 'errors': [ + None, + None + ], + 'result': [ { - 'arrayErrors': None, - 'clientId': 'entry-101', - 'field': 'image', - 'messages': "You don't have permission to attach image: file-1", - 'objectErrors': None + 'attributes': [ + { + 'clientId': 'client-id-old-new-attribute-1', + 'data': { + }, + 'id': '2', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-old-attribute-1', + 'data': { + }, + 'id': '1', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + } + ], + 'clientId': 'entry-old-101 (UPDATED)', + 'droppedExcerpt': 'This is a dropped text (UPDATED)', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text (UPDATED)', + 'highlightHidden': False, + 'id': '2', + 'image': { + 'id': '2', + 'title': 'file-1' + }, + 'informationDate': '2021-01-01', + 'order': 1 + }, + { + 'attributes': [ + { + 'clientId': 'client-id-new-attribute-1', + 'data': { + }, + 'id': '3', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + } + ], + 'clientId': 'entry-new-102', + 'droppedExcerpt': 'This is a dropped text (NEW)', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text (NEW)', + 'highlightHidden': False, + 'id': '3', + 'image': { + 'id': '2', + 'title': 'file-1' + }, + 'informationDate': '2021-01-01', + 'order': 1 } + ] + } + } + } +} + +snapshots['TestEntryMutation::test_entry_bulk success'] = { + 'data': { + 'project': { + 'entryBulk': { + 'deletedResult': [ + ], + 'errors': [ + None, + None ], - 'ok': False, - 'result': None + 'result': [ + { + 'attributes': [ + { + 'clientId': 'client-id-old-new-attribute-1', + 'data': { + }, + 'id': '4', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-old-attribute-1', + 'data': { + }, + 'id': '1', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + } + ], + 'clientId': 'entry-old-101 (UPDATED)', + 'droppedExcerpt': 'This is a dropped text (UPDATED)', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text (UPDATED)', + 'highlightHidden': False, + 'id': '2', + 'image': { + 'id': '3', + 'title': 'file-2' + }, + 'informationDate': '2021-01-01', + 'order': 1 + }, + { + 'attributes': [ + { + 'clientId': 'client-id-new-attribute-1', + 'data': { + }, + 'id': '5', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + } + ], + 'clientId': 'entry-new-102', + 'droppedExcerpt': 'This is a dropped text (NEW)', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text (NEW)', + 'highlightHidden': False, + 'id': '4', + 'image': { + 'id': '3', + 'title': 'file-2' + }, + 'informationDate': '2021-01-01', + 'order': 1 + } + ] } } } } -snapshots['TestEntryMutation::test_entry_create success'] = { +snapshots['TestEntryMutation::test_entry_create error'] = { 'data': { 'project': { 'entryCreate': { @@ -62,6 +207,225 @@ ], 'clientId': 'entry-101', 'droppedExcerpt': 'This is a dropped text', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text', + 'highlightHidden': False, + 'id': '1', + 'image': { + 'id': '2', + 'title': 'file-1' + }, + 'informationDate': '2021-01-01', + 'order': 1 + } + } + } + } +} + +snapshots['TestEntryMutation::test_entry_create lead-preview-attachment-success'] = { + 'data': { + 'project': { + 'entryCreate': { + 'errors': None, + 'ok': True, + 'result': { + 'attributes': [ + { + 'clientId': 'client-id-attribute-1', + 'data': { + }, + 'id': '7', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-2', + 'data': { + }, + 'id': '8', + 'widget': '2', + 'widgetType': 'TIME' + }, + { + 'clientId': 'client-id-attribute-3', + 'data': { + }, + 'id': '9', + 'widget': '3', + 'widgetType': 'GEO' + } + ], + 'clientId': 'entry-101', + 'droppedExcerpt': 'This is a dropped text', + 'entryAttachment': { + 'entryFileType': 'IMAGE', + 'file': { + 'name': 'entry/attachment/example_47FH0Sj.jpg', + 'url': 'http://testserver/media/entry/attachment/example_47FH0Sj.jpg' + }, + 'filePreview': { + 'name': 'entry/attachment/example_47FH0Sj.jpg', + 'url': 'http://testserver/media/entry/attachment/example_47FH0Sj.jpg' + }, + 'id': '1', + 'leadAttachmentId': '1' + }, + 'entryType': 'ATTACHMENT', + 'excerpt': '', + 'highlightHidden': False, + 'id': '3', + 'image': { + 'id': '3', + 'title': 'file-2' + }, + 'informationDate': '2021-01-01', + 'order': 1 + } + } + } + } +} + +snapshots['TestEntryMutation::test_entry_create success'] = { + 'data': { + 'project': { + 'entryCreate': { + 'errors': None, + 'ok': True, + 'result': { + 'attributes': [ + { + 'clientId': 'client-id-attribute-1', + 'data': { + }, + 'id': '4', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-2', + 'data': { + }, + 'id': '5', + 'widget': '2', + 'widgetType': 'TIME' + }, + { + 'clientId': 'client-id-attribute-3', + 'data': { + }, + 'id': '6', + 'widget': '3', + 'widgetType': 'GEO' + } + ], + 'clientId': 'entry-101', + 'droppedExcerpt': 'This is a dropped text', + 'entryAttachment': None, + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text', + 'highlightHidden': False, + 'id': '2', + 'image': { + 'id': '3', + 'title': 'file-2' + }, + 'informationDate': '2021-01-01', + 'order': 1 + } + } + } + } +} + +snapshots['TestEntryMutation::test_entry_update error'] = { + 'data': { + 'project': { + 'entryUpdate': { + 'errors': None, + 'ok': True, + 'result': { + 'attributes': [ + { + 'clientId': 'client-id-attribute-3', + 'data': { + }, + 'id': '3', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-1', + 'data': { + }, + 'id': '1', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-2', + 'data': { + }, + 'id': '2', + 'widget': '2', + 'widgetType': 'TIME' + } + ], + 'clientId': 'entry-101', + 'droppedExcerpt': 'This is a dropped text', + 'entryType': 'EXCERPT', + 'excerpt': 'This is a text', + 'highlightHidden': False, + 'id': '1', + 'image': { + 'id': '2', + 'title': 'file-1' + }, + 'informationDate': '2021-01-01', + 'order': 1 + } + } + } + } +} + +snapshots['TestEntryMutation::test_entry_update success'] = { + 'data': { + 'project': { + 'entryUpdate': { + 'errors': None, + 'ok': True, + 'result': { + 'attributes': [ + { + 'clientId': 'client-id-attribute-3', + 'data': { + }, + 'id': '6', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-1', + 'data': { + }, + 'id': '4', + 'widget': '1', + 'widgetType': 'TIME_RANGE' + }, + { + 'clientId': 'client-id-attribute-2', + 'data': { + }, + 'id': '5', + 'widget': '2', + 'widgetType': 'TIME' + } + ], + 'clientId': 'entry-101', + 'droppedExcerpt': 'This is a dropped text', 'entryType': 'EXCERPT', 'excerpt': 'This is a text', 'highlightHidden': False, diff --git a/apps/entry/tests/test_mutations.py b/apps/entry/tests/test_mutations.py index cd557e1cd1..3e3eda3e38 100644 --- a/apps/entry/tests/test_mutations.py +++ b/apps/entry/tests/test_mutations.py @@ -46,6 +46,19 @@ class TestEntryMutation(GraphQLSnapShotTestCase): data clientId } + entryAttachment{ + id + entryFileType + leadAttachmentId + file{ + name + url + } + filePreview{ + name + url + } + } } } } @@ -110,6 +123,19 @@ class TestEntryMutation(GraphQLSnapShotTestCase): data clientId } + entryAttachment{ + id + entryFileType + leadAttachmentId + file{ + name + url + } + filePreview{ + name + url + } + } } } } @@ -255,9 +281,10 @@ def _query_check(**kwargs): self.assertMatchSnapshot(response, 'success') # Valid input with LeadPreviewAttachment id + minput['entryType'] = self.genum(Entry.TagType.ATTACHMENT) minput['leadAttachment'] = self.leadattachment.id response = _query_check() - self.assertMatchSnapshot(response, 'success') + self.assertMatchSnapshot(response, 'lead-preview-attachment-success') def test_entry_update(self): """ diff --git a/apps/entry/utils.py b/apps/entry/utils.py index a70078da50..e80f050081 100644 --- a/apps/entry/utils.py +++ b/apps/entry/utils.py @@ -72,15 +72,26 @@ def base64_to_deep_image(image, lead, user): def leadattachment_to_entryattachment(lead_attachment: LeadPreviewAttachment): + lead_attachment_file_name = str(lead_attachment.file).split('/')[-1] + lead_attachment_file_preview_name = str(lead_attachment.file_preview).split('/')[-1] entry_attachment = EntryAttachment.objects.create( lead_attachment_id=lead_attachment.id, entry_file_type=lead_attachment.type # lead attachement type and entry attachment gave same enum ) if lead_attachment.type == LeadPreviewAttachment.AttachmentFileType.IMAGE: - entry_attachment.file.save(lead_attachment.file.name, lead_attachment.file) + entry_attachment.file.save( + lead_attachment_file_name, + lead_attachment.file, + ) entry_attachment.file_preview = entry_attachment.file else: - entry_attachment.file_preview.save(lead_attachment.file_preview.name, lead_attachment.file) - entry_attachment.file_preview.save(lead_attachment.file_preview.name, lead_attachment.file_preview) - + entry_attachment.file.save( + lead_attachment_file_name, + lead_attachment.file + ) + entry_attachment.file_preview.save( + lead_attachment_file_preview_name, + lead_attachment.file_preview + ) + entry_attachment.save() return entry_attachment diff --git a/apps/lead/factories.py b/apps/lead/factories.py index 2f60d667d8..b254519c14 100644 --- a/apps/lead/factories.py +++ b/apps/lead/factories.py @@ -3,6 +3,8 @@ from factory import fuzzy from factory.django import DjangoModelFactory +from django.core.files.base import ContentFile + from project.factories import ProjectFactory from gallery.factories import FileFactory from .models import ( @@ -88,6 +90,21 @@ class LeadPreviewAttachmentFactory(DjangoModelFactory): class Meta: model = LeadPreviewAttachment + file = factory.LazyAttribute( + lambda _: ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), 'example.jpg' + ) + ) + file_preview = factory.LazyAttribute( + lambda _: ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), 'example.jpg' + ) + ) + class UserSavedLeadFilterFactory(DjangoModelFactory): class Meta: diff --git a/apps/lead/filter_set.py b/apps/lead/filter_set.py index 6a44a04593..da1c0d4c11 100644 --- a/apps/lead/filter_set.py +++ b/apps/lead/filter_set.py @@ -555,17 +555,17 @@ def filter_title(self, qs, name, value): class LeadPreviewAttachmentGQFilterSet(UserResourceGqlFilterSet): type = MultipleInputFilter(LeadPreviewAttachmentTypeEnum, field_name='type') - exclude_attachment_id = IDListFilter(method='filter_exclude_lead_attachment_id') + exclude_attachment_ids = IDListFilter(method='filter_exclude_lead_attachment_ids') class Meta: model = LeadPreviewAttachment fields = [ 'lead', 'page_number', - 'exclude_attachment_id', + 'exclude_attachment_ids', ] - def filter_exclude_lead_attachment_id(self, qs, _, value): + def filter_exclude_lead_attachment_ids(self, qs, _, value): if value: qs = qs.exclude(id__in=value) return qs diff --git a/schema.graphql b/schema.graphql index 954b4f846c..3aa54dd23a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -5296,7 +5296,7 @@ type ProjectDetailType { leadGroups(createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, page: Int = 1, ordering: String, pageSize: Int): LeadGroupListType emmEntities(name: String, page: Int = 1, ordering: String, pageSize: Int): EmmEntityListType leadEmmTriggers(lead: ID, emmKeyword: String, emmRiskFactor: String, count: Int, page: Int = 1, ordering: String, pageSize: Int): LeadEmmTriggerListType - leadPreviewAttachments(lead: ID, pageNumber: Int, excludeAttachmentId: [ID!], createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], type: [LeadPreviewAttachmentTypeEnum!], page: Int = 1, ordering: String, pageSize: Int): LeadPreviewAttachmentListType + leadPreviewAttachments(lead: ID, pageNumber: Int, excludeAttachmentIds: [ID!], createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], type: [LeadPreviewAttachmentTypeEnum!], page: Int = 1, ordering: String, pageSize: Int): LeadPreviewAttachmentListType emmKeywords: [EmmKeyWordType!] emmRiskFactors: [EmmKeyRiskFactorType!] userSavedLeadFilter: UserSavedLeadFilterType From 0a521cf3b3a279c90d7c4918aba60f0fcb90de4d Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 14 Jun 2024 14:44:24 +0545 Subject: [PATCH 06/17] Adjust file path for LeadPreviewAttachmentFactory - Issue related to random string attached to same filepath --- apps/entry/models.py | 25 ++++++++++++++ apps/entry/serializers.py | 5 +-- .../tests/snapshots/snap_test_mutations.py | 8 ++--- apps/entry/utils.py | 29 +--------------- apps/lead/factories.py | 33 ++++++++++++------- 5 files changed, 54 insertions(+), 46 deletions(-) diff --git a/apps/entry/models.py b/apps/entry/models.py index 95d1894616..7df0da4b2c 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -1,3 +1,4 @@ +import os from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres.aggregates.general import ArrayAgg from django.contrib.postgres.fields import ArrayField @@ -27,6 +28,13 @@ class EntryFileType(models.IntegerChoices): XLSX = 1, 'XLSX' IMAGE = 2, 'Image' + LEAD_TO_ENTRY_TYPE = { + LeadPreviewAttachment.AttachmentFileType.XLSX: EntryFileType.XLSX, + LeadPreviewAttachment.AttachmentFileType.IMAGE: EntryFileType.IMAGE, + } + assert len(list(LeadPreviewAttachment.AttachmentFileType)) == len(LEAD_TO_ENTRY_TYPE.keys()), \ + 'Make sure to sync LEAD_TO_ENTRY_TYPE with LeadPreviewAttachment.AttachmentFileType' + lead_attachment = models.ForeignKey(LeadPreviewAttachment, on_delete=models.SET_NULL, null=True) entry_file_type = models.PositiveSmallIntegerField( choices=EntryFileType.choices, @@ -38,6 +46,23 @@ class EntryFileType(models.IntegerChoices): def __str__(self): return f'{self.file}' + @classmethod + def clone_from_lead_attachment(cls, lead_attachment: LeadPreviewAttachment) -> 'EntryAttachment': + lead_attachment_file_name = os.path.basename(lead_attachment.file.name) + lead_attachment_file_preview_name = os.path.basename(lead_attachment.file_preview.name) + entry_attachment = EntryAttachment.objects.create( + lead_attachment_id=lead_attachment.pk, + entry_file_type=cls.LEAD_TO_ENTRY_TYPE[lead_attachment.type], + ) + if lead_attachment.type == LeadPreviewAttachment.AttachmentFileType.IMAGE: + entry_attachment.file.save(lead_attachment_file_name, lead_attachment.file) + entry_attachment.file_preview = entry_attachment.file + else: + entry_attachment.file.save(lead_attachment_file_name, lead_attachment.file) + entry_attachment.file_preview.save(lead_attachment_file_preview_name, lead_attachment.file_preview) + entry_attachment.save() + return entry_attachment + class Entry(UserResource, ProjectEntityMixin): """ diff --git a/apps/entry/serializers.py b/apps/entry/serializers.py index 4bbf445e8a..de6d397963 100644 --- a/apps/entry/serializers.py +++ b/apps/entry/serializers.py @@ -33,12 +33,13 @@ EntryCommentText, ExportData, FilterData, + EntryAttachment, # Entry Grouping ProjectEntryLabel, LeadEntryGroup, EntryGroupLabel, ) -from .utils import base64_to_deep_image, leadattachment_to_entryattachment +from .utils import base64_to_deep_image logger = logging.getLogger(__name__) @@ -689,7 +690,7 @@ def validate(self, data): 'lead_attachment': f'You don\'t have permission to attach lead attachment: {lead_attachment}', }) - data['entry_attachment'] = leadattachment_to_entryattachment(lead_attachment) + data['entry_attachment'] = EntryAttachment.clone_from_lead_attachment(lead_attachment) # ---------------- Set/validate image properly elif image_raw: diff --git a/apps/entry/tests/snapshots/snap_test_mutations.py b/apps/entry/tests/snapshots/snap_test_mutations.py index 1be5591c67..78225f4cf9 100644 --- a/apps/entry/tests/snapshots/snap_test_mutations.py +++ b/apps/entry/tests/snapshots/snap_test_mutations.py @@ -262,12 +262,12 @@ 'entryAttachment': { 'entryFileType': 'IMAGE', 'file': { - 'name': 'entry/attachment/example_47FH0Sj.jpg', - 'url': 'http://testserver/media/entry/attachment/example_47FH0Sj.jpg' + 'name': 'entry/attachment/example_1_1.png', + 'url': 'http://testserver/media/entry/attachment/example_1_1.png' }, 'filePreview': { - 'name': 'entry/attachment/example_47FH0Sj.jpg', - 'url': 'http://testserver/media/entry/attachment/example_47FH0Sj.jpg' + 'name': 'entry/attachment/example_1_1.png', + 'url': 'http://testserver/media/entry/attachment/example_1_1.png' }, 'id': '1', 'leadAttachmentId': '1' diff --git a/apps/entry/utils.py b/apps/entry/utils.py index e80f050081..c88dd1ffa5 100644 --- a/apps/entry/utils.py +++ b/apps/entry/utils.py @@ -1,6 +1,5 @@ -from entry.models import Attribute, EntryAttachment +from entry.models import Attribute from gallery.models import File -from lead.models import LeadPreviewAttachment from utils.image import decode_base64_if_possible from .widgets.utils import set_filter_data, set_export_data @@ -69,29 +68,3 @@ def base64_to_deep_image(image, lead, user): file.file.save(decoded_file.name, decoded_file) file.projects.add(lead.project) return file - - -def leadattachment_to_entryattachment(lead_attachment: LeadPreviewAttachment): - lead_attachment_file_name = str(lead_attachment.file).split('/')[-1] - lead_attachment_file_preview_name = str(lead_attachment.file_preview).split('/')[-1] - entry_attachment = EntryAttachment.objects.create( - lead_attachment_id=lead_attachment.id, - entry_file_type=lead_attachment.type # lead attachement type and entry attachment gave same enum - ) - if lead_attachment.type == LeadPreviewAttachment.AttachmentFileType.IMAGE: - entry_attachment.file.save( - lead_attachment_file_name, - lead_attachment.file, - ) - entry_attachment.file_preview = entry_attachment.file - else: - entry_attachment.file.save( - lead_attachment_file_name, - lead_attachment.file - ) - entry_attachment.file_preview.save( - lead_attachment_file_preview_name, - lead_attachment.file_preview - ) - entry_attachment.save() - return entry_attachment diff --git a/apps/lead/factories.py b/apps/lead/factories.py index b254519c14..7138540fea 100644 --- a/apps/lead/factories.py +++ b/apps/lead/factories.py @@ -87,23 +87,32 @@ class Meta: class LeadPreviewAttachmentFactory(DjangoModelFactory): + sequence_number = factory.Sequence(lambda n: n) + class Meta: model = LeadPreviewAttachment - file = factory.LazyAttribute( - lambda _: ContentFile( - factory.django.ImageField()._make_data( - {'width': 1024, 'height': 768} - ), 'example.jpg' + @classmethod + def _create(cls, model_class, *args, **kwargs): + sequence_number = kwargs.pop('sequence_number') + instance = super()._create(model_class, *args, **kwargs) + instance.file.save( + f'example_{instance.id}_{sequence_number}.png', + ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), + ), ) - ) - file_preview = factory.LazyAttribute( - lambda _: ContentFile( - factory.django.ImageField()._make_data( - {'width': 1024, 'height': 768} - ), 'example.jpg' + instance.file_preview.save( + f'example_{instance.id}_{sequence_number}_preview.png', + ContentFile( + factory.django.ImageField()._make_data( + {'width': 1024, 'height': 768} + ), + ), ) - ) + return instance class UserSavedLeadFilterFactory(DjangoModelFactory): From 3c4501faab261243ca6d7c093d5b5e24a3c3c20b Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 14 Jun 2024 17:28:28 +0545 Subject: [PATCH 07/17] Update testcases and SnapShot --- .../tests/snapshots/snap_test_mutations.py | 175 ++++++++---------- apps/entry/tests/test_mutations.py | 38 +++- 2 files changed, 111 insertions(+), 102 deletions(-) diff --git a/apps/entry/tests/snapshots/snap_test_mutations.py b/apps/entry/tests/snapshots/snap_test_mutations.py index 78225f4cf9..397613617c 100644 --- a/apps/entry/tests/snapshots/snap_test_mutations.py +++ b/apps/entry/tests/snapshots/snap_test_mutations.py @@ -17,12 +17,12 @@ 'clientId': '1', 'droppedExcerpt': '', 'entryType': 'DATA_SERIES', - 'excerpt': 'fWdhjOkYRBMeyyMDHqJaRUhRIWrXPvhsBkDaUUqGWlGgOtOGMmjxWkIXHaMuFbhxZtpdpKffUFeWIXiiQEJkqHMBnIWUSmTtzQPx', + 'excerpt': 'HChpoevbLJoLoaeTOdoecveGprQFnIiUKKEpYEZAmggQBwBADUdRPPgdzUvZgpmmICiBlrDpeCZJgdPIafWpkAFEnzdkyayqYYDs', 'highlightHidden': False, 'id': '1', 'image': { - 'id': '4', - 'title': 'file-3' + 'id': '5', + 'title': 'file-4' }, 'informationDate': None, 'order': 1 @@ -30,7 +30,24 @@ ], 'errors': [ None, - None + [ + { + 'arrayErrors': None, + 'clientId': 'entry-new-102', + 'field': 'leadAttachment', + 'messages': "You don't have permission to attach lead attachment: Image extracted for 2", + 'objectErrors': None + } + ], + [ + { + 'arrayErrors': None, + 'clientId': 'entry-new-103', + 'field': 'leadAttachment', + 'messages': 'LeadPreviewAttachment is required with entry_tag is attachment', + 'objectErrors': None + } + ] ], 'result': [ { @@ -41,7 +58,7 @@ }, 'id': '2', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-old-attribute-1', @@ -49,7 +66,7 @@ }, 'id': '1', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' } ], 'clientId': 'entry-old-101 (UPDATED)', @@ -60,37 +77,14 @@ 'highlightHidden': False, 'id': '2', 'image': { - 'id': '2', - 'title': 'file-1' + 'id': '6', + 'title': 'file-5' }, 'informationDate': '2021-01-01', 'order': 1 }, - { - 'attributes': [ - { - 'clientId': 'client-id-new-attribute-1', - 'data': { - }, - 'id': '3', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - } - ], - 'clientId': 'entry-new-102', - 'droppedExcerpt': 'This is a dropped text (NEW)', - 'entryAttachment': None, - 'entryType': 'EXCERPT', - 'excerpt': 'This is a text (NEW)', - 'highlightHidden': False, - 'id': '3', - 'image': { - 'id': '2', - 'title': 'file-1' - }, - 'informationDate': '2021-01-01', - 'order': 1 - } + None, + None ] } } @@ -105,7 +99,24 @@ ], 'errors': [ None, - None + [ + { + 'arrayErrors': None, + 'clientId': 'entry-new-102', + 'field': 'leadAttachment', + 'messages': "You don't have permission to attach lead attachment: Image extracted for 2", + 'objectErrors': None + } + ], + [ + { + 'arrayErrors': None, + 'clientId': 'entry-new-103', + 'field': 'leadAttachment', + 'messages': 'LeadPreviewAttachment is required with entry_tag is attachment', + 'objectErrors': None + } + ] ], 'result': [ { @@ -114,9 +125,9 @@ 'clientId': 'client-id-old-new-attribute-1', 'data': { }, - 'id': '4', + 'id': '3', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-old-attribute-1', @@ -124,7 +135,7 @@ }, 'id': '1', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' } ], 'clientId': 'entry-old-101 (UPDATED)', @@ -135,37 +146,14 @@ 'highlightHidden': False, 'id': '2', 'image': { - 'id': '3', - 'title': 'file-2' + 'id': '6', + 'title': 'file-5' }, 'informationDate': '2021-01-01', 'order': 1 }, - { - 'attributes': [ - { - 'clientId': 'client-id-new-attribute-1', - 'data': { - }, - 'id': '5', - 'widget': '1', - 'widgetType': 'TIME_RANGE' - } - ], - 'clientId': 'entry-new-102', - 'droppedExcerpt': 'This is a dropped text (NEW)', - 'entryAttachment': None, - 'entryType': 'EXCERPT', - 'excerpt': 'This is a text (NEW)', - 'highlightHidden': False, - 'id': '4', - 'image': { - 'id': '3', - 'title': 'file-2' - }, - 'informationDate': '2021-01-01', - 'order': 1 - } + None, + None ] } } @@ -186,7 +174,7 @@ }, 'id': '1', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-2', @@ -194,7 +182,7 @@ }, 'id': '2', 'widget': '2', - 'widgetType': 'TIME' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-3', @@ -202,7 +190,7 @@ }, 'id': '3', 'widget': '3', - 'widgetType': 'GEO' + 'widgetType': 'SCALE' } ], 'clientId': 'entry-101', @@ -213,8 +201,8 @@ 'highlightHidden': False, 'id': '1', 'image': { - 'id': '2', - 'title': 'file-1' + 'id': '3', + 'title': 'file-2' }, 'informationDate': '2021-01-01', 'order': 1 @@ -238,7 +226,7 @@ }, 'id': '7', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-2', @@ -246,7 +234,7 @@ }, 'id': '8', 'widget': '2', - 'widgetType': 'TIME' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-3', @@ -254,7 +242,7 @@ }, 'id': '9', 'widget': '3', - 'widgetType': 'GEO' + 'widgetType': 'SCALE' } ], 'clientId': 'entry-101', @@ -262,12 +250,12 @@ 'entryAttachment': { 'entryFileType': 'IMAGE', 'file': { - 'name': 'entry/attachment/example_1_1.png', - 'url': 'http://testserver/media/entry/attachment/example_1_1.png' + 'name': 'entry/attachment/example_1_2.png', + 'url': 'http://testserver/media/entry/attachment/example_1_2.png' }, 'filePreview': { - 'name': 'entry/attachment/example_1_1.png', - 'url': 'http://testserver/media/entry/attachment/example_1_1.png' + 'name': 'entry/attachment/example_1_2.png', + 'url': 'http://testserver/media/entry/attachment/example_1_2.png' }, 'id': '1', 'leadAttachmentId': '1' @@ -276,10 +264,7 @@ 'excerpt': '', 'highlightHidden': False, 'id': '3', - 'image': { - 'id': '3', - 'title': 'file-2' - }, + 'image': None, 'informationDate': '2021-01-01', 'order': 1 } @@ -302,7 +287,7 @@ }, 'id': '4', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-2', @@ -310,7 +295,7 @@ }, 'id': '5', 'widget': '2', - 'widgetType': 'TIME' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-3', @@ -318,7 +303,7 @@ }, 'id': '6', 'widget': '3', - 'widgetType': 'GEO' + 'widgetType': 'SCALE' } ], 'clientId': 'entry-101', @@ -329,8 +314,8 @@ 'highlightHidden': False, 'id': '2', 'image': { - 'id': '3', - 'title': 'file-2' + 'id': '4', + 'title': 'file-3' }, 'informationDate': '2021-01-01', 'order': 1 @@ -354,7 +339,7 @@ }, 'id': '3', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-1', @@ -362,7 +347,7 @@ }, 'id': '1', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-2', @@ -370,7 +355,7 @@ }, 'id': '2', 'widget': '2', - 'widgetType': 'TIME' + 'widgetType': 'MATRIX1D' } ], 'clientId': 'entry-101', @@ -380,8 +365,8 @@ 'highlightHidden': False, 'id': '1', 'image': { - 'id': '2', - 'title': 'file-1' + 'id': '3', + 'title': 'file-2' }, 'informationDate': '2021-01-01', 'order': 1 @@ -405,7 +390,7 @@ }, 'id': '6', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-1', @@ -413,7 +398,7 @@ }, 'id': '4', 'widget': '1', - 'widgetType': 'TIME_RANGE' + 'widgetType': 'MATRIX1D' }, { 'clientId': 'client-id-attribute-2', @@ -421,7 +406,7 @@ }, 'id': '5', 'widget': '2', - 'widgetType': 'TIME' + 'widgetType': 'MATRIX1D' } ], 'clientId': 'entry-101', @@ -431,8 +416,8 @@ 'highlightHidden': False, 'id': '1', 'image': { - 'id': '3', - 'title': 'file-2' + 'id': '4', + 'title': 'file-3' }, 'informationDate': '2021-01-01', 'order': 1 diff --git a/apps/entry/tests/test_mutations.py b/apps/entry/tests/test_mutations.py index 3e3eda3e38..17d5f20d40 100644 --- a/apps/entry/tests/test_mutations.py +++ b/apps/entry/tests/test_mutations.py @@ -209,6 +209,7 @@ def setUp(self): super().setUp() self.af = AnalysisFrameworkFactory.create() self.project = ProjectFactory.create(analysis_framework=self.af) + self.other_project = ProjectFactory.create(analysis_framework=self.af) # User with role self.non_member_user = UserFactory.create() self.readonly_member_user = UserFactory.create() @@ -216,6 +217,7 @@ def setUp(self): self.project.add_member(self.readonly_member_user, role=self.project_role_reader_non_confidential) self.project.add_member(self.member_user, role=self.project_role_member) self.lead = LeadFactory.create(project=self.project) + self.other_lead = LeadFactory.create(project=self.other_project) self.widget1 = WidgetFactory.create(analysis_framework=self.af) self.widget2 = WidgetFactory.create(analysis_framework=self.af) self.widget3 = WidgetFactory.create(analysis_framework=self.af) @@ -226,6 +228,10 @@ def setUp(self): lead=self.lead, type=LeadPreviewAttachment.AttachmentFileType.IMAGE ) + self.other_leadattachment = LeadPreviewAttachmentFactory.create( + lead=self.other_lead, + type=LeadPreviewAttachment.AttachmentFileType.IMAGE + ) self.dummy_data = dict({}) def test_entry_create(self): @@ -283,8 +289,13 @@ def _query_check(**kwargs): # Valid input with LeadPreviewAttachment id minput['entryType'] = self.genum(Entry.TagType.ATTACHMENT) minput['leadAttachment'] = self.leadattachment.id + minput.pop('image') response = _query_check() self.assertMatchSnapshot(response, 'lead-preview-attachment-success') + self.assertEqual( + response['data']['project']['entryCreate']['result']['entryAttachment']['leadAttachmentId'], + str(self.leadattachment.id) + ) def test_entry_update(self): """ @@ -403,8 +414,6 @@ def test_entry_bulk(self): order=1, lead=self.lead.pk, informationDate=self.get_date_str(timezone.now()), - image=self.other_file.pk, - # leadImage='', highlightHidden=False, excerpt='This is a text (UPDATED)', entryType=self.genum(Entry.TagType.EXCERPT), @@ -423,13 +432,30 @@ def test_entry_bulk(self): order=1, lead=self.lead.pk, informationDate=self.get_date_str(timezone.now()), - image=self.other_file.pk, - # leadImage='', highlightHidden=False, excerpt='This is a text (NEW)', - entryType=self.genum(Entry.TagType.EXCERPT), + entryType=self.genum(Entry.TagType.ATTACHMENT), droppedExcerpt='This is a dropped text (NEW)', + leadAttachment=self.other_leadattachment.id, clientId='entry-new-102', + ), + dict( + attributes=[ + dict( + widget=self.widget1.pk, + data=self.dummy_data, + clientId='client-id-new-attribute-1', + widgetVersion=1, + ), + ], + order=1, + lead=self.lead.pk, + informationDate=self.get_date_str(timezone.now()), + highlightHidden=False, + excerpt='This is a text (NEW)', + entryType=self.genum(Entry.TagType.ATTACHMENT), + droppedExcerpt='This is a dropped text (NEW)', + clientId='entry-new-103', ) ], ) @@ -459,8 +485,6 @@ def _query_check(**kwargs): self.assertMatchSnapshot(response, 'error') # Valid input - minput['items'][0]['image'] = self.our_file.pk - minput['items'][1]['image'] = self.our_file.pk response = _query_check() self.assertMatchSnapshot(response, 'success') From 38784d9a9665c4121bd6d4c6a183a3c91c3660c5 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 18 Jun 2024 10:27:42 +0545 Subject: [PATCH 08/17] Add image url for attachment preview and image Add dataloader in image url --- apps/entry/dataloaders.py | 27 ++++++++++++++++++++++++++- apps/entry/schema.py | 6 ++++++ schema.graphql | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/apps/entry/dataloaders.py b/apps/entry/dataloaders.py index 722ac52212..1dd0df1816 100644 --- a/apps/entry/dataloaders.py +++ b/apps/entry/dataloaders.py @@ -1,5 +1,5 @@ from collections import defaultdict - +from deep.serializers import URLCachedFileField from promise import Promise from django.utils.functional import cached_property from django.db import models @@ -120,6 +120,27 @@ def batch_load_fn(self, keys): return Promise.resolve([counts.get(key, 0) for key in keys]) +class EntryImageUrlLoader(DataLoaderWithContext): + def batch_load_fn(self, keys): + entry_qs = Entry.objects.filter(id__in=keys) + entry_dict = {entry.id: entry for entry in entry_qs} + results = [] + for key in keys: + entry = entry_dict.get(key) + if entry.entry_type == Entry.TagType.IMAGE: + url = self.context.request.build_absolute_uri( + URLCachedFileField.name_to_representation(entry.image) + ) + elif entry.entry_type == Entry.TagType.ATTACHMENT: + url = self.context.request.build_absolute_uri( + URLCachedFileField.name_to_representation(entry.entry_attachment.file) + ) + else: + url = None + results.append(url) + return Promise.resolve(results) + + class DataLoaders(WithContextMixin): @cached_property def entry(self): @@ -152,3 +173,7 @@ def verified_by(self): @cached_property def verified_by_count(self): return EntryVerifiedByCountLoader(context=self.context) + + @cached_property + def entry_image_preview_url(self): + return EntryImageUrlLoader(context=self.context) diff --git a/apps/entry/schema.py b/apps/entry/schema.py index b11e995194..4944d3354e 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -10,6 +10,7 @@ from utils.graphene.fields import DjangoPaginatedListObjectField, DjangoListField from user_resource.schema import UserResourceMixin from deep.permissions import ProjectPermissions as PP +from deep.serializers import URLCachedFileField from lead.models import Lead from user.schema import UserType @@ -118,6 +119,7 @@ class Meta: review_comments_count = graphene.Int(required=True) draft_entry = graphene.ID(source="draft_entry_id") entry_attachment = graphene.Field(EntryAttachmentType, required=False) + canonical_preview_image = graphene.String(required=False) # project_labels TODO: # tabular_field TODO: @@ -152,6 +154,10 @@ def resolve_verified_by_count(root, info, **_): return len(root.verified_by.all()) return info.context.dl.entry.verified_by_count.load(root.pk) + @staticmethod + def resolve_canonical_preview_image(root, info, **_): + return info.context.dl.entry.entry_image_preview_url.load(root.pk) + class EntryListType(CustomDjangoListObjectType): class Meta: diff --git a/schema.graphql b/schema.graphql index 3aa54dd23a..9e264da86b 100644 --- a/schema.graphql +++ b/schema.graphql @@ -4103,6 +4103,7 @@ type EntryType { reviewCommentsCount: Int! draftEntry: ID entryAttachment: EntryAttachmentType + canonicalPreviewImage: String } scalar EnumDescription From 5e15178d06e6c06bc0ba6654f913f9801266adc2 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 19 Jun 2024 14:19:47 +0545 Subject: [PATCH 09/17] Add connector lead attachment and save on lead --- apps/commons/receivers.py | 4 +- apps/deepl_integration/handlers.py | 77 +++++++++++++++---- apps/deepl_integration/serializers.py | 11 ++- apps/entry/schema.py | 2 +- apps/lead/views.py | 6 +- .../migrations/0009_auto_20240618_0924.py | 29 +++++++ apps/unified_connector/models.py | 17 +++- apps/unified_connector/tests/test_mutation.py | 39 ++++++++-- utils/graphene/mutation.py | 8 +- 9 files changed, 155 insertions(+), 38 deletions(-) create mode 100644 apps/unified_connector/migrations/0009_auto_20240618_0924.py diff --git a/apps/commons/receivers.py b/apps/commons/receivers.py index 088ce696d3..9b420e29f9 100644 --- a/apps/commons/receivers.py +++ b/apps/commons/receivers.py @@ -7,14 +7,14 @@ LeadPreview, LeadPreviewAttachment, ) -from unified_connector.models import ConnectorLeadPreviewImage +from unified_connector.models import ConnectorLeadPreviewAttachment # Lead @receiver(models.signals.post_delete, sender=LeadPreview) @receiver(models.signals.post_delete, sender=LeadPreviewAttachment) # Unified Connector -@receiver(models.signals.post_delete, sender=ConnectorLeadPreviewImage) +@receiver(models.signals.post_delete, sender=ConnectorLeadPreviewAttachment) def cleanup_file_on_instance_delete(sender, instance, **kwargs): files = [] for field in instance._meta.get_fields(): diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 4858875eaa..c64a37c2d5 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -3,7 +3,7 @@ import copy import requests import logging -from typing import List, Type +from typing import Dict, List, Type from functools import reduce from urllib.parse import urlparse @@ -30,7 +30,7 @@ ) from unified_connector.models import ( ConnectorLead, - ConnectorLeadPreviewImage, + ConnectorLeadPreviewAttachment, ConnectorSource, UnifiedConnector, ) @@ -706,6 +706,7 @@ def save_data( return lead @staticmethod + @transaction.atomic def save_lead_data_using_connector_lead( lead: Lead, connector_lead: ConnectorLead, @@ -724,11 +725,16 @@ def save_lead_data_using_connector_lead( ) # Save extracted images as LeadPreviewAttachment instances # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? - for connector_lead_preview_image in connector_lead.preview_images.all(): - lead_image = LeadPreviewAttachment(lead=lead) - lead_image.file.save( - connector_lead_preview_image.image.name, - connector_lead_preview_image.image, + for connector_lead_attachment in connector_lead.preview_images.all(): + lead_attachment = LeadPreviewAttachment(lead=lead) + lead_attachment.order = connector_lead_attachment.order + lead_attachment.file.save( + connector_lead_attachment.file.name, + connector_lead_attachment.file, + ) + lead_attachment.file_preview.save( + connector_lead_attachment.file_preview.name, + connector_lead_attachment.file_preview ) lead.update_extraction_status(Lead.ExtractionStatus.SUCCESS) return True @@ -742,7 +748,8 @@ class UnifiedConnectorLeadHandler(BaseHandler): def save_data( connector_lead: ConnectorLead, text_source_uri: str, - images_uri: List[str], + images_uri: List[Dict], + table_uri: List[Dict], word_count: int, page_count: int, text_extraction_id: str, @@ -751,16 +758,54 @@ def save_data( connector_lead.word_count = word_count connector_lead.page_count = page_count connector_lead.text_extraction_id = text_extraction_id - image_base_path = f'{connector_lead.pk}' + + attachment_base_path = f'{connector_lead.pk}' for image_uri in images_uri: - lead_image = ConnectorLeadPreviewImage(connector_lead=connector_lead) - image_obj = RequestHelper(url=image_uri, ignore_error=True).get_file() - if image_obj: - lead_image.image.save( - os.path.join(image_base_path, os.path.basename(urlparse(image_uri).path)), - image_obj, + for image in image_uri['images']: + image_obj = RequestHelper(url=image, ignore_error=True).get_file() + if image_obj: + connector_lead_attachment = ConnectorLeadPreviewAttachment(connector_lead=connector_lead) + connector_lead_attachment.file.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(image).path + ) + ), + image_obj, + ) + connector_lead_attachment.page_number = image_uri['page_number'] + connector_lead_attachment.type = ConnectorLeadPreviewAttachment.ConnectorAttachmentFileType.IMAGE + connector_lead_attachment.file_preview = connector_lead_attachment.file + connector_lead_attachment.save() + + for table in table_uri: + table_img = RequestHelper(url=table['image_link'], ignore_error=True).get_file() + table_attachment = RequestHelper(url=table['content_link'], ignore_error=True).get_file() + if table_img: + connector_lead_attachment = ConnectorLeadPreviewAttachment(connector_lead=connector_lead) + connector_lead_attachment.file_preview.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(table['image_link']).path + ) + ), + table_img, ) - lead_image.save() + connector_lead_attachment.page_number = table['page_number'] + connector_lead_attachment.type = ConnectorLeadPreviewAttachment.ConnectorAttachmentFileType.XLSX + connector_lead_attachment.file.save( + os.path.join( + attachment_base_path, + os.path.basename( + urlparse(table['content_link']).path + ) + ), + table_attachment, + ) + connector_lead_attachment.save() + connector_lead.update_extraction_status(ConnectorLead.ExtractionStatus.SUCCESS, commit=False) connector_lead.save() return connector_lead diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index ffe69995fe..3915bea38f 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -143,9 +143,13 @@ class UnifiedConnectorLeadExtractCallbackSerializer(DeeplServerBaseCallbackSeria Serialize deepl extractor """ # Data fields - images_path = serializers.ListField( - child=serializers.CharField(allow_blank=True), - required=False, default=[], + images_path = serializers.ListSerializer( + child=ImagePathSerializer(required=True), + required=False + ) + tables_path = serializers.ListSerializer( + child=TablePathSerializer(required=True), + required=False ) text_path = serializers.CharField(required=False, allow_null=True) total_words_count = serializers.IntegerField(required=False, default=0, allow_null=True) @@ -175,6 +179,7 @@ def create(self, data): connector_lead, data['text_path'], data.get('images_path', [])[:10], # TODO: Support for more images, to much image will error. + data['tables_path'], data['total_words_count'], data['total_pages'], data['text_extraction_id'], diff --git a/apps/entry/schema.py b/apps/entry/schema.py index 4944d3354e..34f43282ca 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -10,7 +10,6 @@ from utils.graphene.fields import DjangoPaginatedListObjectField, DjangoListField from user_resource.schema import UserResourceMixin from deep.permissions import ProjectPermissions as PP -from deep.serializers import URLCachedFileField from lead.models import Lead from user.schema import UserType @@ -155,6 +154,7 @@ def resolve_verified_by_count(root, info, **_): return info.context.dl.entry.verified_by_count.load(root.pk) @staticmethod + # NOTE: Client might not need this field so we have not refactor the dataloader def resolve_canonical_preview_image(root, info, **_): return info.context.dl.entry.entry_image_preview_url.load(root.pk) diff --git a/apps/lead/views.py b/apps/lead/views.py index d56e8428b8..339563cb9d 100644 --- a/apps/lead/views.py +++ b/apps/lead/views.py @@ -433,8 +433,10 @@ def post(self, request, version=None): # Dynamic Options 'lead_groups': LeadGroup.objects.filter(project_filter, id__in=lead_groups_id).distinct(), - 'members': _filter_users_by_projects_memberships(members_qs, projects)\ - .prefetch_related('profile').distinct(), + 'members': _filter_users_by_projects_memberships( + members_qs, + projects, + ).prefetch_related('profile').distinct(), 'organizations': Organization.objects.filter(id__in=organizations_id).distinct(), # EMM specific options diff --git a/apps/unified_connector/migrations/0009_auto_20240618_0924.py b/apps/unified_connector/migrations/0009_auto_20240618_0924.py new file mode 100644 index 0000000000..87e58dc174 --- /dev/null +++ b/apps/unified_connector/migrations/0009_auto_20240618_0924.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.25 on 2024-06-18 09:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('unified_connector', '0008_connectorlead_text_extraction_id'), + ] + + operations = [ + migrations.CreateModel( + name='ConnectorLeadPreviewAttachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.IntegerField(default=0)), + ('page_number', models.IntegerField(default=0)), + ('type', models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1)), + ('file', models.FileField(upload_to='connector-lead/attachments/')), + ('file_preview', models.FileField(upload_to='connector-lead/attachments-preview/')), + ('connector_lead', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='preview_images', to='unified_connector.connectorlead')), + ], + ), + migrations.DeleteModel( + name='ConnectorLeadPreviewImage', + ), + ] diff --git a/apps/unified_connector/models.py b/apps/unified_connector/models.py index efb04a7293..0f0485d6c1 100644 --- a/apps/unified_connector/models.py +++ b/apps/unified_connector/models.py @@ -49,7 +49,7 @@ class ExtractionStatus(models.IntegerChoices): ) def __init__(self, *args, **kwargs): - self.preview_images: models.QuerySet[ConnectorLeadPreviewImage] + self.preview_images: models.QuerySet[ConnectorLeadPreviewAttachment] super().__init__(*args, **kwargs) @classmethod @@ -78,9 +78,20 @@ def update_extraction_status(self, new_status, commit=True): self.save(update_fields=('extraction_status',)) -class ConnectorLeadPreviewImage(models.Model): +class ConnectorLeadPreviewAttachment(models.Model): + class ConnectorAttachmentFileType(models.IntegerChoices): + XLSX = 1, 'XLSX' + IMAGE = 2, 'Image' + connector_lead = models.ForeignKey(ConnectorLead, on_delete=models.CASCADE, related_name='preview_images') - image = models.FileField(upload_to='connector-lead/preview-images/', max_length=255) + order = models.IntegerField(default=0) + page_number = models.IntegerField(default=0) + type = models.PositiveSmallIntegerField( + choices=ConnectorAttachmentFileType.choices, + default=ConnectorAttachmentFileType.XLSX + ) + file = models.FileField(upload_to='connector-lead/attachments/') + file_preview = models.FileField(upload_to='connector-lead/attachments-preview/') class UnifiedConnector(UserResource): diff --git a/apps/unified_connector/tests/test_mutation.py b/apps/unified_connector/tests/test_mutation.py index 8d36c0ae54..6ce6f1edfd 100644 --- a/apps/unified_connector/tests/test_mutation.py +++ b/apps/unified_connector/tests/test_mutation.py @@ -12,7 +12,7 @@ from unified_connector.models import ( ConnectorLead, ConnectorSource, - ConnectorLeadPreviewImage, + ConnectorLeadPreviewAttachment ) from deepl_integration.handlers import UnifiedConnectorLeadHandler from deepl_integration.serializers import DeeplServerBaseCallbackSerializer @@ -492,7 +492,23 @@ def _check_connector_lead_status(connector_lead, status): # ------ Extraction FAILED data = dict( client_id='some-random-client-id', - images_path=['https://example.com/sample-file-1.jpg'], + images_path=[ + { + 'page_number': 1, + 'images': [ + 'http://random.com/image1.jpeg', + 'http://random.com/image2.jpeg' + ], + } + ], + tables_path=[ + { + "page_number": 1, + "order": 0, + "image_link": "http://random.com/timetable.png", + "content_link": "http://random.com/table_timetable.xlsx" + } + ], text_path='https://example.com/url-where-data-is-fetched-from-mock-response', total_words_count=100, total_pages=10, @@ -520,7 +536,16 @@ def _check_connector_lead_status(connector_lead, status): # ------ Extraction SUCCESS data = dict( client_id='some-random-client-id', - images_path=['https://example.com/sample-file-1.jpg', 'https://example.com/sample-file-2.jpg'], + images_path=[ + { + 'page_number': 1, + 'images': [ + 'http://random.com/image1.jpeg', + 'http://random.com/image2.jpeg' + ], + } + ], + tables_path=[], text_path='https://example.com/url-where-data-is-fetched-from-mock-response', total_words_count=100, total_pages=10, @@ -542,8 +567,8 @@ def _check_connector_lead_status(connector_lead, status): assert connector_lead2.page_count == 10 _check_connector_lead_status(connector_lead2, ConnectorLead.ExtractionStatus.SUCCESS) - preview_image_qs = ConnectorLeadPreviewImage.objects.filter(connector_lead=connector_lead2) - preview_image = preview_image_qs.first() + preview_attachment_qs = ConnectorLeadPreviewAttachment.objects.filter(connector_lead=connector_lead2) + preview_attachment = preview_attachment_qs.first() self.assertEqual(connector_lead2.simplified_text, SAMPLE_SIMPLIFIED_TEXT) - self.assertEqual(preview_image_qs.count(), 2) - self.assertIsNotNone(preview_image and preview_image.image.name) + self.assertEqual(preview_attachment_qs.count(), 2) + self.assertIsNotNone(preview_attachment and preview_attachment.file) diff --git a/utils/graphene/mutation.py b/utils/graphene/mutation.py index aa5f1bb99c..a90b7162b5 100644 --- a/utils/graphene/mutation.py +++ b/utils/graphene/mutation.py @@ -176,10 +176,10 @@ def fields_for_serializer( is_excluded = any( [ name in exclude_fields, - field.write_only and - not is_input, # don't show write_only fields in Query - field.read_only and is_input \ - and lookup_field != name, # don't show read_only fields in Input + # don't show write_only fields in Query + field.write_only and not is_input, + # don't show read_only fields in Input + field.read_only and is_input and lookup_field != name, ] ) From bb15587ea604829b27c01974174937bd20e4c550 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 20 Jun 2024 11:39:40 +0545 Subject: [PATCH 10/17] Remove canonical_preview_image from entry schema --- apps/entry/dataloaders.py | 26 -------------------------- apps/entry/schema.py | 6 ------ schema.graphql | 1 - 3 files changed, 33 deletions(-) diff --git a/apps/entry/dataloaders.py b/apps/entry/dataloaders.py index 1dd0df1816..d24f3e6317 100644 --- a/apps/entry/dataloaders.py +++ b/apps/entry/dataloaders.py @@ -1,5 +1,4 @@ from collections import defaultdict -from deep.serializers import URLCachedFileField from promise import Promise from django.utils.functional import cached_property from django.db import models @@ -120,27 +119,6 @@ def batch_load_fn(self, keys): return Promise.resolve([counts.get(key, 0) for key in keys]) -class EntryImageUrlLoader(DataLoaderWithContext): - def batch_load_fn(self, keys): - entry_qs = Entry.objects.filter(id__in=keys) - entry_dict = {entry.id: entry for entry in entry_qs} - results = [] - for key in keys: - entry = entry_dict.get(key) - if entry.entry_type == Entry.TagType.IMAGE: - url = self.context.request.build_absolute_uri( - URLCachedFileField.name_to_representation(entry.image) - ) - elif entry.entry_type == Entry.TagType.ATTACHMENT: - url = self.context.request.build_absolute_uri( - URLCachedFileField.name_to_representation(entry.entry_attachment.file) - ) - else: - url = None - results.append(url) - return Promise.resolve(results) - - class DataLoaders(WithContextMixin): @cached_property def entry(self): @@ -173,7 +151,3 @@ def verified_by(self): @cached_property def verified_by_count(self): return EntryVerifiedByCountLoader(context=self.context) - - @cached_property - def entry_image_preview_url(self): - return EntryImageUrlLoader(context=self.context) diff --git a/apps/entry/schema.py b/apps/entry/schema.py index 34f43282ca..b11e995194 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -118,7 +118,6 @@ class Meta: review_comments_count = graphene.Int(required=True) draft_entry = graphene.ID(source="draft_entry_id") entry_attachment = graphene.Field(EntryAttachmentType, required=False) - canonical_preview_image = graphene.String(required=False) # project_labels TODO: # tabular_field TODO: @@ -153,11 +152,6 @@ def resolve_verified_by_count(root, info, **_): return len(root.verified_by.all()) return info.context.dl.entry.verified_by_count.load(root.pk) - @staticmethod - # NOTE: Client might not need this field so we have not refactor the dataloader - def resolve_canonical_preview_image(root, info, **_): - return info.context.dl.entry.entry_image_preview_url.load(root.pk) - class EntryListType(CustomDjangoListObjectType): class Meta: diff --git a/schema.graphql b/schema.graphql index 9e264da86b..3aa54dd23a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -4103,7 +4103,6 @@ type EntryType { reviewCommentsCount: Int! draftEntry: ID entryAttachment: EntryAttachmentType - canonicalPreviewImage: String } scalar EnumDescription From 288cca37daaccbdc280f1bc785077d5425683aaf Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 25 Jun 2024 13:54:27 +0545 Subject: [PATCH 11/17] Update migrations of connector-lead , lead, entry --- apps/deepl_integration/handlers.py | 1 + apps/deepl_integration/serializers.py | 2 +- ...611_0810.py => 0041_auto_20240621_1149.py} | 4 +- apps/entry/models.py | 3 ++ .../migrations/0050_auto_20240606_0608.py | 29 ------------ .../migrations/0050_auto_20240621_1149.py | 17 +++++++ .../0051_alter_leadpreviewattachment_type.py | 18 -------- .../migrations/0051_auto_20240625_0509.py | 45 +++++++++++++++++++ .../0052_alter_leadpreviewattachment_type.py | 18 -------- .../0053_alter_leadpreviewattachment_type.py | 18 -------- .../migrations/0009_auto_20240618_0924.py | 29 ------------ .../migrations/0009_auto_20240621_1149.py | 27 +++++++++++ .../migrations/0010_auto_20240625_0806.py | 44 ++++++++++++++++++ server.diff | 0 14 files changed, 140 insertions(+), 115 deletions(-) rename apps/entry/migrations/{0041_auto_20240611_0810.py => 0041_auto_20240621_1149.py} (86%) delete mode 100644 apps/lead/migrations/0050_auto_20240606_0608.py create mode 100644 apps/lead/migrations/0050_auto_20240621_1149.py delete mode 100644 apps/lead/migrations/0051_alter_leadpreviewattachment_type.py create mode 100644 apps/lead/migrations/0051_auto_20240625_0509.py delete mode 100644 apps/lead/migrations/0052_alter_leadpreviewattachment_type.py delete mode 100644 apps/lead/migrations/0053_alter_leadpreviewattachment_type.py delete mode 100644 apps/unified_connector/migrations/0009_auto_20240618_0924.py create mode 100644 apps/unified_connector/migrations/0009_auto_20240621_1149.py create mode 100644 apps/unified_connector/migrations/0010_auto_20240625_0806.py create mode 100644 server.diff diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index c64a37c2d5..d3f0c0e4f9 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -736,6 +736,7 @@ def save_lead_data_using_connector_lead( connector_lead_attachment.file_preview.name, connector_lead_attachment.file_preview ) + lead_attachment.save() lead.update_extraction_status(Lead.ExtractionStatus.SUCCESS) return True diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 3915bea38f..a2fe470cbd 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -125,7 +125,7 @@ def create(self, data: List[Dict]): lead, data['text_path'], data.get('images_path', [])[:10], # TODO: Support for more images, too much image will error. - data.get('tables_path', []), + data.get('tables_path', [])[:10], data.get('total_words_count'), data.get('total_pages'), data.get('text_extraction_id'), diff --git a/apps/entry/migrations/0041_auto_20240611_0810.py b/apps/entry/migrations/0041_auto_20240621_1149.py similarity index 86% rename from apps/entry/migrations/0041_auto_20240611_0810.py rename to apps/entry/migrations/0041_auto_20240621_1149.py index b57e395c17..b296b52be9 100644 --- a/apps/entry/migrations/0041_auto_20240611_0810.py +++ b/apps/entry/migrations/0041_auto_20240621_1149.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.25 on 2024-06-11 08:10 +# Generated by Django 3.2.25 on 2024-06-21 11:49 from django.db import migrations, models import django.db.models.deletion @@ -7,8 +7,8 @@ class Migration(migrations.Migration): dependencies = [ - ('lead', '0053_alter_leadpreviewattachment_type'), ('entry', '0040_alter_entryattachment_entry_file_type'), + ('lead', '0051_auto_20240625_0509') ] operations = [ diff --git a/apps/entry/models.py b/apps/entry/models.py index 7df0da4b2c..4ae3209e22 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -5,6 +5,7 @@ from django.db import models from deep.middleware import get_current_user +from unified_connector.models import ConnectorLeadPreviewAttachment from utils.common import parse_number from project.mixins import ProjectEntityMixin from project.permissions import PROJECT_PERMISSIONS @@ -31,6 +32,8 @@ class EntryFileType(models.IntegerChoices): LEAD_TO_ENTRY_TYPE = { LeadPreviewAttachment.AttachmentFileType.XLSX: EntryFileType.XLSX, LeadPreviewAttachment.AttachmentFileType.IMAGE: EntryFileType.IMAGE, + ConnectorLeadPreviewAttachment.ConnectorAttachmentFileType.XLSX: EntryFileType.XLSX, + ConnectorLeadPreviewAttachment.ConnectorAttachmentFileType.IMAGE: EntryFileType.IMAGE } assert len(list(LeadPreviewAttachment.AttachmentFileType)) == len(LEAD_TO_ENTRY_TYPE.keys()), \ 'Make sure to sync LEAD_TO_ENTRY_TYPE with LeadPreviewAttachment.AttachmentFileType' diff --git a/apps/lead/migrations/0050_auto_20240606_0608.py b/apps/lead/migrations/0050_auto_20240606_0608.py deleted file mode 100644 index bf17c0883c..0000000000 --- a/apps/lead/migrations/0050_auto_20240606_0608.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-06 06:08 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('lead', '0049_auto_20231121_0926_squashed_0054_auto_20231218_0552'), - ] - - operations = [ - migrations.CreateModel( - name='LeadPreviewAttachment', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.IntegerField(default=0)), - ('page_number', models.IntegerField(default=0)), - ('type', models.CharField(choices=[('XLSX', 'XLSX'), ('image', 'Image')], max_length=20)), - ('file', models.FileField(upload_to='lead-preview/attachments/')), - ('file_preview', models.FileField(upload_to='lead-preview/attachments-preview/')), - ('lead', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to='lead.lead')), - ], - ), - migrations.DeleteModel( - name='LeadPreviewImage', - ), - ] diff --git a/apps/lead/migrations/0050_auto_20240621_1149.py b/apps/lead/migrations/0050_auto_20240621_1149.py new file mode 100644 index 0000000000..64fb0c238e --- /dev/null +++ b/apps/lead/migrations/0050_auto_20240621_1149.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2024-06-21 11:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0049_auto_20231121_0926_squashed_0054_auto_20231218_0552'), + ] + + operations = [ + migrations.RenameModel( + old_name='LeadPreviewImage', + new_name='LeadPreviewAttachment' + ), + ] diff --git a/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py deleted file mode 100644 index bfa87741c4..0000000000 --- a/apps/lead/migrations/0051_alter_leadpreviewattachment_type.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-07 06:42 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('lead', '0050_auto_20240606_0608'), - ] - - operations = [ - migrations.AlterField( - model_name='leadpreviewattachment', - name='type', - field=models.SmallIntegerField(choices=[('1', 'XLSX'), ('2', 'Image')]), - ), - ] diff --git a/apps/lead/migrations/0051_auto_20240625_0509.py b/apps/lead/migrations/0051_auto_20240625_0509.py new file mode 100644 index 0000000000..ba958eabe0 --- /dev/null +++ b/apps/lead/migrations/0051_auto_20240625_0509.py @@ -0,0 +1,45 @@ +# Generated by Django 3.2.25 on 2024-06-25 05:09 + +from django.db import migrations, models + + +def set_file_preview(apps, schema_editor): + LeadPreviewAttachment = apps.get_model('lead', 'LeadPreviewAttachment') + lead_attachments = LeadPreviewAttachment.objects.all() + for lead_attachment in lead_attachments: + lead_attachment.file_preview = lead_attachment.file + lead_attachment.type = 2 # default type is image + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0050_auto_20240621_1149'), + ] + + operations = [ + migrations.AddField( + model_name='leadpreviewattachment', + name='order', + field=models.IntegerField(default=0) + ), + migrations.AddField( + model_name='leadpreviewattachment', + name='page_number', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='leadpreviewattachment', + name='file_preview', + field=models.FileField(upload_to='lead-preview/attachments-preview/') + ), + migrations.AddField( + model_name='leadpreviewattachment', + name='type', + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')],max_length=20) + ), + migrations.RunPython( + set_file_preview, + reverse_code=migrations.RunPython.noop, + ) + ] diff --git a/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py deleted file mode 100644 index 0185485a73..0000000000 --- a/apps/lead/migrations/0052_alter_leadpreviewattachment_type.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-07 08:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('lead', '0051_alter_leadpreviewattachment_type'), - ] - - operations = [ - migrations.AlterField( - model_name='leadpreviewattachment', - name='type', - field=models.PositiveSmallIntegerField(choices=[('1', 'XLSX'), ('2', 'Image')], default='1'), - ), - ] diff --git a/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py b/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py deleted file mode 100644 index a47c92bd98..0000000000 --- a/apps/lead/migrations/0053_alter_leadpreviewattachment_type.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-09 10:27 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('lead', '0052_alter_leadpreviewattachment_type'), - ] - - operations = [ - migrations.AlterField( - model_name='leadpreviewattachment', - name='type', - field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1), - ), - ] diff --git a/apps/unified_connector/migrations/0009_auto_20240618_0924.py b/apps/unified_connector/migrations/0009_auto_20240618_0924.py deleted file mode 100644 index 87e58dc174..0000000000 --- a/apps/unified_connector/migrations/0009_auto_20240618_0924.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-18 09:24 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('unified_connector', '0008_connectorlead_text_extraction_id'), - ] - - operations = [ - migrations.CreateModel( - name='ConnectorLeadPreviewAttachment', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.IntegerField(default=0)), - ('page_number', models.IntegerField(default=0)), - ('type', models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1)), - ('file', models.FileField(upload_to='connector-lead/attachments/')), - ('file_preview', models.FileField(upload_to='connector-lead/attachments-preview/')), - ('connector_lead', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='preview_images', to='unified_connector.connectorlead')), - ], - ), - migrations.DeleteModel( - name='ConnectorLeadPreviewImage', - ), - ] diff --git a/apps/unified_connector/migrations/0009_auto_20240621_1149.py b/apps/unified_connector/migrations/0009_auto_20240621_1149.py new file mode 100644 index 0000000000..23a41939c7 --- /dev/null +++ b/apps/unified_connector/migrations/0009_auto_20240621_1149.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-06-21 11:49 + +from django.db import migrations,models + + +class Migration(migrations.Migration): + + dependencies = [ + ('unified_connector', '0008_connectorlead_text_extraction_id'), + ] + + operations = [ + migrations.RenameField( + model_name='connectorleadpreviewimage', + old_name='image', + new_name='file', + ), + migrations.AlterField( + model_name='connectorleadpreviewimage', + name='file', + field=models.FileField(upload_to='connector-lead/attachments/') + ), + migrations.RenameModel( + old_name='ConnectorLeadPreviewImage', + new_name='ConnectorLeadPreviewAttachment' + ), + ] diff --git a/apps/unified_connector/migrations/0010_auto_20240625_0806.py b/apps/unified_connector/migrations/0010_auto_20240625_0806.py new file mode 100644 index 0000000000..bd1a4ef1dc --- /dev/null +++ b/apps/unified_connector/migrations/0010_auto_20240625_0806.py @@ -0,0 +1,44 @@ +# Generated by Django 3.2.25 on 2024-06-25 08:06 + +from django.db import migrations, models + +def set_file_preview(apps, schema_editor): + ConnectorLeadPreviewAttachment = apps.get_model('unified_connector', 'ConnectorLeadPreviewAttachment') + connector_lead_attachments = ConnectorLeadPreviewAttachment.objects.all() + for connector_lead_attachment in connector_lead_attachments: + connector_lead_attachment.file_preview = connector_lead_attachment.file + connector_lead_attachment.type = 2 # default type is image + + +class Migration(migrations.Migration): + + dependencies = [ + ('unified_connector', '0009_auto_20240621_1149'), + ] + + operations = [ + migrations.AddField( + model_name='connectorleadpreviewattachment', + name='order', + field=models.IntegerField(default=0) + ), + migrations.AddField( + model_name='connectorleadpreviewattachment', + name='page_number', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='connectorleadpreviewattachment', + name='file_preview', + field=models.FileField(upload_to='connector-lead/attachments-preview/') + ), + migrations.AddField( + model_name='connectorleadpreviewattachment', + name='type', + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')]) + ), + migrations.RunPython( + set_file_preview, + reverse_code=migrations.RunPython.noop, + ) + ] diff --git a/server.diff b/server.diff new file mode 100644 index 0000000000..e69de29bb2 From 0350cdce149f78ddf5056a9dddc3196a186fbd2d Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 27 Jun 2024 12:24:34 +0545 Subject: [PATCH 12/17] - Refactor migrations --- apps/lead/migrations/0050_auto_20240621_1149.py | 7 ++++++- apps/lead/migrations/0051_auto_20240625_0509.py | 3 ++- .../migrations/0010_auto_20240625_0806.py | 3 ++- server.diff | 0 4 files changed, 10 insertions(+), 3 deletions(-) delete mode 100644 server.diff diff --git a/apps/lead/migrations/0050_auto_20240621_1149.py b/apps/lead/migrations/0050_auto_20240621_1149.py index 64fb0c238e..174b45d6ac 100644 --- a/apps/lead/migrations/0050_auto_20240621_1149.py +++ b/apps/lead/migrations/0050_auto_20240621_1149.py @@ -1,6 +1,6 @@ # Generated by Django 3.2.25 on 2024-06-21 11:49 -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -10,6 +10,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AlterField( + model_name='leadpreviewimage', + name='file', + field=models.FileField(upload_to='lead-preview/attachments/') + ), migrations.RenameModel( old_name='LeadPreviewImage', new_name='LeadPreviewAttachment' diff --git a/apps/lead/migrations/0051_auto_20240625_0509.py b/apps/lead/migrations/0051_auto_20240625_0509.py index ba958eabe0..cece84578e 100644 --- a/apps/lead/migrations/0051_auto_20240625_0509.py +++ b/apps/lead/migrations/0051_auto_20240625_0509.py @@ -9,6 +9,7 @@ def set_file_preview(apps, schema_editor): for lead_attachment in lead_attachments: lead_attachment.file_preview = lead_attachment.file lead_attachment.type = 2 # default type is image + lead_attachment.save(updated_fields=['file_preview','type']) class Migration(migrations.Migration): @@ -36,7 +37,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='leadpreviewattachment', name='type', - field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')],max_length=20) + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1) ), migrations.RunPython( set_file_preview, diff --git a/apps/unified_connector/migrations/0010_auto_20240625_0806.py b/apps/unified_connector/migrations/0010_auto_20240625_0806.py index bd1a4ef1dc..fae4142a9d 100644 --- a/apps/unified_connector/migrations/0010_auto_20240625_0806.py +++ b/apps/unified_connector/migrations/0010_auto_20240625_0806.py @@ -8,6 +8,7 @@ def set_file_preview(apps, schema_editor): for connector_lead_attachment in connector_lead_attachments: connector_lead_attachment.file_preview = connector_lead_attachment.file connector_lead_attachment.type = 2 # default type is image + connector_lead_attachment.save(updated_fields=['file_preview','type']) class Migration(migrations.Migration): @@ -35,7 +36,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='connectorleadpreviewattachment', name='type', - field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')]) + field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1) ), migrations.RunPython( set_file_preview, diff --git a/server.diff b/server.diff deleted file mode 100644 index e69de29bb2..0000000000 From 925f3730562314eaef0d1b59bea0ae1f63241352 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 2 Jul 2024 11:17:05 +0545 Subject: [PATCH 13/17] - Update migrations --- .../migrations/0050_auto_20240621_1149.py | 6 ++++++ .../migrations/0051_auto_20240625_0509.py | 21 +++++++++---------- .../migrations/0010_auto_20240625_0806.py | 10 ++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/apps/lead/migrations/0050_auto_20240621_1149.py b/apps/lead/migrations/0050_auto_20240621_1149.py index 174b45d6ac..4c710310de 100644 --- a/apps/lead/migrations/0050_auto_20240621_1149.py +++ b/apps/lead/migrations/0050_auto_20240621_1149.py @@ -19,4 +19,10 @@ class Migration(migrations.Migration): old_name='LeadPreviewImage', new_name='LeadPreviewAttachment' ), + + migrations.AddField( + model_name='leadpreviewattachment', + name='file_preview', + field=models.FileField(upload_to='lead-preview/attachments-preview/', null=True) + ), ] diff --git a/apps/lead/migrations/0051_auto_20240625_0509.py b/apps/lead/migrations/0051_auto_20240625_0509.py index cece84578e..943ace7d92 100644 --- a/apps/lead/migrations/0051_auto_20240625_0509.py +++ b/apps/lead/migrations/0051_auto_20240625_0509.py @@ -5,11 +5,10 @@ def set_file_preview(apps, schema_editor): LeadPreviewAttachment = apps.get_model('lead', 'LeadPreviewAttachment') - lead_attachments = LeadPreviewAttachment.objects.all() - for lead_attachment in lead_attachments: - lead_attachment.file_preview = lead_attachment.file - lead_attachment.type = 2 # default type is image - lead_attachment.save(updated_fields=['file_preview','type']) + LeadPreviewAttachment.objects.update( + file_preview=models.F('file'), + type=2, + ) class Migration(migrations.Migration): @@ -29,11 +28,6 @@ class Migration(migrations.Migration): name='page_number', field=models.IntegerField(default=0), ), - migrations.AddField( - model_name='leadpreviewattachment', - name='file_preview', - field=models.FileField(upload_to='lead-preview/attachments-preview/') - ), migrations.AddField( model_name='leadpreviewattachment', name='type', @@ -42,5 +36,10 @@ class Migration(migrations.Migration): migrations.RunPython( set_file_preview, reverse_code=migrations.RunPython.noop, - ) + ), + migrations.AlterField( + model_name='leadpreviewattachment', + name='file_preview', + field=models.FileField(upload_to='lead-preview/attachments-preview/') + ), ] diff --git a/apps/unified_connector/migrations/0010_auto_20240625_0806.py b/apps/unified_connector/migrations/0010_auto_20240625_0806.py index fae4142a9d..d51c840097 100644 --- a/apps/unified_connector/migrations/0010_auto_20240625_0806.py +++ b/apps/unified_connector/migrations/0010_auto_20240625_0806.py @@ -2,13 +2,13 @@ from django.db import migrations, models + def set_file_preview(apps, schema_editor): ConnectorLeadPreviewAttachment = apps.get_model('unified_connector', 'ConnectorLeadPreviewAttachment') - connector_lead_attachments = ConnectorLeadPreviewAttachment.objects.all() - for connector_lead_attachment in connector_lead_attachments: - connector_lead_attachment.file_preview = connector_lead_attachment.file - connector_lead_attachment.type = 2 # default type is image - connector_lead_attachment.save(updated_fields=['file_preview','type']) + ConnectorLeadPreviewAttachment.objects.update( + file_preview=models.F('file'), + type=2, + ) class Migration(migrations.Migration): From 7b03c64837c400ca0c5951963b6c6669db1d6794 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 5 Jul 2024 11:14:00 +0545 Subject: [PATCH 14/17] Add Export in ocr integration --- apps/entry/models.py | 18 ++++++++++++++++++ apps/export/entries/excel_exporter.py | 2 ++ apps/export/entries/report_exporter.py | 2 ++ apps/export/models.py | 1 + apps/gallery/enums.py | 6 ++++++ apps/gallery/views.py | 21 ++++++++++++++++++++- deep/urls.py | 6 ++++++ schema.graphql | 1 + 8 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 apps/gallery/enums.py diff --git a/apps/entry/models.py b/apps/entry/models.py index 4ae3209e22..bf7b8a1a9d 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -3,6 +3,10 @@ from django.contrib.postgres.aggregates.general import ArrayAgg from django.contrib.postgres.fields import ArrayField from django.db import models +from django.conf import settings +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode from deep.middleware import get_current_user from unified_connector.models import ConnectorLeadPreviewAttachment @@ -21,6 +25,7 @@ Exportable, ) from assisted_tagging.models import DraftEntry +from gallery.enums import ModuleTypeEnum class EntryAttachment(models.Model): @@ -66,6 +71,19 @@ def clone_from_lead_attachment(cls, lead_attachment: LeadPreviewAttachment) -> ' entry_attachment.save() return entry_attachment + def get_file_url(self): + return '{protocol}://{domain}{url}'.format( + protocol=settings.HTTP_PROTOCOL, + domain=settings.DJANGO_API_HOST, + url=reverse( + 'external_private_url', + kwargs={ + 'module': ModuleTypeEnum.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.id)) + } + ) + ) + class Entry(UserResource, ProjectEntityMixin): """ diff --git a/apps/export/entries/excel_exporter.py b/apps/export/entries/excel_exporter.py index ec8d0452a8..649241311f 100644 --- a/apps/export/entries/excel_exporter.py +++ b/apps/export/entries/excel_exporter.py @@ -272,6 +272,8 @@ def add_entries_from_excel_data_for_static_column( if self.modified_excerpt_exists: return [entry_excerpt, entry.dropped_excerpt] return entry_excerpt + elif exportable == Export.StaticColumn.LEAD_ENTRY_ENTRY_ATTACHMENT_FILE_PREVIEW: + return f'{entry.entry_attachment.get_file_url()}' def add_entries_from_excel_data(self, rows, data, export_data): export_type = data.get('type') diff --git a/apps/export/entries/report_exporter.py b/apps/export/entries/report_exporter.py index 745919a6bd..f49a63df68 100644 --- a/apps/export/entries/report_exporter.py +++ b/apps/export/entries/report_exporter.py @@ -592,6 +592,8 @@ def _generate_for_entry(self, entry): if h_stats.get(key): image_text += f', {key.title()} values: {h_stats.get(key)}' if h_stats.get(key) else '' + if entry.entry_type == Entry.TagType.ATTACHMENT: + image = entry.entry_attachment.file_preview if image: self.doc.add_image(image) if image_text: diff --git a/apps/export/models.py b/apps/export/models.py index 25bf7680dc..925748d3cf 100644 --- a/apps/export/models.py +++ b/apps/export/models.py @@ -146,6 +146,7 @@ class StaticColumn(models.TextChoices): ENTRY_ID = 'entry_id', 'Entry Id' LEAD_ENTRY_ID = 'lead_entry_id', 'Source-Entry Id' ENTRY_EXCERPT = 'entry_excerpt', 'Modified Excerpt, Original Excerpt' + LEAD_ENTRY_ENTRY_ATTACHMENT_FILE_PREVIEW = 'lead_entry_entry_attachment_file_preview', 'EntryAttachment Url' # Used by extra options for Report class CitationStyle(models.IntegerChoices): diff --git a/apps/gallery/enums.py b/apps/gallery/enums.py new file mode 100644 index 0000000000..9ac6b501c7 --- /dev/null +++ b/apps/gallery/enums.py @@ -0,0 +1,6 @@ +import graphene + + +class ModuleTypeEnum(graphene.Enum): + ENTRY_ATTACHMENT = 'entry-attachment' + LEAD_PREVIEW_ATTACHMENT = 'lead-preview-attachment' diff --git a/apps/gallery/views.py b/apps/gallery/views.py index 79d8b21439..1977e83beb 100644 --- a/apps/gallery/views.py +++ b/apps/gallery/views.py @@ -7,6 +7,7 @@ from django.utils.http import urlsafe_base64_decode from django.shortcuts import redirect, get_object_or_404 +from gallery.enums import ModuleTypeEnum from rest_framework import ( views, viewsets, @@ -24,7 +25,7 @@ from deep.permalinks import Permalink from project.models import Project from lead.models import Lead -from entry.models import Entry +from entry.models import Entry, EntryAttachment from user_resource.filters import UserResourceFilterSet from utils.extractor.formats import ( @@ -82,6 +83,24 @@ def get(self, request, uuid=None, filename=None): ) +class AttachmentFileView(views.APIView): + permission_classes = [permissions.IsAuthenticated] + + def get(self, request, module=None, identifier=None): + + if module == ModuleTypeEnum.ENTRY_ATTACHMENT.value: + id = force_text(urlsafe_base64_decode(identifier)) + qs = get_object_or_404(EntryAttachment, id=id) + if qs: + return redirect(request.build_absolute_uri(qs.file.url)) + return response.Response({ + 'error': 'File doesn\'t exists', + }, status=status.HTTP_404_NOT_FOUND) + return response.Response({ + 'error': 'Access Forbidden, Contact Admin', + }, status=status.HTTP_403_FORBIDDEN) + + class DeprecatedPrivateFileView(views.APIView): permission_classes = [permissions.IsAuthenticated] diff --git a/deep/urls.py b/deep/urls.py index 88b1f0ca7b..b63c873fb6 100644 --- a/deep/urls.py +++ b/deep/urls.py @@ -26,6 +26,7 @@ unsubscribe_email, ) from gallery.views import ( + AttachmentFileView, FileView, FileViewSet, GoogleDriveFileViewSet, @@ -435,6 +436,11 @@ def get_api_path(path): DeprecatedPrivateFileView.as_view(), name='deprecated_gallery_private_url', ), + path( + 'external/private-file//', + AttachmentFileView.as_view(), + name='external_private_url', + ), re_path( r'^public-file/(?P[0-9A-Za-z]+)/(?P.+)/(?P.*)$', PublicFileView.as_view(), diff --git a/schema.graphql b/schema.graphql index 3aa54dd23a..420a742f55 100644 --- a/schema.graphql +++ b/schema.graphql @@ -4263,6 +4263,7 @@ enum ExportExcelSelectedStaticColumnEnum { ENTRY_ID LEAD_ENTRY_ID ENTRY_EXCERPT + LEAD_ENTRY_ENTRY_ATTACHMENT_FILE_PREVIEW } enum ExportExportTypeEnum { From 8116ede275c35a4b7888afdfc9b8aafa717ab452 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 5 Jul 2024 15:58:05 +0545 Subject: [PATCH 15/17] Add testcases --- apps/entry/models.py | 4 +- apps/entry/schema.py | 2 +- apps/export/entries/excel_exporter.py | 4 +- apps/gallery/enums.py | 2 +- apps/gallery/tests/test_apis.py | 41 ++++++++++++++++++- apps/gallery/utils.py | 18 ++++++++ apps/gallery/views.py | 23 +++++------ .../migrations/0051_auto_20240625_0509.py | 2 +- apps/lead/schema.py | 17 ++++++++ .../migrations/0010_auto_20240625_0806.py | 2 +- deep/urls.py | 4 +- schema.graphql | 2 +- 12 files changed, 98 insertions(+), 23 deletions(-) create mode 100644 apps/gallery/utils.py diff --git a/apps/entry/models.py b/apps/entry/models.py index bf7b8a1a9d..c108e4c208 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -25,7 +25,7 @@ Exportable, ) from assisted_tagging.models import DraftEntry -from gallery.enums import ModuleTypeEnum +from gallery.enums import PrivateFileModuleType class EntryAttachment(models.Model): @@ -78,7 +78,7 @@ def get_file_url(self): url=reverse( 'external_private_url', kwargs={ - 'module': ModuleTypeEnum.ENTRY_ATTACHMENT.value, + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, 'identifier': urlsafe_base64_encode(force_bytes(self.id)) } ) diff --git a/apps/entry/schema.py b/apps/entry/schema.py index b11e995194..48631f79ae 100644 --- a/apps/entry/schema.py +++ b/apps/entry/schema.py @@ -86,7 +86,7 @@ def resolve_geo_selected_options(root, info, **_): class EntryAttachmentType(DjangoObjectType): - lead_attachment_id = graphene.ID(required=True) + lead_attachment_id = graphene.ID(required=False) file = graphene.Field(FileFieldType, required=True) file_preview = graphene.Field(FileFieldType, required=True) entry_file_type = graphene.Field(EntryAttachmentTypeEnum, required=True) diff --git a/apps/export/entries/excel_exporter.py b/apps/export/entries/excel_exporter.py index 649241311f..6135239c3a 100644 --- a/apps/export/entries/excel_exporter.py +++ b/apps/export/entries/excel_exporter.py @@ -15,6 +15,8 @@ from lead.models import Lead from export.models import Export +from gallery.utils import get_private_file_url +from gallery.enums import PrivateFileModuleType logger = logging.getLogger(__name__) @@ -273,7 +275,7 @@ def add_entries_from_excel_data_for_static_column( return [entry_excerpt, entry.dropped_excerpt] return entry_excerpt elif exportable == Export.StaticColumn.LEAD_ENTRY_ENTRY_ATTACHMENT_FILE_PREVIEW: - return f'{entry.entry_attachment.get_file_url()}' + return get_private_file_url(PrivateFileModuleType.ENTRY_ATTACHMENT.value, entry.entry_attachment.id) def add_entries_from_excel_data(self, rows, data, export_data): export_type = data.get('type') diff --git a/apps/gallery/enums.py b/apps/gallery/enums.py index 9ac6b501c7..ecb8f46dca 100644 --- a/apps/gallery/enums.py +++ b/apps/gallery/enums.py @@ -1,6 +1,6 @@ import graphene -class ModuleTypeEnum(graphene.Enum): +class PrivateFileModuleType(graphene.Enum): ENTRY_ATTACHMENT = 'entry-attachment' LEAD_PREVIEW_ATTACHMENT = 'lead-preview-attachment' diff --git a/apps/gallery/tests/test_apis.py b/apps/gallery/tests/test_apis.py index 167849e166..8fe6cd7e7d 100644 --- a/apps/gallery/tests/test_apis.py +++ b/apps/gallery/tests/test_apis.py @@ -6,11 +6,15 @@ from django.utils.http import urlsafe_base64_encode from django.utils.encoding import force_bytes +from rest_framework import status + from deep.tests import TestCase from gallery.models import File, FilePreview from lead.models import Lead from project.models import Project -from entry.models import Entry +from entry.models import Entry, EntryAttachment +from user.models import User +from gallery.enums import PrivateFileModuleType class GalleryTests(TestCase): @@ -205,3 +209,38 @@ def save_file_with_api(self, kwargs={}): return response.data['id'] # NOTE: Test for files + + +class PrivateAttachmentFileViewTest(TestCase): + def setUp(self): + # Create a test user + self.user = User.objects.create_user(username='testuser', password='testpassword') + + # Create a test entry attachment + self.project = Project.objects.create() + self.lead = Lead.objects.create(project=Project) + self.attachment = EntryAttachment.objects.create() + self.entry = Entry.objects.create( + lead=self.lead, + project=self.project, + entry=self.attachment + ) + self.url = reverse('private_attachment_file_view', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.attachment.id)), + }) + + def test_access_by_authenticated_user(self): + self.authenticate() + response = self.client.get(self.url) + self.assert_200(response) + + def test_access_forbidden(self): + self.authenticate() + invalid_url = reverse('private_attachment_file_view', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(999999)), + }) + response = self.client.get(invalid_url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.data['error'], "Access Forbidden Or File doesn't exists, Contact Admin") diff --git a/apps/gallery/utils.py b/apps/gallery/utils.py new file mode 100644 index 0000000000..73887ef9e7 --- /dev/null +++ b/apps/gallery/utils.py @@ -0,0 +1,18 @@ +from django.conf import settings +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode + + +def get_private_file_url(obj, id): + return '{protocol}://{domain}{url}'.format( + protocol=settings.HTTP_PROTOCOL, + domain=settings.DJANGO_API_HOST, + url=reverse( + 'external_private_url', + kwargs={ + 'module': obj, + 'identifier': urlsafe_base64_encode(force_bytes(id)) + } + ) + ) diff --git a/apps/gallery/views.py b/apps/gallery/views.py index 1977e83beb..b506d9e48f 100644 --- a/apps/gallery/views.py +++ b/apps/gallery/views.py @@ -7,7 +7,7 @@ from django.utils.http import urlsafe_base64_decode from django.shortcuts import redirect, get_object_or_404 -from gallery.enums import ModuleTypeEnum +from gallery.enums import PrivateFileModuleType from rest_framework import ( views, viewsets, @@ -83,21 +83,20 @@ def get(self, request, uuid=None, filename=None): ) -class AttachmentFileView(views.APIView): +class PrivateAttachmentFileView(views.APIView): permission_classes = [permissions.IsAuthenticated] def get(self, request, module=None, identifier=None): - - if module == ModuleTypeEnum.ENTRY_ATTACHMENT.value: - id = force_text(urlsafe_base64_decode(identifier)) - qs = get_object_or_404(EntryAttachment, id=id) - if qs: - return redirect(request.build_absolute_uri(qs.file.url)) - return response.Response({ - 'error': 'File doesn\'t exists', - }, status=status.HTTP_404_NOT_FOUND) + id = force_text(urlsafe_base64_decode(identifier)) + user = request.user + obj = None + if module == PrivateFileModuleType.ENTRY_ATTACHMENT.value: + obj = get_object_or_404(EntryAttachment, id=id) + obj.entry.get_for(user) + if obj: + return redirect(request.build_absolute_uri(obj.file.url)) return response.Response({ - 'error': 'Access Forbidden, Contact Admin', + 'error': 'Access Forbidden Or File does\'t exists, Contact Admin', }, status=status.HTTP_403_FORBIDDEN) diff --git a/apps/lead/migrations/0051_auto_20240625_0509.py b/apps/lead/migrations/0051_auto_20240625_0509.py index 943ace7d92..bc79203bd6 100644 --- a/apps/lead/migrations/0051_auto_20240625_0509.py +++ b/apps/lead/migrations/0051_auto_20240625_0509.py @@ -7,7 +7,7 @@ def set_file_preview(apps, schema_editor): LeadPreviewAttachment = apps.get_model('lead', 'LeadPreviewAttachment') LeadPreviewAttachment.objects.update( file_preview=models.F('file'), - type=2, + type=2, # default type is Image ) diff --git a/apps/lead/schema.py b/apps/lead/schema.py index c0a83ec5f4..8d75d53e31 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -61,6 +61,15 @@ def get_lead_qs(info): return Lead.objects.none() +def get_lead_preview_attachment(info): + lead_attachment_qs = LeadPreviewAttachment.objects.filter( + lead__project=info.context.active_project + ).order_by('-page_number') + if PP.check_permission(info, PP.Permission.VIEW_ALL_LEAD): + return lead_attachment_qs + return LeadPreviewAttachment.objects.none() + + def get_lead_group_qs(info): lead_group_qs = LeadGroup.objects.filter(project=info.context.active_project) if PP.check_permission(info, PP.Permission.VIEW_ALL_LEAD): @@ -232,6 +241,10 @@ class Meta: 'page_number', ) + @staticmethod + def get_custom_queryset(queryset, info, **kwargs): + return get_lead_preview_attachment(info) + class LeadEmmTriggerType(DjangoObjectType): class Meta: @@ -523,6 +536,10 @@ class Query: def resolve_leads(root, info, **kwargs) -> QuerySet: return get_lead_qs(info) + @staticmethod + def resolve_lead_preview_attachments(root, info, **kwargs) -> QuerySet: + return get_lead_preview_attachment(info) + @staticmethod def resolve_lead_groups(root, info, **kwargs) -> QuerySet: return get_lead_group_qs(info) diff --git a/apps/unified_connector/migrations/0010_auto_20240625_0806.py b/apps/unified_connector/migrations/0010_auto_20240625_0806.py index d51c840097..a8a93143ca 100644 --- a/apps/unified_connector/migrations/0010_auto_20240625_0806.py +++ b/apps/unified_connector/migrations/0010_auto_20240625_0806.py @@ -7,7 +7,7 @@ def set_file_preview(apps, schema_editor): ConnectorLeadPreviewAttachment = apps.get_model('unified_connector', 'ConnectorLeadPreviewAttachment') ConnectorLeadPreviewAttachment.objects.update( file_preview=models.F('file'), - type=2, + type=2, # default type is image ) diff --git a/deep/urls.py b/deep/urls.py index b63c873fb6..6b28a9ffba 100644 --- a/deep/urls.py +++ b/deep/urls.py @@ -26,7 +26,7 @@ unsubscribe_email, ) from gallery.views import ( - AttachmentFileView, + PrivateAttachmentFileView, FileView, FileViewSet, GoogleDriveFileViewSet, @@ -438,7 +438,7 @@ def get_api_path(path): ), path( 'external/private-file//', - AttachmentFileView.as_view(), + PrivateAttachmentFileView.as_view(), name='external_private_url', ), re_path( diff --git a/schema.graphql b/schema.graphql index 420a742f55..66f6e20708 100644 --- a/schema.graphql +++ b/schema.graphql @@ -3936,7 +3936,7 @@ type EntriesFilterDataType { type EntryAttachmentType { id: ID! - leadAttachmentId: ID! + leadAttachmentId: ID file: FileFieldType! filePreview: FileFieldType! entryFileType: EntryFileType! From 35b4dbb85b1dc28b6850fa8eb518deb6f9e19326 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Mon, 8 Jul 2024 15:40:41 +0545 Subject: [PATCH 16/17] Upadte testcase in privateattachmentfileview --- apps/entry/models.py | 18 ---- apps/export/entries/excel_exporter.py | 6 +- apps/gallery/tests/test_apis.py | 147 ++++++++++++++++++++++---- apps/gallery/utils.py | 21 +++- apps/gallery/views.py | 20 +++- deep/urls.py | 2 +- 6 files changed, 166 insertions(+), 48 deletions(-) diff --git a/apps/entry/models.py b/apps/entry/models.py index c108e4c208..4ae3209e22 100644 --- a/apps/entry/models.py +++ b/apps/entry/models.py @@ -3,10 +3,6 @@ from django.contrib.postgres.aggregates.general import ArrayAgg from django.contrib.postgres.fields import ArrayField from django.db import models -from django.conf import settings -from django.urls import reverse -from django.utils.encoding import force_bytes -from django.utils.http import urlsafe_base64_encode from deep.middleware import get_current_user from unified_connector.models import ConnectorLeadPreviewAttachment @@ -25,7 +21,6 @@ Exportable, ) from assisted_tagging.models import DraftEntry -from gallery.enums import PrivateFileModuleType class EntryAttachment(models.Model): @@ -71,19 +66,6 @@ def clone_from_lead_attachment(cls, lead_attachment: LeadPreviewAttachment) -> ' entry_attachment.save() return entry_attachment - def get_file_url(self): - return '{protocol}://{domain}{url}'.format( - protocol=settings.HTTP_PROTOCOL, - domain=settings.DJANGO_API_HOST, - url=reverse( - 'external_private_url', - kwargs={ - 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, - 'identifier': urlsafe_base64_encode(force_bytes(self.id)) - } - ) - ) - class Entry(UserResource, ProjectEntityMixin): """ diff --git a/apps/export/entries/excel_exporter.py b/apps/export/entries/excel_exporter.py index 6135239c3a..c5d9fd5154 100644 --- a/apps/export/entries/excel_exporter.py +++ b/apps/export/entries/excel_exporter.py @@ -275,7 +275,11 @@ def add_entries_from_excel_data_for_static_column( return [entry_excerpt, entry.dropped_excerpt] return entry_excerpt elif exportable == Export.StaticColumn.LEAD_ENTRY_ENTRY_ATTACHMENT_FILE_PREVIEW: - return get_private_file_url(PrivateFileModuleType.ENTRY_ATTACHMENT.value, entry.entry_attachment.id) + return get_private_file_url( + PrivateFileModuleType.ENTRY_ATTACHMENT, + entry.id, + entry.entry_attachment.file.name + ) def add_entries_from_excel_data(self, rows, data, export_data): export_type = data.get('type') diff --git a/apps/gallery/tests/test_apis.py b/apps/gallery/tests/test_apis.py index 8fe6cd7e7d..71ac0a85a6 100644 --- a/apps/gallery/tests/test_apis.py +++ b/apps/gallery/tests/test_apis.py @@ -1,20 +1,24 @@ import os import tempfile +from analysis_framework.factories import AnalysisFrameworkFactory from django.urls import reverse from django.conf import settings from django.utils.http import urlsafe_base64_encode from django.utils.encoding import force_bytes -from rest_framework import status +from entry.factories import EntryAttachmentFactory, EntryFactory +from lead.factories import LeadFactory from deep.tests import TestCase from gallery.models import File, FilePreview from lead.models import Lead from project.models import Project -from entry.models import Entry, EntryAttachment -from user.models import User +from entry.models import Entry +from user.factories import UserFactory +from project.factories import ProjectFactory from gallery.enums import PrivateFileModuleType +from utils.graphene.tests import GraphQLTestCase class GalleryTests(TestCase): @@ -211,36 +215,139 @@ def save_file_with_api(self, kwargs={}): # NOTE: Test for files -class PrivateAttachmentFileViewTest(TestCase): +class PrivateAttachmentFileViewTest(GraphQLTestCase): def setUp(self): + super().setUp() # Create a test user - self.user = User.objects.create_user(username='testuser', password='testpassword') + self.member_user = UserFactory.create() + self.member_user1 = UserFactory.create() + self.normal_user = UserFactory.create() + self.af = AnalysisFrameworkFactory.create() # Create a test entry attachment - self.project = Project.objects.create() - self.lead = Lead.objects.create(project=Project) - self.attachment = EntryAttachment.objects.create() - self.entry = Entry.objects.create( + # for normal user + self.project = ProjectFactory.create() + self.project.add_member(self.member_user1, role=self.project_role_reader_non_confidential) + # for member user + self.project1 = ProjectFactory.create() + self.project1.add_member(self.member_user, role=self.project_role_admin) + + self.lead = LeadFactory.create(project=self.project) + # UNPROTECTED lead + self.lead1 = LeadFactory.create( + project=self.project1, + confidentiality=Lead.Confidentiality.UNPROTECTED + ) + # RESTRICTED lead + self.lead2 = LeadFactory.create( + project=self.project1, + confidentiality=Lead.Confidentiality.RESTRICTED + ) + # CONFIDENTIAL lead + self.lead3 = LeadFactory.create( + project=self.project1, + confidentiality=Lead.Confidentiality.CONFIDENTIAL + ) + + self.attachment = EntryAttachmentFactory.create() + self.attachment1 = EntryAttachmentFactory.create() + self.attachment2 = EntryAttachmentFactory.create() + self.attachment3 = EntryAttachmentFactory.create() + self.attachment4 = EntryAttachmentFactory.create() + + self.entry = EntryFactory.create( + analysis_framework=self.af, lead=self.lead, project=self.project, - entry=self.attachment + entry_attachment=self.attachment3 + ) + self.entry1 = EntryFactory.create( + analysis_framework=self.af, + lead=self.lead1, + project=self.project1, + entry_attachment=self.attachment ) - self.url = reverse('private_attachment_file_view', kwargs={ + self.entry1 = EntryFactory.create( + analysis_framework=self.af, + lead=self.lead3, + project=self.project1, + entry_attachment=self.attachment1 + ) + + self.entry2 = EntryFactory.create( + analysis_framework=self.af, + lead=self.lead, + project=self.project1, + entry_attachment=self.attachment2 + ) + + self.entry3 = EntryFactory.create( + analysis_framework=self.af, + lead=self.lead1, + project=self.project, + entry_attachment=self.attachment4 + ) + + self.url1 = reverse('external_private_url', kwargs={ 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, - 'identifier': urlsafe_base64_encode(force_bytes(self.attachment.id)), + 'identifier': urlsafe_base64_encode(force_bytes(self.entry.id)), + 'filename': "test.pdf" }) - def test_access_by_authenticated_user(self): - self.authenticate() - response = self.client.get(self.url) - self.assert_200(response) + self.url2 = reverse('external_private_url', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.entry1.id)), + 'filename': "test.pdf" + }) + + self.url3 = reverse('external_private_url', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.entry2.id)), + 'filename': "test.pdf" + }) + + self.url4 = reverse('external_private_url', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.entry.id)), + 'filename': "test.pdf" + }) + + self.url5 = reverse('external_private_url', kwargs={ + 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, + 'identifier': urlsafe_base64_encode(force_bytes(self.entry3.id)), + 'filename': "test.pdf" + }) + + def test_without_login(self): + response = self.client.get(self.url1) + self.assertEqual(401, response.status_code) + + def test_access_by_normal_user(self): + self.force_login(self.normal_user) + response = self.client.get(self.url2) + self.assertEqual(403, response.status_code) + + def test_access_by_non_member_user(self): + self.force_login(self.member_user) + response = self.client.get(self.url4) + self.assertEqual(403, response.status_code) + + def test_access_by_member_user(self): + self.force_login(self.member_user) + response = self.client.get(self.url2) + self.assertEqual(302, response.status_code) + + def test_with_memeber_non_confidential_access(self): + self.force_login(self.member_user1) + response = self.client.get(self.url5) + self.assertEqual(302, response.status_code) def test_access_forbidden(self): - self.authenticate() - invalid_url = reverse('private_attachment_file_view', kwargs={ + self.force_login(self.member_user) + invalid_url = reverse('external_private_url', kwargs={ 'module': PrivateFileModuleType.ENTRY_ATTACHMENT.value, 'identifier': urlsafe_base64_encode(force_bytes(999999)), + 'filename': 'test.pdf' }) response = self.client.get(invalid_url) - self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - self.assertEqual(response.data['error'], "Access Forbidden Or File doesn't exists, Contact Admin") + self.assertEqual(404, response.status_code) diff --git a/apps/gallery/utils.py b/apps/gallery/utils.py index 73887ef9e7..006a3fcf0e 100644 --- a/apps/gallery/utils.py +++ b/apps/gallery/utils.py @@ -3,16 +3,31 @@ from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode +from gallery.enums import PrivateFileModuleType +from deep.permissions import ProjectPermissions as PP +from lead.models import Lead -def get_private_file_url(obj, id): + +def get_private_file_url(private_file_module_type: PrivateFileModuleType, id: int, filename: str): return '{protocol}://{domain}{url}'.format( protocol=settings.HTTP_PROTOCOL, domain=settings.DJANGO_API_HOST, url=reverse( 'external_private_url', kwargs={ - 'module': obj, - 'identifier': urlsafe_base64_encode(force_bytes(id)) + 'module': private_file_module_type.value, + 'identifier': urlsafe_base64_encode(force_bytes(id)), + 'filename': filename } ) ) + + +def check_private_condifential_level_permission(user, project, level): + permission = PP.get_permissions(project, user) + if PP.Permission.VIEW_ENTRY in permission: + if PP.Permission.VIEW_ALL_LEAD in permission: + return True + elif PP.Permission.VIEW_ONLY_UNPROTECTED_LEAD in permission: + return level == Lead.Confidentiality.UNPROTECTED + return False diff --git a/apps/gallery/views.py b/apps/gallery/views.py index b506d9e48f..c74538bf82 100644 --- a/apps/gallery/views.py +++ b/apps/gallery/views.py @@ -7,7 +7,8 @@ from django.utils.http import urlsafe_base64_decode from django.shortcuts import redirect, get_object_or_404 -from gallery.enums import PrivateFileModuleType +from gallery.enums import PrivateFileModuleType +from gallery.utils import check_private_condifential_level_permission from rest_framework import ( views, viewsets, @@ -25,7 +26,7 @@ from deep.permalinks import Permalink from project.models import Project from lead.models import Lead -from entry.models import Entry, EntryAttachment +from entry.models import Entry from user_resource.filters import UserResourceFilterSet from utils.extractor.formats import ( @@ -86,13 +87,21 @@ def get(self, request, uuid=None, filename=None): class PrivateAttachmentFileView(views.APIView): permission_classes = [permissions.IsAuthenticated] - def get(self, request, module=None, identifier=None): + def get(self, request, module=None, identifier=None, filename=None): id = force_text(urlsafe_base64_decode(identifier)) user = request.user obj = None if module == PrivateFileModuleType.ENTRY_ATTACHMENT.value: - obj = get_object_or_404(EntryAttachment, id=id) - obj.entry.get_for(user) + entry = get_object_or_404(Entry, id=id) + if not entry.project.is_member(user): + return response.Response({ + 'error': 'Unauthorized for the content' + }, status.HTTP_403_FORBIDDEN) + if not check_private_condifential_level_permission(user, entry.project, entry.lead.confidentiality): + return response.Response({ + 'error': 'Access Denied' + }, status.HTTP_403_FORBIDDEN) + obj = entry.entry_attachment if obj: return redirect(request.build_absolute_uri(obj.file.url)) return response.Response({ @@ -134,6 +143,7 @@ class PublicFileView(View): """ NOTE: Public File API is deprecated. """ + def get(self, request, fidb64=None, token=None, filename=None): file_id = force_text(urlsafe_base64_decode(fidb64)) file = get_object_or_404(File, id=file_id) diff --git a/deep/urls.py b/deep/urls.py index 6b28a9ffba..80fe97464e 100644 --- a/deep/urls.py +++ b/deep/urls.py @@ -437,7 +437,7 @@ def get_api_path(path): name='deprecated_gallery_private_url', ), path( - 'external/private-file//', + 'external/private-file///', PrivateAttachmentFileView.as_view(), name='external_private_url', ), From 0c6fb39b70a45a05d7c927f0a7e1282ebdda7146 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 9 Jul 2024 10:04:05 +0545 Subject: [PATCH 17/17] Update entry migrations --- ...609_0904.py => 0038_auto_20240709_0417.py} | 12 ++++++--- .../migrations/0039_auto_20240609_0933.py | 23 ----------------- ...0_alter_entryattachment_entry_file_type.py | 18 ------------- .../migrations/0041_auto_20240621_1149.py | 25 ------------------- apps/gallery/utils.py | 2 +- apps/gallery/views.py | 4 +-- 6 files changed, 12 insertions(+), 72 deletions(-) rename apps/entry/migrations/{0038_auto_20240609_0904.py => 0038_auto_20240709_0417.py} (64%) delete mode 100644 apps/entry/migrations/0039_auto_20240609_0933.py delete mode 100644 apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py delete mode 100644 apps/entry/migrations/0041_auto_20240621_1149.py diff --git a/apps/entry/migrations/0038_auto_20240609_0904.py b/apps/entry/migrations/0038_auto_20240709_0417.py similarity index 64% rename from apps/entry/migrations/0038_auto_20240609_0904.py rename to apps/entry/migrations/0038_auto_20240709_0417.py index 14029c166b..faa815b897 100644 --- a/apps/entry/migrations/0038_auto_20240609_0904.py +++ b/apps/entry/migrations/0038_auto_20240709_0417.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.25 on 2024-06-09 09:04 +# Generated by Django 3.2.25 on 2024-07-09 04:17 from django.db import migrations, models import django.db.models.deletion @@ -7,6 +7,7 @@ class Migration(migrations.Migration): dependencies = [ + ('lead', '0051_auto_20240625_0509'), ('entry', '0037_merge_20220401_0527'), ] @@ -20,10 +21,15 @@ class Migration(migrations.Migration): name='EntryAttachment', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('entry_file_type', models.PositiveSmallIntegerField(choices=[('1', 'XLSX')], default='1')), + ('entry_file_type', models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1)), ('file', models.FileField(upload_to='entry/attachment/')), ('file_preview', models.FileField(upload_to='entry/attachment-preview')), - ('entry', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='entry.entry')), + ('lead_attachment', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='lead.leadpreviewattachment')), ], ), + migrations.AddField( + model_name='entry', + name='entry_attachment', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='entry.entryattachment'), + ), ] diff --git a/apps/entry/migrations/0039_auto_20240609_0933.py b/apps/entry/migrations/0039_auto_20240609_0933.py deleted file mode 100644 index 53a8985094..0000000000 --- a/apps/entry/migrations/0039_auto_20240609_0933.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-09 09:33 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('entry', '0038_auto_20240609_0904'), - ] - - operations = [ - migrations.RemoveField( - model_name='entryattachment', - name='entry', - ), - migrations.AddField( - model_name='entry', - name='entry_attachment', - field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='entry.entryattachment'), - ), - ] diff --git a/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py b/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py deleted file mode 100644 index ad7aeccb31..0000000000 --- a/apps/entry/migrations/0040_alter_entryattachment_entry_file_type.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-09 10:27 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('entry', '0039_auto_20240609_0933'), - ] - - operations = [ - migrations.AlterField( - model_name='entryattachment', - name='entry_file_type', - field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX')], default=1), - ), - ] diff --git a/apps/entry/migrations/0041_auto_20240621_1149.py b/apps/entry/migrations/0041_auto_20240621_1149.py deleted file mode 100644 index b296b52be9..0000000000 --- a/apps/entry/migrations/0041_auto_20240621_1149.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 3.2.25 on 2024-06-21 11:49 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('entry', '0040_alter_entryattachment_entry_file_type'), - ('lead', '0051_auto_20240625_0509') - ] - - operations = [ - migrations.AddField( - model_name='entryattachment', - name='lead_attachment', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='lead.leadpreviewattachment'), - ), - migrations.AlterField( - model_name='entryattachment', - name='entry_file_type', - field=models.PositiveSmallIntegerField(choices=[(1, 'XLSX'), (2, 'Image')], default=1), - ), - ] diff --git a/apps/gallery/utils.py b/apps/gallery/utils.py index 006a3fcf0e..f52ec3c55e 100644 --- a/apps/gallery/utils.py +++ b/apps/gallery/utils.py @@ -23,7 +23,7 @@ def get_private_file_url(private_file_module_type: PrivateFileModuleType, id: in ) -def check_private_condifential_level_permission(user, project, level): +def check_private_confidential_level_permission(user, project, level): permission = PP.get_permissions(project, user) if PP.Permission.VIEW_ENTRY in permission: if PP.Permission.VIEW_ALL_LEAD in permission: diff --git a/apps/gallery/views.py b/apps/gallery/views.py index c74538bf82..f4c023df8f 100644 --- a/apps/gallery/views.py +++ b/apps/gallery/views.py @@ -8,7 +8,7 @@ from django.shortcuts import redirect, get_object_or_404 from gallery.enums import PrivateFileModuleType -from gallery.utils import check_private_condifential_level_permission +from gallery.utils import check_private_confidential_level_permission from rest_framework import ( views, viewsets, @@ -97,7 +97,7 @@ def get(self, request, module=None, identifier=None, filename=None): return response.Response({ 'error': 'Unauthorized for the content' }, status.HTTP_403_FORBIDDEN) - if not check_private_condifential_level_permission(user, entry.project, entry.lead.confidentiality): + if not check_private_confidential_level_permission(user, entry.project, entry.lead.confidentiality): return response.Response({ 'error': 'Access Denied' }, status.HTTP_403_FORBIDDEN)