From ab3de6c1361cae769ce1ea2022b9a2169f195012 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 7 Dec 2023 11:46:54 +0545 Subject: [PATCH] Add custom date format support --- apps/export/entries/excel_exporter.py | 35 ++++++++++++++++---- apps/export/entries/report_exporter.py | 46 ++++++++++++++++++++------ apps/export/enums.py | 6 ++++ apps/export/models.py | 28 +++++++++++++++- apps/export/serializers.py | 3 ++ apps/export/tasks/tasks_entries.py | 4 +++ schema.graphql | 7 ++++ utils/common.py | 13 +++++++- 8 files changed, 122 insertions(+), 20 deletions(-) diff --git a/apps/export/entries/excel_exporter.py b/apps/export/entries/excel_exporter.py index 860d6c0032..69d38a0d0c 100644 --- a/apps/export/entries/excel_exporter.py +++ b/apps/export/entries/excel_exporter.py @@ -4,12 +4,13 @@ from deep.permalinks import Permalink from utils.common import ( - deep_date_format, excel_column_name, - get_valid_xml_string as xstr + get_valid_xml_string as xstr, + deep_date_parse, ) from export.formats.xlsx import WorkBook, RowsBuilder +from analysis_framework.models import Widget from entry.models import Entry, ExportData, ProjectEntryLabel, LeadEntryGroup from lead.models import Lead from export.models import Export @@ -35,7 +36,16 @@ class ColumnsData: ] if self.modified_excerpt_exists else ['Excerpt'], } - def __init__(self, export_object, entries, project, columns=None, decoupled=True, is_preview=False): + def __init__( + self, + export_object, + entries, + project, + date_format, + columns=None, + decoupled=True, + is_preview=False, + ): self.project = project self.export_object = export_object self.is_preview = is_preview @@ -43,6 +53,9 @@ def __init__(self, export_object, entries, project, columns=None, decoupled=True # XXX: Limit memory usage? (Or use redis?) self.geoarea_data_cache = {} + # Date Format + self.date_renderer = Export.get_date_renderer(date_format) + # Create worksheets(Main, Grouped, Entry Groups, Bibliography) if decoupled: self.split = self.wb.get_active_sheet()\ @@ -225,11 +238,11 @@ def add_entries_from_excel_data_for_static_column( assignee, ): if exportable == Export.StaticColumn.LEAD_PUBLISHED_ON: - return deep_date_format(lead.published_on) + return self.date_renderer(lead.published_on) if exportable == Export.StaticColumn.ENTRY_CREATED_BY: return entry.created_by and entry.created_by.profile.get_display_name() elif exportable == Export.StaticColumn.ENTRY_CREATED_AT: - return deep_date_format(entry.created_at) + return self.date_renderer(entry.created_at) elif exportable == Export.StaticColumn.ENTRY_CONTROL_STATUS: return 'Controlled' if entry.controlled else 'Uncontrolled' elif exportable == Export.StaticColumn.LEAD_ID: @@ -313,7 +326,15 @@ def add_entries_from_excel_data(self, rows, data, export_data): col_span, ) else: - rows.add_value_list(export_data.get('values')) + export_data_values = export_data.get('values') + if export_data.get('widget_key') == Widget.WidgetType.DATE_RANGE.value: + if len(export_data_values) == 2 and any(export_data_values): + rows.add_value_list([ + self.date_renderer(deep_date_parse(export_data_values[0], raise_exception=False)), + self.date_renderer(deep_date_parse(export_data_values[1], raise_exception=False)), + ]) + else: + rows.add_value_list(export_data_values) else: rows.add_value_list([''] * col_span) @@ -548,7 +569,7 @@ def add_bibliography_sheet(self, leads_qs): [[ lead.get_authors_display(), lead.get_source_display(), - deep_date_format(lead.published_on), + self.date_renderer(lead.published_on), get_hyperlink(lead.url, lead.title) if lead.url else lead.title, lead.filtered_entry_count, ]] diff --git a/apps/export/entries/report_exporter.py b/apps/export/entries/report_exporter.py index 70b43e3d13..8b19b9923f 100644 --- a/apps/export/entries/report_exporter.py +++ b/apps/export/entries/report_exporter.py @@ -13,7 +13,7 @@ ) from docx.shared import Inches from deep.permalinks import Permalink -from utils.common import deep_date_format +from utils.common import deep_date_parse from export.formats.docx import Document @@ -93,11 +93,24 @@ def _get_date_range_widget_data(cls, data, bold, **kwargs): - tuple (from, to) as described here: apps.entry.widgets.date_range_widget._get_date """ + date_renderer = kwargs['date_renderer'] values = data.get('values', []) if len(values) == 2 and any(values): label = '{} - {}'.format( - values[0] or "00-00-00", - values[1] or "00-00-00", + date_renderer( + deep_date_parse( + values[0], + raise_exception=False + ), + fallback="00-00-00", + ), + date_renderer( + deep_date_parse( + values[1], + raise_exception=False + ), + fallback="00-00-00", + ), ) return cls._add_common, label, bold @@ -124,12 +137,18 @@ def _get_date_widget_data(cls, data, bold, **kwargs): as described here: apps.entry.widgets.date_widget """ value = data.get('value') - if value: - return cls._add_common, value, bold + if not value: + return + date_renderer = kwargs['date_renderer'] + _value = date_renderer(deep_date_parse(value, raise_exception=False)) + if _value: + return cls._add_common, _value, bold @classmethod def _get_time_widget_data(cls, data, bold, **kwargs): - cls._get_date_widget_data(data, bold, **kwargs) + value = data.get('value') + if value: + return cls._add_common, value, bold @classmethod def _get_select_widget_data(cls, data, bold, **kwargs): @@ -201,6 +220,7 @@ def get_widget_information_into_report( class ReportExporter: def __init__( self, + date_format: Export.DateFormat, citation_style: Export.CitationStyle, exporting_widgets=None, # eg: ["517", "43", "42"] is_preview=False, @@ -234,6 +254,9 @@ def __init__( # Citation self.citation_style = citation_style or Export.CitationStyle.DEFAULT + # Date Format + self.date_renderer = Export.get_date_renderer(date_format) + def load_exportables(self, exportables, regions): exportables = exportables.filter( data__report__levels__isnull=False, @@ -483,7 +506,8 @@ def _generate_for_entry_widget_data(self, entry, para): if resp := WidgetExporter.get_widget_information_into_report( data, bold=True, - _get_geo_admin_level_1_data=self._get_geo_admin_level_1_data + _get_geo_admin_level_1_data=self._get_geo_admin_level_1_data, + date_renderer=self.date_renderer, ): export_data.append(resp) except ExportDataVersionMismatch: @@ -620,7 +644,7 @@ def _generate_for_entry(self, entry): elif lead.confidentiality == Lead.Confidentiality.RESTRICTED: para.add_run(' (restricted)') - if self.citation_style == Export.CitationStyle.STYLE_1: + if not self.citation_style == Export.CitationStyle.STYLE_1: pass else: # Default # Add lead title if available @@ -629,7 +653,7 @@ def _generate_for_entry(self, entry): # Finally add date if date: - para.add_run(f", {deep_date_format(date)}") + para.add_run(f", {self.date_renderer(date)}") para.add_run(')') # --- Reference End @@ -749,7 +773,7 @@ def pre_build_document(self, project): """ self.doc.add_heading( 'DEEP Export — {} — {}'.format( - deep_date_format(datetime.today()), + self.date_renderer(datetime.today()), project.title, ), 1, @@ -853,7 +877,7 @@ def export(self, pdf=False): para.add_run(f' {source}.') para.add_run(f' {lead.title}.') if lead.published_on: - para.add_run(f" {deep_date_format(lead.published_on)}. ") + para.add_run(f" {self.date_renderer(lead.published_on)}. ") para = self.doc.add_paragraph() url = lead.url or Permalink.lead_share_view(lead.uuid) diff --git a/apps/export/enums.py b/apps/export/enums.py index c846478ff4..fd17b40bca 100644 --- a/apps/export/enums.py +++ b/apps/export/enums.py @@ -13,6 +13,7 @@ Export.StaticColumn, name='ExportExcelSelectedStaticColumnEnum', ) +ExportDateFormatEnum = convert_enum_to_graphene_enum(Export.DateFormat, name='ExportDateFormatEnum') ExportReportCitationStyleEnum = convert_enum_to_graphene_enum( Export.CitationStyle, name='ExportReportCitationStyleEnum', @@ -44,6 +45,11 @@ field_name='static_column', serializer_name='ExportExcelSelectedColumnSerializer', ): ExportExcelSelectedStaticColumnEnum, + get_enum_name_from_django_field( + None, + field_name='date_format', + serializer_name='ExportExtraOptionsSerializer', + ): ExportDateFormatEnum, get_enum_name_from_django_field( None, field_name='report_citation_style', diff --git a/apps/export/models.py b/apps/export/models.py index 7fb078b8e9..2ac8e522d7 100644 --- a/apps/export/models.py +++ b/apps/export/models.py @@ -1,3 +1,6 @@ +import typing +import datetime + from django.db import models from django.core.cache import cache from django.contrib.auth.models import User @@ -150,6 +153,13 @@ class CitationStyle(models.IntegerChoices): DEFAULT = 1, 'Default' STYLE_1 = 2, 'Sample 1' # TODO: Update naming + # Used by extra options + # NOTE: Value should always be usable by date.strftime + # TODO: Add a unit test to make sure all label are valid + class DateFormat(models.IntegerChoices): + DEFAULT = 1, '%d-%m-%Y' + FORMAT_1 = 2, '%d/%m/%Y' + # NOTE: Also used to validate which combination is supported DEFAULT_TITLE_LABEL = { (DataType.ENTRIES, ExportType.EXCEL, Format.XLSX): 'Entries Excel Export', @@ -165,7 +175,7 @@ class CitationStyle(models.IntegerChoices): CELERY_TASK_CACHE_KEY = CacheKey.EXPORT_TASK_CACHE_KEY_FORMAT - # Number of entries to proccess if is_preview is True + # Number of entries to process if is_preview is True PREVIEW_ENTRY_SIZE = 10 PREVIEW_ASSESSMENT_SIZE = 10 @@ -196,6 +206,22 @@ def generate_title(cls, data_type, export_type, export_format): time_str = deep_date_format(timezone.now()) return f'{time_str} DEEP {file_label}' + @classmethod + def get_date_renderer(cls, date_format: typing.Optional[DateFormat]) -> typing.Callable: + date_format = cls.DateFormat.FORMAT_1 + # if date_format is None or date_format == Export.DateFormat.DEFAULT: + # return deep_date_format + + def custom_format(d, fallback: typing.Optional[str] = ''): + if d and ( + isinstance(d, datetime.datetime) or + isinstance(d, datetime.date) + ): + return d.strftime(cls.DateFormat(date_format).label) + return fallback + + return custom_format + def save(self, *args, **kwargs): self.title = self.title or self.generate_title(self.type, self.export_type, self.format) return super().save(*args, **kwargs) diff --git a/apps/export/serializers.py b/apps/export/serializers.py index 205859cfcf..5cadb05c83 100644 --- a/apps/export/serializers.py +++ b/apps/export/serializers.py @@ -215,6 +215,9 @@ def validate(self, data): class ExportExtraOptionsSerializer(ProjectPropertySerializerMixin, serializers.Serializer): + # Common + date_format = serializers.ChoiceField(choices=Export.DateFormat.choices, required=False) + # Excel excel_decoupled = serializers.BooleanField( help_text="Don't group entries tags. Slower export generation.", required=False) diff --git a/apps/export/tasks/tasks_entries.py b/apps/export/tasks/tasks_entries.py index a76d5ea8f3..60888c44ed 100644 --- a/apps/export/tasks/tasks_entries.py +++ b/apps/export/tasks/tasks_entries.py @@ -73,6 +73,8 @@ def export_entries(export): ).distinct() regions = Region.objects.filter(project=project).distinct() + date_format = extra_options.get('date_format') + if export_type == Export.ExportType.EXCEL: decoupled = extra_options.get('excel_decoupled', False) columns = extra_options.get('excel_columns') @@ -80,6 +82,7 @@ def export_entries(export): export, entries_qs, project, + date_format, columns=columns, decoupled=decoupled, is_preview=is_preview, @@ -104,6 +107,7 @@ def export_entries(export): show_groups = extra_options.get('report_show_groups') export_data = ( ReportExporter( + date_format, citation_style, exporting_widgets=exporting_widgets, is_preview=is_preview, diff --git a/schema.graphql b/schema.graphql index f9775d89fc..94ef75e2db 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1960,6 +1960,11 @@ enum ExportDataTypeEnum { ANALYSES } +enum ExportDateFormatEnum { + DEFAULT + FORMAT_1 +} + input ExportExcelSelectedColumnInputType { isWidget: Boolean! widgetKey: String @@ -1998,6 +2003,7 @@ enum ExportExportTypeEnum { } input ExportExtraOptionsInputType { + dateFormat: ExportDateFormatEnum excelDecoupled: Boolean excelColumns: [ExportExcelSelectedColumnInputType!] reportShowGroups: Boolean @@ -2012,6 +2018,7 @@ input ExportExtraOptionsInputType { } type ExportExtraOptionsType { + dateFormat: ExportDateFormatEnum excelDecoupled: Boolean excelColumns: [ExportExcelSelectedColumnType!] reportShowGroups: Boolean diff --git a/utils/common.py b/utils/common.py index fe18ae654f..a6f3d52c44 100644 --- a/utils/common.py +++ b/utils/common.py @@ -143,11 +143,22 @@ def deep_date_format( date: Optional[Union[datetime.date, datetime.datetime]], fallback: Optional[str] = '' ) -> Optional[str]: - if date: + if date and ( + isinstance(date, datetime.datetime) or + isinstance(date, datetime.date) + ): return date.strftime('%d-%m-%Y') return fallback +def deep_date_parse(date_str: str, raise_exception=True) -> Optional[datetime.date]: + try: + return datetime.datetime.strptime(date_str, '%d-%m-%Y').date() + except (ValueError, TypeError) as e: + if raise_exception: + raise e + + def parse_date(date_str): try: return date_str and datetime.datetime.strptime(date_str, '%d-%m-%Y')