From b8088270a96696794b7d4100f0b2749f9f35eb4c Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:36:20 -0500 Subject: [PATCH] fix: Various minor bugs on homepage. chore(refactor): Replace Tastypie /api/dataset with djangorestframework /datasets/ #74 fix: - Fix sorting by name - Make sorting stable for size and collection_id - Fix missing column if kingfisher_metadata not available - Format undefined as blank, not 0 chore: - Remove extra methods on Dataset model that required extra, repetitive queries - Rename dataset_original (related_name=dataset_filter_parent) to parent (children) - Rename dataset_filtered (related_name=dataset_filter_child) to dataset (filtered) - Use meta.compiled_releases.total_unique_ocids consistently (instead of progress_monitor_dataset.size) - Remove some   #111 --- backend/api/api.py | 20 +------- backend/api/models.py | 35 ++------------ backend/api/urls.py | 10 ++-- backend/controller/views.py | 49 ++++++++++++++++++-- backend/tests/api/test_views.py | 6 +++ docs/backend/reference/integration.rst | 4 +- frontend/src/components/DatasetPicker.vue | 18 ++++--- frontend/src/components/DatasetPickerRow.vue | 22 +++------ frontend/src/config.js | 9 ++-- frontend/src/plugins/numeral.js | 3 ++ frontend/src/store.js | 2 +- frontend/src/views/Overview.vue | 4 +- 12 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 backend/tests/api/test_views.py diff --git a/backend/api/api.py b/backend/api/api.py index b9b2784d..75331a61 100644 --- a/backend/api/api.py +++ b/backend/api/api.py @@ -1,8 +1,8 @@ # myapp/api.py -from tastypie.fields import CharField, DictField, IntegerField, ListField +from tastypie.fields import DictField, IntegerField from tastypie.resources import ModelResource -from api.models import DataItem, Dataset +from api.models import DataItem class DataItemResource(ModelResource): @@ -12,19 +12,3 @@ class DataItemResource(ModelResource): class Meta: queryset = DataItem.objects.all() resource_name = "data_item" - - -class DatasetResource(ModelResource): - meta = DictField(attribute="meta") - state = CharField(attribute="get_state", readonly=True) - phase = CharField(attribute="get_phase", readonly=True) - size = IntegerField(attribute="get_size", readonly=True) - filtered_children_ids = ListField(attribute="get_filtered_children_ids", readonly=True) - filtered_parent_id = IntegerField(attribute="get_filtered_parent_id", readonly=True) - filtered_parent_name = CharField(attribute="get_filtered_parent_name", readonly=True) - filter_message = DictField(attribute="get_filter_message", readonly=True) - - class Meta: - queryset = Dataset.objects.all() - resource_name = "dataset" - limit = 1000 diff --git a/backend/api/models.py b/backend/api/models.py index dd11ae50..23ad08e8 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -32,33 +32,6 @@ class Dataset(models.Model): created = models.DateTimeField(blank=True, null=True) modified = models.DateTimeField(blank=True, null=True) - def get_state(self): - if self.progress: - return self.progress.state - - def get_phase(self): - if self.progress: - return self.progress.phase - - def get_size(self): - if self.progress: - return self.progress.size - - def get_filtered_children_ids(self): - return [item.dataset_filtered.id for item in self.dataset_filter_parent.all()] - - def get_filtered_parent_id(self): - items = self.dataset_filter_child.all() - return items[0].dataset_original.id if items else None - - def get_filtered_parent_name(self): - items = self.dataset_filter_child.all() - return items[0].dataset_original.name if items else None - - def get_filter_message(self): - items = self.dataset_filter_child.all() - return items[0].filter_message if items else None - class Meta: managed = False db_table = "dataset" @@ -66,11 +39,11 @@ class Meta: class DatasetFilter(models.Model): id = models.BigAutoField(primary_key=True) - dataset_original = models.ForeignKey( - Dataset, on_delete=models.CASCADE, related_name="dataset_filter_parent", db_column="dataset_id_original" + parent = models.ForeignKey( + Dataset, on_delete=models.CASCADE, related_name="children", db_column="dataset_id_original" ) - dataset_filtered = models.ForeignKey( - Dataset, on_delete=models.CASCADE, related_name="dataset_filter_child", db_column="dataset_id_filtered" + dataset = models.ForeignKey( + Dataset, on_delete=models.CASCADE, related_name="filtered", db_column="dataset_id_filtered" ) filter_message = models.JSONField() created = models.DateTimeField(blank=True, null=True) diff --git a/backend/api/urls.py b/backend/api/urls.py index 6578cb3f..596468f0 100644 --- a/backend/api/urls.py +++ b/backend/api/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path -from api.api import DataItemResource, DatasetResource +from api.api import DataItemResource from api.views import ( dataset_distinct_values, dataset_filter_items, @@ -12,19 +12,18 @@ time_variance_level_stats, ) -dataset_resource = DatasetResource() data_item_resource = DataItemResource() urlpatterns = [ - # Stats + # Check stats path("api/field_level_stats/", field_level_stats, name="field_level_stats"), path("api/resource_level_stats/", resource_level_stats, name="resource_level_stats"), path("api/dataset_level_stats/", dataset_level_stats, name="dataset_level_stats"), path("api/time_variance_level_stats/", time_variance_level_stats, name="time_variance_level_stats"), - # Detail + # Check details path("api/field_level_detail//", field_level_detail, name="field_level_detail"), path("api/resource_level_detail//", resource_level_detail, name="resource_level_detail"), - # Rest + # Filter datasets path("api/dataset_filter_items", dataset_filter_items, name="dataset_filter_items"), path( "api/dataset_distinct_values//", dataset_distinct_values, name="dataset_distinct_values" @@ -35,6 +34,5 @@ name="dataset_distinct_values", ), # Tastypie - path("api/", include(dataset_resource.urls)), path("api/", include(data_item_resource.urls)), ] diff --git a/backend/controller/views.py b/backend/controller/views.py index a9d5b6b3..f2bfb79f 100644 --- a/backend/controller/views.py +++ b/backend/controller/views.py @@ -1,4 +1,5 @@ from django.db import connections +from django.db.models import F, OuterRef, Subquery from django.shortcuts import get_object_or_404 from psycopg2.sql import SQL, Identifier from rest_framework import serializers, status, viewsets @@ -6,10 +7,34 @@ from rest_framework.response import Response from rest_framework.schemas.openapi import AutoSchema -from api.models import Dataset, FieldLevelCheck, ProgressMonitorDataset +from api.models import Dataset, DatasetFilter, FieldLevelCheck, ProgressMonitorDataset from controller.rabbitmq import publish +class DatasetSerializer(serializers.ModelSerializer): + phase = serializers.CharField() + state = serializers.CharField() + parent_id = serializers.IntegerField() + parent_name = serializers.CharField() + filter_message = serializers.JSONField() + + class Meta: + model = Dataset + fields = [ + "id", + "name", + "meta", + "ancestor_id", + "created", + "modified", + "phase", + "state", + "parent_id", + "parent_name", + "filter_message", + ] + + class CreateDatasetSerializer(serializers.Serializer): name = serializers.CharField(help_text="The name to assign to the dataset") collection_id = serializers.IntegerField(help_text="The collection ID in Kingfisher Process") @@ -33,11 +58,11 @@ class FilterDatasetSerializer(serializers.Serializer): class CustomSchema(AutoSchema): def get_responses(self, path, method): responses = super().get_responses(path, method) - # POST requests return HTTP 202 with no content. + # POST requests return HTTP 202 with no content (not 201). if "201" in responses: responses["202"] = responses.pop("201") del responses["202"]["content"] - # GET requests return a JSON object. + # GET requests return a JSON object (not string). elif "200" in responses: responses["200"]["content"]["application/json"]["schema"] = {"type": "object"} return responses @@ -57,10 +82,26 @@ def get_object(self): return get_object_or_404(self.get_queryset(), pk=self.kwargs["pk"]) def get_serializer(self, *args, **kwargs): - if self.action == "create": + if self.action == "list": + return DatasetSerializer(*args, **kwargs) + elif self.action == "create": return CreateDatasetSerializer(*args, **kwargs) elif self.action == "filter": return FilterDatasetSerializer(*args, **kwargs) + else: + raise NotImplementedError(f"no serializer for action {self.action}") + + def list(self, request, *args, **kwargs): + dataset_filter = DatasetFilter.objects.filter(dataset=OuterRef("pk"))[:1] + queryset = self.get_queryset().annotate( + phase=F("progress__phase"), + state=F("progress__state"), + parent_id=Subquery(dataset_filter.values("parent__id")), + parent_name=Subquery(dataset_filter.values("parent__name")), + filter_message=Subquery(dataset_filter.values("filter_message")), + ) + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) def create(self, request): """ diff --git a/backend/tests/api/test_views.py b/backend/tests/api/test_views.py new file mode 100644 index 00000000..69a3112c --- /dev/null +++ b/backend/tests/api/test_views.py @@ -0,0 +1,6 @@ +from tests import TestCase + + +class ViewsTests(TestCase): + def test_dataset_list(self): + pass diff --git a/docs/backend/reference/integration.rst b/docs/backend/reference/integration.rst index 0e0b8f74..31c30ee7 100644 --- a/docs/backend/reference/integration.rst +++ b/docs/backend/reference/integration.rst @@ -10,8 +10,8 @@ To update ``backend/api/models.py`` following changes to Pelican backend's datab - Replace ``models.DO_NOTHING`` with ``on_delete=models.CASCADE`` - ``Dataset``: Add methods - ``Dataset.meta``: Add ``blank=True, default=dict`` -- ``DatasetFilter.dataset_id_original``: Rename to ``dataset_original``, add ``related_name="dataset_filter_parent"`` -- ``DatasetFilter.dataset_id_filtered``: Rename to ``dataset_filtered``, add ``related_name="dataset_filter_child"`` +- ``DatasetFilter.dataset_id_original``: Rename to ``parent``, add ``related_name="children"`` +- ``DatasetFilter.dataset_id_filtered``: Rename to ``dataset``, add ``related_name="filtered"`` - ``ProgressMonitorDataset.dataset``: Add ``related_name="progress"`` - ``ProgressMonitorItem.item``: Rename to ``data_item`` - ``Report.type``: Change ``TextField`` to ``CharField``, add ``max_length=255``, and remove ``# This field type is a guess.`` diff --git a/frontend/src/components/DatasetPicker.vue b/frontend/src/components/DatasetPicker.vue index b0d4f187..2aa29b4e 100644 --- a/frontend/src/components/DatasetPicker.vue +++ b/frontend/src/components/DatasetPicker.vue @@ -13,7 +13,7 @@
{ - this.datasets = buildDatasetsTree(response["data"]["objects"], null); + this.datasets = buildDatasetsTree(response["data"], null); var self = this; this.datasets.forEach(function (item) { @@ -228,12 +228,16 @@ export default { } else if (by == "name") { comp = (a, b) => a.name.localeCompare(b.name); } else if (by == "size") { - comp = (a, b) => this.compareNumbers(a.size, b.size); + comp = (a, b) => + this.compareNumbers( + a.meta.compiled_releases?.total_unique_ocids || -1, + b.meta.compiled_releases?.total_unique_ocids || -1 + ); } else if (by == "collection_id") { comp = (a, b) => this.compareNumbers( - a.meta.kingfisher_metadata.collection_id, - b.meta.kingfisher_metadata.collection_id + a.meta.kingfisher_metadata?.collection_id || -1, + b.meta.kingfisher_metadata?.collection_id || -1 ); } else if (by == "phase") { comp = (a, b) => { diff --git a/frontend/src/components/DatasetPickerRow.vue b/frontend/src/components/DatasetPickerRow.vue index 44bda6d1..3b858f1c 100644 --- a/frontend/src/components/DatasetPickerRow.vue +++ b/frontend/src/components/DatasetPickerRow.vue @@ -4,19 +4,13 @@ :class="['row', 'tr', 'align-items-center']" >
- -    - -      + + {{ dataset.name }} - (Id {{ dataset.id }}) -   + (Id {{ dataset.id }}) -  
- {{ dataset.size | formatNumber }} + {{ dataset.meta.compiled_releases?.total_unique_ocids | formatNumber }}
-
- {{ dataset.meta.kingfisher_metadata.collection_id }} +
+ {{ dataset.meta.kingfisher_metadata?.collection_id }}