fix: Various minor bugs on homepage. chore(refactor): Replace Tastypi…

…e /api/dataset with djangorestframework /datasets/ #74

fix:
- Fix sorting by name
- Make sorting stable for size and collection_id
- Fix missing column if kingfisher_metadata not available
- Format undefined as blank, not 0

chore:
- Remove extra methods on Dataset model that required extra, repetitive queries
- Rename dataset_original (related_name=dataset_filter_parent) to parent (children)
- Rename dataset_filtered (related_name=dataset_filter_child) to dataset (filtered)
- Use meta.compiled_releases.total_unique_ocids consistently (instead of progress_monitor_dataset.size)
- Remove some   #111

backend/api/api.py

@@ -1,8 +1,8 @@
 # myapp/api.py
-from tastypie.fields import CharField, DictField, IntegerField, ListField
+from tastypie.fields import DictField, IntegerField
 from tastypie.resources import ModelResource
 
-from api.models import DataItem, Dataset
+from api.models import DataItem
 
 
 class DataItemResource(ModelResource):
@@ -12,19 +12,3 @@ class DataItemResource(ModelResource):
     class Meta:
         queryset = DataItem.objects.all()
         resource_name = "data_item"
-
-
-class DatasetResource(ModelResource):
-    meta = DictField(attribute="meta")
-    state = CharField(attribute="get_state", readonly=True)
-    phase = CharField(attribute="get_phase", readonly=True)
-    size = IntegerField(attribute="get_size", readonly=True)
-    filtered_children_ids = ListField(attribute="get_filtered_children_ids", readonly=True)
-    filtered_parent_id = IntegerField(attribute="get_filtered_parent_id", readonly=True)
-    filtered_parent_name = CharField(attribute="get_filtered_parent_name", readonly=True)
-    filter_message = DictField(attribute="get_filter_message", readonly=True)
-
-    class Meta:
-        queryset = Dataset.objects.all()
-        resource_name = "dataset"
-        limit = 1000

backend/api/models.py

@@ -32,45 +32,18 @@ class Dataset(models.Model):
     created = models.DateTimeField(blank=True, null=True)
     modified = models.DateTimeField(blank=True, null=True)
 
-    def get_state(self):
-        if self.progress:
-            return self.progress.state
-
-    def get_phase(self):
-        if self.progress:
-            return self.progress.phase
-
-    def get_size(self):
-        if self.progress:
-            return self.progress.size
-
-    def get_filtered_children_ids(self):
-        return [item.dataset_filtered.id for item in self.dataset_filter_parent.all()]
-
-    def get_filtered_parent_id(self):
-        items = self.dataset_filter_child.all()
-        return items[0].dataset_original.id if items else None
-
-    def get_filtered_parent_name(self):
-        items = self.dataset_filter_child.all()
-        return items[0].dataset_original.name if items else None
-
-    def get_filter_message(self):
-        items = self.dataset_filter_child.all()
-        return items[0].filter_message if items else None
-
     class Meta:
         managed = False
         db_table = "dataset"
 
 
 class DatasetFilter(models.Model):
     id = models.BigAutoField(primary_key=True)
-    dataset_original = models.ForeignKey(
-        Dataset, on_delete=models.CASCADE, related_name="dataset_filter_parent", db_column="dataset_id_original"
+    parent = models.ForeignKey(
+        Dataset, on_delete=models.CASCADE, related_name="children", db_column="dataset_id_original"
     )
-    dataset_filtered = models.ForeignKey(
-        Dataset, on_delete=models.CASCADE, related_name="dataset_filter_child", db_column="dataset_id_filtered"
+    dataset = models.ForeignKey(
+        Dataset, on_delete=models.CASCADE, related_name="filtered", db_column="dataset_id_filtered"
     )
     filter_message = models.JSONField()
     created = models.DateTimeField(blank=True, null=True)

backend/api/urls.py

@@ -1,6 +1,6 @@
 from django.urls import include, path
 
-from api.api import DataItemResource, DatasetResource
+from api.api import DataItemResource
 from api.views import (
     dataset_distinct_values,
     dataset_filter_items,
@@ -12,19 +12,18 @@
     time_variance_level_stats,
 )
 
-dataset_resource = DatasetResource()
 data_item_resource = DataItemResource()
 
 urlpatterns = [
-    # Stats
+    # Check stats
     path("api/field_level_stats/<dataset_id>", field_level_stats, name="field_level_stats"),
     path("api/resource_level_stats/<dataset_id>", resource_level_stats, name="resource_level_stats"),
     path("api/dataset_level_stats/<dataset_id>", dataset_level_stats, name="dataset_level_stats"),
     path("api/time_variance_level_stats/<dataset_id>", time_variance_level_stats, name="time_variance_level_stats"),
-    # Detail
+    # Check details
     path("api/field_level_detail/<dataset_id>/<path>", field_level_detail, name="field_level_detail"),
     path("api/resource_level_detail/<dataset_id>/<check_name>", resource_level_detail, name="resource_level_detail"),
-    # Rest
+    # Filter datasets
     path("api/dataset_filter_items", dataset_filter_items, name="dataset_filter_items"),
     path(
         "api/dataset_distinct_values/<dataset_id>/<json_path>", dataset_distinct_values, name="dataset_distinct_values"
@@ -35,6 +34,5 @@
         name="dataset_distinct_values",
     ),
     # Tastypie
-    path("api/", include(dataset_resource.urls)),
     path("api/", include(data_item_resource.urls)),
 ]

backend/controller/views.py

@@ -1,15 +1,40 @@
 from django.db import connections
+from django.db.models import F, OuterRef, Subquery
 from django.shortcuts import get_object_or_404
 from psycopg2.sql import SQL, Identifier
 from rest_framework import serializers, status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.schemas.openapi import AutoSchema
 
-from api.models import Dataset, FieldLevelCheck, ProgressMonitorDataset
+from api.models import Dataset, DatasetFilter, FieldLevelCheck, ProgressMonitorDataset
 from controller.rabbitmq import publish
 
 
+class DatasetSerializer(serializers.ModelSerializer):
+    phase = serializers.CharField()
+    state = serializers.CharField()
+    parent_id = serializers.IntegerField()
+    parent_name = serializers.CharField()
+    filter_message = serializers.JSONField()
+
+    class Meta:
+        model = Dataset
+        fields = [
+            "id",
+            "name",
+            "meta",
+            "ancestor_id",
+            "created",
+            "modified",
+            "phase",
+            "state",
+            "parent_id",
+            "parent_name",
+            "filter_message",
+        ]
+
+
 class CreateDatasetSerializer(serializers.Serializer):
     name = serializers.CharField(help_text="The name to assign to the dataset")
     collection_id = serializers.IntegerField(help_text="The collection ID in Kingfisher Process")
@@ -33,11 +58,11 @@ class FilterDatasetSerializer(serializers.Serializer):
 class CustomSchema(AutoSchema):
     def get_responses(self, path, method):
         responses = super().get_responses(path, method)
-        # POST requests return HTTP 202 with no content.
+        # POST requests return HTTP 202 with no content (not 201).
         if "201" in responses:
             responses["202"] = responses.pop("201")
             del responses["202"]["content"]
-        # GET requests return a JSON object.
+        # GET requests return a JSON object (not string).
         elif "200" in responses:
             responses["200"]["content"]["application/json"]["schema"] = {"type": "object"}
         return responses
@@ -57,10 +82,26 @@ def get_object(self):
         return get_object_or_404(self.get_queryset(), pk=self.kwargs["pk"])
 
     def get_serializer(self, *args, **kwargs):
-        if self.action == "create":
+        if self.action == "list":
+            return DatasetSerializer(*args, **kwargs)
+        elif self.action == "create":
             return CreateDatasetSerializer(*args, **kwargs)
         elif self.action == "filter":
             return FilterDatasetSerializer(*args, **kwargs)
+        else:
+            raise NotImplementedError(f"no serializer for action {self.action}")
+
+    def list(self, request, *args, **kwargs):
+        dataset_filter = DatasetFilter.objects.filter(dataset=OuterRef("pk"))[:1]
+        queryset = self.get_queryset().annotate(
+            phase=F("progress__phase"),
+            state=F("progress__state"),
+            parent_id=Subquery(dataset_filter.values("parent__id")),
+            parent_name=Subquery(dataset_filter.values("parent__name")),
+            filter_message=Subquery(dataset_filter.values("filter_message")),
+        )
+        serializer = self.get_serializer(queryset, many=True)
+        return Response(serializer.data)
 
     def create(self, request):
         """

backend/tests/api/test_views.py

@@ -0,0 +1,6 @@
+from tests import TestCase
+
+
+class ViewsTests(TestCase):
+    def test_dataset_list(self):
+        pass

docs/backend/reference/integration.rst

@@ -10,8 +10,8 @@ To update ``backend/api/models.py`` following changes to Pelican backend's datab
 -  Replace ``models.DO_NOTHING`` with ``on_delete=models.CASCADE``
 -  ``Dataset``: Add methods
 -  ``Dataset.meta``: Add ``blank=True, default=dict``
--  ``DatasetFilter.dataset_id_original``: Rename to ``dataset_original``, add ``related_name="dataset_filter_parent"``
--  ``DatasetFilter.dataset_id_filtered``: Rename to ``dataset_filtered``, add ``related_name="dataset_filter_child"``
+-  ``DatasetFilter.dataset_id_original``: Rename to ``parent``, add ``related_name="children"``
+-  ``DatasetFilter.dataset_id_filtered``: Rename to ``dataset``, add ``related_name="filtered"``
 -  ``ProgressMonitorDataset.dataset``: Add ``related_name="progress"``
 -  ``ProgressMonitorItem.item``: Rename to ``data_item``
 -  ``Report.type``: Change ``TextField`` to ``CharField``, add ``max_length=255``, and remove ``# This field type is a guess.``

frontend/src/components/DatasetPicker.vue

@@ -13,7 +13,7 @@
       <div class="thr row">
         <div
           class="th col-4 align-self-center clickable"
-          @click="sortBy('id')"
+          @click="sortBy('name')"
         >
           <SortButtons
             :label="$t('dataset.name')"
@@ -173,10 +173,10 @@ export default {
         }
     },
     mounted() {
-        var buildDatasetsTree = function (datasets, filtered_parent_id) {
+        var buildDatasetsTree = function (datasets, parent_id) {
             var result = [];
             datasets.forEach(function (item) {
-                if (item.filtered_parent_id == filtered_parent_id) {
+                if (item.parent_id == parent_id) {
                     item.filtered_children = buildDatasetsTree(datasets, item.id);
                     result.push(item);
                 }
@@ -188,7 +188,7 @@ export default {
         axios
             .get(url)
             .then(response => {
-                this.datasets = buildDatasetsTree(response["data"]["objects"], null);
+                this.datasets = buildDatasetsTree(response["data"], null);
 
                 var self = this;
                 this.datasets.forEach(function (item) {
@@ -228,12 +228,16 @@ export default {
             } else if (by == "name") {
                 comp = (a, b) => a.name.localeCompare(b.name);
             } else if (by == "size") {
-                comp = (a, b) => this.compareNumbers(a.size, b.size);
+                comp = (a, b) =>
+                    this.compareNumbers(
+                        a.meta.compiled_releases?.total_unique_ocids || -1,
+                        b.meta.compiled_releases?.total_unique_ocids || -1
+                    );
             } else if (by == "collection_id") {
                 comp = (a, b) =>
                     this.compareNumbers(
-                        a.meta.kingfisher_metadata.collection_id,
-                        b.meta.kingfisher_metadata.collection_id
+                        a.meta.kingfisher_metadata?.collection_id || -1,
+                        b.meta.kingfisher_metadata?.collection_id || -1
                     );
             } else if (by == "phase") {
                 comp = (a, b) => {

frontend/src/components/DatasetPickerRow.vue

@@ -4,27 +4,20 @@
       :class="['row', 'tr', 'align-items-center']"
     >
       <div class="td col-4">
-        <span v-if="depth > 0">
-          <span
-            v-for="d in depth"
-            :key="d"
-          >&nbsp;&nbsp;</span>
-
-          <font-awesome-icon :icon="['fas', 'long-arrow-alt-right']" />&nbsp;&nbsp;&nbsp;&nbsp;
+        <span v-if="depth > 0" :style="{'padding-left': depth / 2 + 'rem'}">
+          <font-awesome-icon :icon="['fas', 'long-arrow-alt-right']" />
         </span>
         <b-link :to="{ name: 'overview', params: { datasetId: dataset.id } }" :disabled="!isDatasetImported(dataset)">
           {{ dataset.name }}
         </b-link>
-        <span class="dataset_id"> (Id {{ dataset.id }})</span>
-        <span v-if="depth == 0">&nbsp;</span>
+        <span class="dataset_id">(Id {{ dataset.id }})</span>
         <a
           v-if="isDatasetImported(dataset) && depth == 0"
           href="#"
           @click.stop.prevent="$emit('dataset-filter', dataset)"
         >
           <font-awesome-icon :icon="['fas', 'filter']" />
         </a>
-        &nbsp;
         <a
           v-if="isDatasetImported(dataset)"
           href="#"
@@ -34,13 +27,10 @@
         </a>
       </div>
       <div class="td col-1 numeric text-right">
-        {{ dataset.size | formatNumber }}
+        {{ dataset.meta.compiled_releases?.total_unique_ocids | formatNumber }}
       </div>
-      <div
-        v-if="dataset.meta.kingfisher_metadata"
-        class="td col-1 numeric text-right"
-      >
-        {{ dataset.meta.kingfisher_metadata.collection_id }}
+      <div class="td col-1 numeric text-right">
+        {{ dataset.meta.kingfisher_metadata?.collection_id }}
       </div>
       <div class="td col-1 phase_cell align-items-center align-middle">
         <template v-if="dataset.phase == 'CHECKED' && dataset.state == 'OK'">

frontend/src/config.js

@@ -1,17 +1,18 @@
 var config = {
     apiBaseUrl: process.env.VUE_APP_API_BASE_URL,
     apiEndpoints: {
-        dataset: "api/dataset",
+        // List/detail
+        dataset: "datasets/",
         dataItem: "api/data_item",
-        // Stats
+        // Check stats
         fieldStats: "api/field_level_stats",
         resourceLevelStats: "api/resource_level_stats",
         datasetLevelStats: "api/dataset_level_stats",
         timeVarianceLevelStats: "api/time_variance_level_stats",
-        // Detail
+        // Check detail
         fieldDetail: "api/field_level_detail",
         resourceLevelCheckDetail: "api/resource_level_detail",
-        // Rest
+        // Filter datasets
         datasetFilterItems: "api/dataset_filter_items",
         datasetDistinctValues: "api/dataset_distinct_values",
         // Controller

frontend/src/plugins/numeral.js

@@ -3,6 +3,9 @@ import Vue from "vue";
 var numeral = require("numeral");
 
 Vue.filter("formatNumber", function (value) {
+    if (value === undefined) {
+        return value;
+    }
     return numeral(value).format("0,0");
 });
 

frontend/src/store.js

@@ -266,7 +266,7 @@ export default new Vuex.Store({
     },
     actions: {
         loadDataset({ dispatch, commit }, datasetId) {
-            var url = CONFIG.apiBaseUrl + CONFIG.apiEndpoints.dataset + "/" + datasetId;
+            var url = CONFIG.apiBaseUrl + CONFIG.apiEndpoints.dataset + datasetId;
 
             return new Promise(resolve => {
                 axios

frontend/src/views/Overview.vue

@@ -21,8 +21,8 @@
                 {{ $t("overview.filtered.original") }}
               </div>
               <div class="td col col-6">
-                {{ dataset.filtered_parent_name }}
-                <span class="dataset_id">(Id {{ dataset.filtered_parent_id }})</span>&nbsp;
+                {{ dataset.parent_name }}
+                <span class="dataset_id">(Id {{ dataset.parent_id }})</span>
               </div>
             </div>
             <div class="tr row">