Merge pull request #4 from PDOK/PDOK-10867

PDOK-10867 bepalen updated velden op basis van minio bestanden

atom_generator/minio_client.py

@@ -1,3 +1,4 @@
+import time
 import warnings
 import io
 import mimetypes
@@ -157,6 +158,34 @@ def source_object_size(self, filename):
             )
         return object_stat.size
 
+    def source_object_date(self, filename):
+        """
+        Gets the last_modified date from a file on the given minio source.
+
+        The file will be taken from the bucket and prepended with the given
+        source_prefix.
+
+        Args:
+            filename: filename in the minio source bucket without the source prefix.
+
+        Returns:
+            str: the last_modified date from the file formatted as a string.
+        """
+        if not filename.strip():
+            raise ValueError("object_path can not be empty")
+
+        key = build_uri(self.source_prefix, filename)
+        object_stat = self._client.stat_object(self.source_bucket, key)
+
+        if not object_stat:
+            raise AppError(f"could not find: {key} in {self.source_bucket}")
+        if object_stat.is_dir:
+            raise AppError(
+                f"object_path '{key} in {self.source_bucket}' points to directory -> should point to file"
+            )
+        # format time.struct_time to a str
+        return time.strftime("%Y-%m-%dT%H:%M:%SZ", object_stat.last_modified)
+
     def __str__(self):
         return (
             f"Minio DAO from: {{ {self.source_bucket} }}/{{ {self.source_prefix} }}, "

atom_generator/models.py

@@ -1,5 +1,4 @@
 import os
-import datetime
 from dataclasses import dataclass, field
 from typing import List, Dict, Optional
 
@@ -73,6 +72,10 @@ def download_mimetype(self):
     def download_length(self):
         return self._minio.source_object_size(self.download_file.lstrip("/"))
 
+    @property
+    def download_date(self):
+        return self._minio.source_object_date(self.download_file.lstrip("/"))
+
 
 @nested
 @dataclass
@@ -133,7 +136,7 @@ class ServiceFeed:
     @property
     def updated(self):
         if self.__updated is None:
-            self.__updated = f"{datetime.datetime.utcnow().isoformat()}Z"
+            self.__updated = self.datasets[0].downloads[0].download_date
         return self.__updated
 
     @property

tests/conftest.py

@@ -1,3 +1,4 @@
+import time
 import pytest
 import minio
 from pathlib import Path
@@ -16,6 +17,7 @@ def __init__(self, object_name="test.xml", is_dir=False, size=3):
             self.object_name = object_name
             self.is_dir = is_dir
             self.size = size
+            self.last_modified = time.strptime("Tue Aug 25 13:45:00 2020")
 
     mock_object = None
 

tests/data/expected_values.json

@@ -37,6 +37,7 @@
           "download_file": "gpkg_part.gpkg",
           "download_id": "http://example.com/base/test/path/downloads/gpkg_part.gpkg",
           "download_length": 3,
+          "download_date": "2020-08-25T13:45:00Z",
           "download_mimetype": "application/geopackage+sqlite3",
           "download_title": "Top 10 NL v2 (Geografische Namen) - Parent Feed (CRS) - GeoPackage download (EPSG:28992)",
           "download_url": "http://example.com/base/test/path/downloads/gpkg_part.gpkg",

tests/data/expected_values_old.json

@@ -37,6 +37,7 @@
           "download_file": "gpkg_part.gpkg",
           "download_id": "http://example.com/base/old/test/file/downloads/gpkg_part.gpkg",
           "download_length": 3,
+          "download_date": "2020-08-25T13:45:00Z",
           "download_mimetype": "application/geopackage+sqlite3",
           "download_title": "Top 10 NL v2 (Geografische Namen) - Parent Feed (CRS) - GeoPackage download (EPSG:28992)",
           "download_url": "http://example.com/base/old/test/file/downloads/gpkg_part.gpkg",

tests/test_minio.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import pytest
 import mimetypes
 
@@ -31,6 +32,19 @@ def test_source_media_type_unknown_raises_app_error(minio_empty):
     pytest.raises(AppError, minio_dao.source_media_type, "test.does_not_exist")
 
 
+def test_source_object_date(minio_new):
+    minio_dao = minio_new
+    last_modified = minio_dao.source_object_date("test.gpkg")
+    assert last_modified == "2020-08-25T13:45:00Z"
+
+
+def test_source_object_date_format_does_not_fail(minio_new):
+    minio_dao = minio_new
+    last_modified = minio_dao.source_object_date("test.gpkg")
+    # this throws a ValueError Exception when the date does not match the format
+    datetime.strptime(last_modified, "%Y-%m-%dT%H:%M:%SZ")
+
+
 def test_source_object_size(minio_new):
     minio_dao = minio_new
     size = minio_dao.source_object_size("test.gpkg")