Merge pull request #16 from PDOK/updated-field

Added functionality for most recent updated field

atom.go

@@ -40,12 +40,7 @@ func main() {
 		log.Fatalf("error: %v", err)
 	}
 
-	var processedFeeds []feeds.Feed
-
-	// process Feeds
-	for _, feed := range config.Feeds {
-		processedFeeds = append(processedFeeds, feeds.ProcessFeed(feed))
-	}
+	processedFeeds := feeds.ProcessFeeds(config)
 
 	// write both service and dataset feeds
 	for _, feed := range processedFeeds {

example/feeds/dataset.xml

@@ -18,7 +18,7 @@
     <!-- rights, access restrictions -->
     <rights>Copyright (c) 2012, XYZ; all rights reserved</rights>
     <!-- date/time this feed was last updated -->
-    <updated>2012-03-31T13:45:03Z</updated>
+    <updated>2013-03-31T13:45:03Z</updated>
     <!-- author contact information -->
     <author>
         <name>John Doe</name>

example/feeds/service.xml

@@ -21,7 +21,7 @@
     <!-- rights, access restrictions -->
     <rights>Copyright (c) 2012, XYZ; all rights reserved</rights>
     <!-- date/time this feed was last updated -->
-    <updated>2012-03-31T13:45:03Z</updated>
+    <updated>2013-03-31T13:45:03Z</updated>
     <!-- author contact information -->
     <author>
         <name>John Doe</name>

example/inspire/xyz-example.yaml

@@ -31,7 +31,6 @@ feeds:
       hreflang: de
       title: "An HTML version of this document in German"
    rights: "Copyright (c) 2012, XYZ; all rights reserved"
-   updated: "2012-03-31T13:45:03Z"
    author:
      name: "John Doe"
      email: "[email protected]"
@@ -51,9 +50,8 @@ feeds:
          href: "http://xyz.org/wfs?request=GetCapabilities&service=WFS&version=2.0.0"
          type: "application/xml"
          title: "Service implementing Direct Access operations"
-      id: "http://xyz.org/data/waternetwork_feed.xml"
+      id: "http://xyz.org/data/abc/waternetwork.xml"
       rights: "Copyright (c) 2002-2011, XYZ; all rights reserved"
-      updated: "2012-03-31T13:45:03Z"
       summary: "This is the entry for water network ABC Dataset"
       polygon: "47.202 5.755 55.183 5.755 55.183 15.253 47.202 15.253 47.202 5.755"
       category:
@@ -86,7 +84,6 @@ feeds:
       type: "application/atom+xml"
       title: "The parent service feed document"
    rights: "Copyright (c) 2012, XYZ; all rights reserved"
-   updated: "2012-03-31T13:45:03Z"
    author:
      name: "John Doe"
      email: "[email protected]"
@@ -99,7 +96,7 @@ feeds:
          length: 34987
          title: "Water network dataset encoded as a GML 3.2 document in ETRS89 UTM zone 32N (http://www.opengis.net/def/crs/EPSG/0/25832)"
       id: "http://xyz.org/data/abc/waternetwork_25832.gml"
-      updated: "2011-06-15T11:12:34Z"
+      updated: "2014-06-15T11:12:34Z"
       category:
        - term: "http://www.opengis.net/def/crs/EPSG/0/25832"
          label: "ETRS89 / UTM zone 32N"
@@ -111,7 +108,7 @@ feeds:
          length: 37762
          title: "Water Network encoded as a GML 3.2 document in WGS84 geographic coordinates (http://www.opengis.net/def/crs/OGC/1.3/CRS84)"
       id: "http://xyz.org/data/abc/waternetwork_WGS84.gml"
-      updated: "2011-06-14T12:22:09Z"
+      updated: "2015-06-14T12:22:09Z"
       category:
        - term: "http://www.opengis.net/def/crs/EPSG/0/4258"
          label: ETRS89
@@ -123,7 +120,7 @@ feeds:
          length: 89274
          title: "Water network dataset encoded as a ShapeFile in ETRS89 UTM zone 32N (http://www.opengis.net/def/crs/EPSG/0/25832)"
       id: "http://xyz.org/data/abc/waternetwork_25832.zip"
-      updated: "2011-06-15T11:12:34Z"
+      updated: "2016-06-15T11:12:34Z"
       category:
        - term: "http://www.opengis.net/def/crs/EPSG/0/25832"
          label: "ETRS89 / UTM zone 32N"
@@ -135,7 +132,7 @@ feeds:
          length: 78973
          title: "Water Network encoded as a ShapeFile in WGS84 geographic coordinates (http://www.opengis.net/def/crs/OGC/1.3/CRS84)"
       id: "http://xyz.org/data/abc/waternetwork_WGS84.zip"
-      updated: "2011-06-14T12:22:09Z"
+      updated: "2018-06-14T12:22:09Z"
       category:
        - term: "http://www.opengis.net/def/crs/EPSG/0/4258"
          label: ETRS89

feeds/defaults.go

@@ -17,6 +17,7 @@ const (
 	invalidrights   = "invalid 'rights', cannot be empty see TG Requirement 10"
 	invaliddatetime = "invalid 'updated', needs to be a valid datetime with timezone see TG Requirement 11"
 	invalidauthor   = "invalid 'author', cannot be empty see TG Requirement 12"
+	invalidupdated  = "invalid 'updated', updated is required see TG Requirements 11"
 )
 
 // GetDefaultFeedProperties returns mandatory/static ServiceFeed properties

feeds/feeds.go

@@ -5,10 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // Feeds struct
@@ -38,7 +40,7 @@ type Feed struct {
 	Link []Link `xml:"link" yaml:"link"`
 
 	Rights  string  `xml:"rights" yaml:"rights"`
-	Updated string  `xml:"updated" yaml:"updated"`
+	Updated *string `xml:"updated" yaml:"updated,omitempty"`
 	Author  Author  `xml:"author" yaml:"author"`
 	Entry   []Entry `xml:"entry" yaml:"entry"`
 }
@@ -105,7 +107,19 @@ func (f *Feed) Valid() error {
 
 	// TG Requirement 11
 	// The 'updated' element of a feed shall contain the date, time and timezone at which the feed was last updated.
-	if _, err := time.Parse(`2006-01-02T15:04:05Z`, f.Updated); err != nil {
+
+	for _, entry := range f.Entry {
+		if entry.Updated == nil {
+			return errors.New(invaliddatetime)
+		}
+		if _, err := time.Parse(`2006-01-02T15:04:05Z`, *entry.Updated); err != nil {
+			return errors.New(invaliddatetime)
+		}
+	}
+	if f.Updated == nil {
+		return errors.New(invaliddatetime)
+	}
+	if _, err := time.Parse(`2006-01-02T15:04:05Z`, *f.Updated); err != nil {
 		return errors.New(invaliddatetime)
 	}
 
@@ -118,6 +132,50 @@ func (f *Feed) Valid() error {
 	return nil
 }
 
+// Function that retrieves values from updated fields and returns the most recent updated field
+func (f *Feed) recentUpdated(feeds Feeds) {
+	for index, entry := range f.Entry {
+		if entry.Updated == nil {
+			nestedfeed := entry.nestedFeed(feeds.Feeds)
+			if nestedfeed != nil {
+				updated := nestedfeed.recentUpdatedEntry()
+				f.Entry[index].Updated = updated
+			}
+		}
+	}
+	if f.Updated == nil {
+		feedUpdated := f.recentUpdatedEntry()
+		f.Updated = feedUpdated
+	}
+}
+
+func (e Entry) nestedFeed(feeds []Feed) *Feed {
+	for _, feed := range feeds {
+		if e.ID == feed.ID {
+			return &feed
+		}
+	}
+	return nil
+}
+
+func (f *Feed) recentUpdatedEntry() *string {
+	var updatedfields []string
+
+	for _, entry := range f.Entry {
+		if entry.Updated == nil {
+		} else {
+			updatedfields = append(updatedfields, *entry.Updated)
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(updatedfields)))
+	if len(updatedfields) == 0 {
+		return nil
+	} else {
+		return &updatedfields[0]
+	}
+
+}
+
 // Entry struct
 type Entry struct {
 	ID                                string     `xml:"id" yaml:"id"`
@@ -126,7 +184,7 @@ type Entry struct {
 	Summary                           string     `xml:"summary,omitempty" yaml:"summary"`
 	Link                              []Link     `xml:"link" yaml:"link"`
 	Rights                            string     `xml:"rights,omitempty" yaml:"rights"`
-	Updated                           string     `xml:"updated" yaml:"updated"`
+	Updated                           *string    `xml:"updated" yaml:"updated,omitempty"`
 	Polygon                           string     `xml:"georss:polygon,omitempty" yaml:"polygon"`
 	Category                          []Category `xml:"category" yaml:"category"`
 	SpatialDatasetIdentifierCode      string     `xml:"inspire_dls:spatial_dataset_identifier_code,omitempty" yaml:"spatial_dataset_identifier_code"`

feeds/feeds_test.go

@@ -8,24 +8,14 @@ import (
 )
 
 func TestGenerateATOM(t *testing.T) {
+	var updated = "2012-03-31T13:45:03Z"
+	var recentupdated = "2021-10-01T00:00:00Z"
 	var tests = []struct {
-		input    Feed
+		input    Feeds
+		updated  *string
 		expected string
 	}{
-		0: {input: Feed{},
-			expected: `<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:georss="http://www.georss.org/georss" xml:lang="en">
- <id></id>
- <title></title>
- <subtitle></subtitle>
- <rights></rights>
- <updated></updated>
- <author>
-  <name></name>
-  <email></email>
- </author>
-</feed>`},
-		1: {input: Feed{InspireDls: "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0",
+		0: {input: Feeds{Feeds: []Feed{Feed{InspireDls: "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0",
 			Lang:     sp("en"),
 			ID:       "http://xyz.org/download/en.xml",
 			Title:    "XYZ Example INSPIRE Download Service",
@@ -68,7 +58,7 @@ func TestGenerateATOM(t *testing.T) {
 				},
 			},
 			Rights:  "Copyright (c) 2012, XYZ; all rights reserved",
-			Updated: "2012-03-31T13:45:03Z",
+			Updated: &updated,
 			Author: Author{
 				Name:  "John Doe",
 				Email: "[email protected]",
@@ -77,7 +67,7 @@ func TestGenerateATOM(t *testing.T) {
 				{
 					ID:                                "http://xyz.org/data/waternetwork_feed.xml",
 					Rights:                            "Copyright (c) 2002-2011, XYZ; all rights reserved",
-					Updated:                           "2012-03-31T13:45:03Z",
+					Updated:                           &updated,
 					Summary:                           "This is the entry for water network ABC Dataset",
 					Polygon:                           "47.202 5.755 55.183 5.755 55.183 15.253 47.202 15.253 47.202 5.755",
 					Title:                             "Water network ABC Dataset Feed",
@@ -114,6 +104,8 @@ func TestGenerateATOM(t *testing.T) {
 					},
 				},
 			},
+		},
+		},
 		},
 			expected: `<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom" xmlns:georss="http://www.georss.org/georss" xmlns:inspire_dls="http://inspire.ec.europa.eu/schemas/inspire_dls/1.0" xml:lang="en">
@@ -148,7 +140,7 @@ func TestGenerateATOM(t *testing.T) {
   <inspire_dls:spatial_dataset_identifier_namespace>http://xyz.org/</inspire_dls:spatial_dataset_identifier_namespace>
  </entry>
 </feed>`},
-		2: {input: Feed{InspireDls: "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0",
+		1: {input: Feeds{Feeds: []Feed{Feed{InspireDls: "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0",
 			Lang:     sp("nl"),
 			ID:       "https://service.pdok.nl/kadaster/plu/atom/v1_0/plu.xml",
 			Title:    "INSPIRE Download Service van Ruimtelijke plannen",
@@ -177,7 +169,7 @@ func TestGenerateATOM(t *testing.T) {
 				},
 			},
 			Rights:  "http://creativecommons.org/publicdomain/zero/1.0/deed.nl",
-			Updated: "2021-10-01T00:00:00Z",
+			Updated: &recentupdated,
 			Author: Author{
 				Name:  "PDOK Beheer",
 				Email: "[email protected]",
@@ -186,7 +178,7 @@ func TestGenerateATOM(t *testing.T) {
 				{
 					ID:      "https://service.pdok.nl/kadaster/plu/atom/v1_0/plu.xml",
 					Rights:  "http://creativecommons.org/publicdomain/zero/1.0/deed.nl",
-					Updated: "2021-10-01T00:00:00Z",
+					Updated: &recentupdated,
 					Polygon: "50.6 3.1 50.6 7.3 53.7 7.3 53.7 3.1 50.6 3.1",
 					Title:   "INSPIRE Download Service van Ruimtelijke plannen",
 					Content: "Bestand is opgesplitst per featuretype, elk featuretype heeft een eigen download bestand",
@@ -219,6 +211,7 @@ func TestGenerateATOM(t *testing.T) {
 				},
 			},
 		},
+		}},
 			expected: `<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom" xmlns:georss="http://www.georss.org/georss" xmlns:inspire_dls="http://inspire.ec.europa.eu/schemas/inspire_dls/1.0" xml:lang="nl">
  <id>https://service.pdok.nl/kadaster/plu/atom/v1_0/plu.xml</id>
@@ -250,8 +243,8 @@ func TestGenerateATOM(t *testing.T) {
 	}
 
 	for k, test := range tests {
-		p := ProcessFeed(test.input)
-		output := p.GenerateATOM()
+		p := ProcessFeeds(test.input)
+		output := p[0].GenerateATOM()
 		if string(output) != test.expected {
 			t.Errorf("test: %d, expected: %s \ngot: %s", k, test.expected, string(output))
 		}
@@ -282,6 +275,7 @@ func TestGetFileName(t *testing.T) {
 }
 
 func TestValid(t *testing.T) {
+	var updated = "2021-03-31T13:45:03Z"
 	var tests = []struct {
 		input    Feed
 		expected error
@@ -290,7 +284,7 @@ func TestValid(t *testing.T) {
 			input: Feed{
 				ID:      "http://xyz.org/download/en.xml",
 				Rights:  "Copyright (c) 2012, XYZ; all rights reserved",
-				Updated: "2012-03-31T13:45:03Z",
+				Updated: &updated,
 				Author: Author{
 					Name:  "John Doe",
 					Email: "[email protected]",
@@ -302,14 +296,14 @@ func TestValid(t *testing.T) {
 			input: Feed{
 				ID:      "http://xyz.org/download/en.xml",
 				Rights:  "Copyright (c) 2012, XYZ; all rights reserved",
-				Updated: "2012-03-31T13:45:03Z",
+				Updated: &updated,
 			},
 			expected: errors.New(invalidauthor),
 		},
 		2: {
 			input: Feed{
 				ID:      "http://xyz.org/download/en.xml",
-				Updated: "2012-03-31T13:45:03Z",
+				Updated: &updated,
 				Author: Author{
 					Name:  "John Doe",
 					Email: "[email protected]",
@@ -338,7 +332,6 @@ func TestValid(t *testing.T) {
 
 	for k, test := range tests {
 		b := test.input.Valid()
-
 		if b == nil {
 			if b != test.expected {
 				t.Errorf("test: %d, expected: %t \ngot: %t", k, test.expected, b)