From 2f5be56b12baecd77fa893ec33a70d349755b2db Mon Sep 17 00:00:00 2001 From: red Date: Fri, 6 Jan 2023 15:50:20 +0000 Subject: [PATCH 1/6] Update container base to Ubuntu 20.04 --- Dockerfile.concourse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.concourse b/Dockerfile.concourse index 005465f9..55d47bde 100644 --- a/Dockerfile.concourse +++ b/Dockerfile.concourse @@ -1,4 +1,4 @@ -FROM onsdigital/dp-concourse-tools-ubuntu +FROM onsdigital/dp-concourse-tools-ubuntu-20:ubuntu20.4-rc.1 WORKDIR /app/ From 00c24a5a02a0c68fcfe3f2d26a221cd9d9c488bd Mon Sep 17 00:00:00 2001 From: Rafa Date: Fri, 13 Jan 2023 13:24:19 +0000 Subject: [PATCH 2/6] Assert all fields in dataset --- features/private_datasets.feature | 16 +++++++++++++--- features/steps/steps.go | 11 +++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index 86a7c189..4be613f1 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -9,7 +9,9 @@ Feature: Private Dataset API When I POST "/datasets/ageing-population-estimates" """ { - "title": "CID" + "state": "anything", + "title": "CID", + "type": "filterable" } """ Then the HTTP status code should be "201" @@ -19,7 +21,15 @@ Feature: Private Dataset API "id": "ageing-population-estimates", "state": "created", "title": "CID", - "type": "filterable" + "type": "filterable", + "links": { + "editions": { + "href":"http://localhost:22000/datasets/ageing-population-estimates/editions" + }, + "self": { + "href":"http://localhost:22000/datasets/ageing-population-estimates" + } + } } """ @@ -120,7 +130,7 @@ Feature: Private Dataset API "id": "population-estimates", "related_content": [{ "description": "Related content description", - "href": "http://localhost:22000//datasets/123/relatedContent", + "href": "http://localhost:22000/datasets/123/relatedContent", "title": "Related content" }] } diff --git a/features/steps/steps.go b/features/steps/steps.go index 02f0945c..0488cc12 100644 --- a/features/steps/steps.go +++ b/features/steps/steps.go @@ -72,8 +72,11 @@ func (c *DatasetComponent) theDocumentInTheDatabaseForIdShouldBe(documentId stri document := link.Next - assert.Equal(&c.ErrorFeature, expectedDataset.Title, document.Title) - assert.Equal(&c.ErrorFeature, expectedDataset.State, document.State) + // Remove the last updated value so to be able to compare the datasets + // otherwise the assertion would always fail as last updated would be "now" + document.LastUpdated = time.Time{} + + assert.Equal(&c.ErrorFeature, expectedDataset, *document) return c.ErrorFeature.StepError() } @@ -150,8 +153,8 @@ func (c *DatasetComponent) theseGenerateDownloadsEventsAreProduced(events *godog return nil } -//we are passing the string array as [xxxx,yyyy,zzz] -//this is required to support array being used in kafka messages +// we are passing the string array as [xxxx,yyyy,zzz] +// this is required to support array being used in kafka messages func arrayParser(raw string) (interface{}, error) { //remove the starting and trailing brackets str := strings.Trim(raw, "[]") From ed000f74f8c7d4b1f76b51ad707207c0467400b9 Mon Sep 17 00:00:00 2001 From: Rafa Date: Fri, 3 Feb 2023 11:35:40 +0000 Subject: [PATCH 3/6] Remove links.versions They are not stored because VersionLinks omits this field with json tags --- features/versions.feature | 4 ---- 1 file changed, 4 deletions(-) diff --git a/features/versions.feature b/features/versions.feature index 71ad08ae..1d11d97f 100644 --- a/features/versions.feature +++ b/features/versions.feature @@ -173,10 +173,6 @@ Feature: Dataset API }, "self": { "href": "someurl" - }, - "version": { - "id": "1", - "href": "someurl" } }, "edition": "2021", From f3f9f92cdce9d030d02bdf683912f853592d77b2 Mon Sep 17 00:00:00 2001 From: Nathan Shumoogum Date: Tue, 14 Feb 2023 16:32:11 +0000 Subject: [PATCH 4/6] Metadata endpoint now returns canonical topic and subtopics --- models/metadata.go | 4 ++++ swagger.yaml | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/models/metadata.go b/models/metadata.go index 7f6cc75a..e6046720 100644 --- a/models/metadata.go +++ b/models/metadata.go @@ -9,6 +9,7 @@ import ( // Metadata represents information (metadata) relevant to a version type Metadata struct { Alerts *[]Alert `json:"alerts,omitempty"` + CanonicalTopic string `json:"canonical_topic,omitempty"` Contacts []ContactDetails `json:"contacts,omitempty"` Description string `json:"description,omitempty"` Dimensions []Dimension `json:"dimensions,omitempty"` @@ -30,6 +31,7 @@ type Metadata struct { Temporal *[]TemporalFrequency `json:"temporal,omitempty"` Theme string `json:"theme,omitempty"` Title string `json:"title,omitempty"` + Subtopics []string `json:"subtopics,omitempty"` UnitOfMeasure string `json:"unit_of_measure,omitempty"` URI string `json:"uri,omitempty"` UsageNotes *[]UsageNote `json:"usage_notes,omitempty"` @@ -58,6 +60,7 @@ type MetadataLinks struct { func CreateMetaDataDoc(datasetDoc *Dataset, versionDoc *Version, urlBuilder *url.Builder) *Metadata { metaDataDoc := &Metadata{ Alerts: versionDoc.Alerts, + CanonicalTopic: datasetDoc.CanonicalTopic, Contacts: datasetDoc.Contacts, Description: datasetDoc.Description, Dimensions: versionDoc.Dimensions, @@ -75,6 +78,7 @@ func CreateMetaDataDoc(datasetDoc *Dataset, versionDoc *Version, urlBuilder *url RelatedDatasets: datasetDoc.RelatedDatasets, ReleaseDate: versionDoc.ReleaseDate, ReleaseFrequency: datasetDoc.ReleaseFrequency, + Subtopics: datasetDoc.Subtopics, Temporal: versionDoc.Temporal, Theme: datasetDoc.Theme, Title: datasetDoc.Title, diff --git a/swagger.yaml b/swagger.yaml index f8c8d7e0..767217ca 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1770,6 +1770,9 @@ definitions: type: array items: $ref: '#/definitions/Alert' + canonical_topic: + description: "The canonical topic id for this dataset. This indicates which topic this dataset belongs to within the website taxonomy." + type: string contacts: description: "A list containing contact details of staticians for a dataset" type: array @@ -1889,6 +1892,11 @@ definitions: release_frequency: description: "The release frequency of a dataset" type: string + subtopics: + description: "A list of subtopic ids that the dataset relates to within the website taxonomy." + type: array + items: + type: "string" temporal: $ref: '#/definitions/Temporal' theme: From 37007594e338752802023d82ec48b22f335ce3cf Mon Sep 17 00:00:00 2001 From: Nathan Shumoogum Date: Tue, 14 Feb 2023 16:45:03 +0000 Subject: [PATCH 5/6] Correct expected metadata output in unit test --- models/test_data.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/models/test_data.go b/models/test_data.go index 2b7cfffc..111b2337 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -257,15 +257,16 @@ var badInputData = struct { func expectedMetadataDoc() Metadata { return Metadata{ - Alerts: &[]Alert{alert}, - Contacts: []ContactDetails{contacts}, - Description: "census", - Dimensions: []Dimension{dimension}, - Distribution: []string{"json", "csv", "xls"}, - Downloads: &downloads, - Keywords: []string{"test", "test2"}, - LatestChanges: &[]LatestChange{latestChange}, - License: "Office of National Statistics license", + Alerts: &[]Alert{alert}, + CanonicalTopic: "canonicalTopicID", + Contacts: []ContactDetails{contacts}, + Description: "census", + Dimensions: []Dimension{dimension}, + Distribution: []string{"json", "csv", "xls"}, + Downloads: &downloads, + Keywords: []string{"test", "test2"}, + LatestChanges: &[]LatestChange{latestChange}, + License: "Office of National Statistics license", Links: &MetadataLinks{ AccessRights: &LinkObject{ HRef: "http://ons.gov.uk/accessrights", @@ -286,6 +287,7 @@ func expectedMetadataDoc() Metadata { RelatedDatasets: []GeneralDetails{relatedDatasets}, ReleaseDate: "2017-10-12", ReleaseFrequency: "yearly", + Subtopics: []string{"subtopicID"}, Temporal: &[]TemporalFrequency{temporal}, Theme: "population", Title: "CensusEthnicity", From 4b706b7ef71d22b940748399f60b9578e37cc435 Mon Sep 17 00:00:00 2001 From: Nathan Shumoogum Date: Tue, 14 Feb 2023 17:12:09 +0000 Subject: [PATCH 6/6] Additional mapping of metadata model for Cantabular datasets to include subtopics and canonical_topic --- models/metadata.go | 8 +++++--- models/metadata_test.go | 10 ++++++---- models/test_data.go | 22 ++++++++++++---------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/models/metadata.go b/models/metadata.go index e6046720..4adaea84 100644 --- a/models/metadata.go +++ b/models/metadata.go @@ -140,19 +140,21 @@ func CreateMetaDataDoc(datasetDoc *Dataset, versionDoc *Version, urlBuilder *url func CreateCantabularMetaDataDoc(d *Dataset, v *Version, urlBuilder *url.Builder) *Metadata { m := &Metadata{ CSVHeader: v.Headers, + CanonicalTopic: d.CanonicalTopic, + Contacts: d.Contacts, + DatasetLinks: d.Links, Description: d.Description, Dimensions: v.Dimensions, Downloads: v.Downloads, Keywords: d.Keywords, + RelatedContent: d.RelatedContent, ReleaseDate: v.ReleaseDate, + Subtopics: d.Subtopics, Title: d.Title, UnitOfMeasure: d.UnitOfMeasure, - Contacts: d.Contacts, URI: d.URI, QMI: d.QMI, Version: v.Version, - DatasetLinks: d.Links, - RelatedContent: d.RelatedContent, } m.Distribution = getDistribution(m.Downloads) diff --git a/models/metadata_test.go b/models/metadata_test.go index f4a7b025..03220f38 100644 --- a/models/metadata_test.go +++ b/models/metadata_test.go @@ -39,10 +39,12 @@ func TestCreateCantabularMetadataDoc(t *testing.T) { Convey("Successfully create metadata document with all relavant cantabular fields", t, func() { inputDatasetDoc := &Dataset{ - Description: "census", - Keywords: []string{"test", "test2"}, - Title: "CensusEthnicity", - UnitOfMeasure: "Pounds Sterling", + CanonicalTopic: "1234", + Description: "census", + Keywords: []string{"test", "test2"}, + Subtopics: []string{"5678", "9012"}, + Title: "CensusEthnicity", + UnitOfMeasure: "Pounds Sterling", } inputVersionDoc := &publishedVersion diff --git a/models/test_data.go b/models/test_data.go index 111b2337..a71135fe 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -298,15 +298,17 @@ func expectedMetadataDoc() Metadata { func expectedCantabularMetadataDoc() Metadata { return Metadata{ - Description: "census", - Dimensions: []Dimension{dimension}, - CSVHeader: []string{"cantabular_table", "age"}, - Distribution: []string{"json", "csv", "csvw", "xls"}, - Downloads: &cantabularDownloads, - Keywords: []string{"test", "test2"}, - ReleaseDate: "2017-10-12", - Title: "CensusEthnicity", - UnitOfMeasure: "Pounds Sterling", - Version: 1, + CSVHeader: []string{"cantabular_table", "age"}, + CanonicalTopic: "1234", + Description: "census", + Dimensions: []Dimension{dimension}, + Distribution: []string{"json", "csv", "csvw", "xls"}, + Downloads: &cantabularDownloads, + Keywords: []string{"test", "test2"}, + ReleaseDate: "2017-10-12", + Subtopics: []string{"5678", "9012"}, + Title: "CensusEthnicity", + UnitOfMeasure: "Pounds Sterling", + Version: 1, } }