From 7439ea1ddc32f1d8a40b288caf56387cb4fcccfb Mon Sep 17 00:00:00 2001 From: jitendar Date: Fri, 19 Aug 2022 15:12:07 +0100 Subject: [PATCH 01/24] Support for multivariate --- api/metadata.go | 2 +- instance/instance.go | 2 +- models/dataset.go | 13 ++++++++++++- models/dataset_test.go | 3 ++- service/service.go | 15 ++++++--------- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/api/metadata.go b/api/metadata.go index 71d2fc43..c7ab40c7 100644 --- a/api/metadata.go +++ b/api/metadata.go @@ -92,7 +92,7 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { var metaDataDoc *models.Metadata - if t == models.CantabularBlob || t == models.CantabularTable || t == models.CantabularFlexibleTable { + if t == models.CantabularBlob || t == models.CantabularTable || t == models.CantabularFlexibleTable || t == models.CantabularMultivariateTable { metaDataDoc = models.CreateCantabularMetaDataDoc(doc, versionDoc, api.urlBuilder) } else { // combine version and dataset metadata diff --git a/instance/instance.go b/instance/instance.go index bbf389d1..8a1b629d 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -268,7 +268,7 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { } // update dp-graph instance node (only for non-cantabular types) - if currentInstance.Type == models.CantabularBlob.String() || currentInstance.Type == models.CantabularTable.String() || currentInstance.Type == models.CantabularFlexibleTable.String() { + if currentInstance.Type == models.CantabularBlob.String() || currentInstance.Type == models.CantabularTable.String() || currentInstance.Type == models.CantabularFlexibleTable.String() || currentInstance.Type == models.CantabularMultivariateTable.String() { editionLogData["instance_type"] = instance.Type log.Info(ctx, "skipping dp-graph instance update because it is not required by instance type", editionLogData) } else { diff --git a/models/dataset.go b/models/dataset.go index 61dbc861..33a91233 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -30,10 +30,19 @@ const ( CantabularTable CantabularBlob CantabularFlexibleTable + CantabularMultivariateTable Invalid ) -var datasetTypes = []string{"filterable", "nomis", "cantabular_table", "cantabular_blob", "cantabular_flexible_table", "invalid"} +var datasetTypes = []string{ + "filterable", + "nomis", + "cantabular_table", + "cantabular_blob", + "cantabular_flexible_table", + "cantabular_multivariate_table", + "invalid", +} func (dt DatasetType) String() string { return datasetTypes[dt] @@ -52,6 +61,8 @@ func GetDatasetType(datasetType string) (DatasetType, error) { return CantabularBlob, nil case "cantabular_flexible_table": return CantabularFlexibleTable, nil + case "cantabular_multivariate_table": + return CantabularMultivariateTable, nil default: return Invalid, errs.ErrDatasetTypeInvalid } diff --git a/models/dataset_test.go b/models/dataset_test.go index bb500e7b..92a41fbc 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -63,7 +63,8 @@ func TestString(t *testing.T) { So(datasetTypes[2], ShouldEqual, "cantabular_table") So(datasetTypes[3], ShouldEqual, "cantabular_blob") So(datasetTypes[4], ShouldEqual, "cantabular_flexible_table") - So(datasetTypes[5], ShouldEqual, "invalid") + So(datasetTypes[5], ShouldEqual, "cantabular_multivariate_table") + So(datasetTypes[6], ShouldEqual, "invalid") }) }) diff --git a/service/service.go b/service/service.go index 34105a1d..c11d545c 100644 --- a/service/service.go +++ b/service/service.go @@ -145,11 +145,12 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin } downloadGenerators := map[models.DatasetType]api.DownloadsGenerator{ - models.CantabularBlob: downloadGeneratorCantabular, - models.CantabularTable: downloadGeneratorCantabular, - models.CantabularFlexibleTable: downloadGeneratorCantabular, - models.Filterable: downloadGeneratorCMD, - models.Nomis: downloadGeneratorCMD, + models.CantabularBlob: downloadGeneratorCantabular, + models.CantabularTable: downloadGeneratorCantabular, + models.CantabularFlexibleTable: downloadGeneratorCantabular, + models.CantabularMultivariateTable: downloadGeneratorCantabular, + models.Filterable: downloadGeneratorCMD, + models.Nomis: downloadGeneratorCMD, } // Get Identity Client (only if private endpoints are enabled) @@ -185,10 +186,6 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin svc.generateCantabularDownloadsProducer.Channels().LogErrors(ctx, "generate downloads producer error") } - // Log kafka producer errors in parallel go-routine - if svc.config.EnablePrivateEndpoints { - } - // Run the http server in a new go-routine go func() { if err := svc.server.ListenAndServe(); err != nil { From 45bc716ec8d59a8e4f9802f32caea22aa1b95732 Mon Sep 17 00:00:00 2001 From: Joshua Medley Date: Wed, 21 Sep 2022 13:03:47 +0100 Subject: [PATCH 02/24] make IsAreaType pointer for proper omitempty behaviour --- models/dimension.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/dimension.go b/models/dimension.go index 9396b2a1..169c9a27 100644 --- a/models/dimension.go +++ b/models/dimension.go @@ -14,7 +14,7 @@ type Dimension struct { Name string `bson:"name,omitempty" json:"name,omitempty"` Variable string `bson:"variable,omitempty" json:"variable,omitempty"` NumberOfOptions *int `bson:"number_of_options,omitempty" json:"number_of_options,omitempty"` - IsAreaType bool `bson:"is_area_type,omitempty" json:"is_area_type,omitempty"` + IsAreaType *bool `bson:"is_area_type,omitempty" json:"is_area_type,omitempty"` } // DimensionLink contains all links needed for a dimension From 90bc64624227bad1175fb8b617c4f1d9f14146ba Mon Sep 17 00:00:00 2001 From: Joshua Medley Date: Wed, 21 Sep 2022 15:44:17 +0100 Subject: [PATCH 03/24] component test for PUT /instances is_area_type --- features/instances.feature | 64 +++++++++++++++++++++++++++++++++++--- features/steps/steps.go | 20 ++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/features/instances.feature b/features/instances.feature index ef8e36fa..36e28ff2 100644 --- a/features/instances.feature +++ b/features/instances.feature @@ -1,4 +1,4 @@ -Feature: Dataset API + Feature: Dataset API Background: we have instances Given I have these instances: @@ -30,6 +30,15 @@ Feature: Dataset API "id": "income" } } + }, + { + "id": "test-item-4", + "state": "created", + "links": { + "dataset": { + "id": "other" + } + } } ] """ @@ -42,8 +51,20 @@ Feature: Dataset API Then I should receive the following JSON response with status "200": """ { - "count": 3, + "count": 4, "items": [ + { + "id": "test-item-4", + "import_tasks": null, + "last_updated": "2021-01-01T00:00:03Z", + "links": { + "dataset": { + "id": "other" + }, + "job": null + }, + "state": "created" + }, { "id": "test-item-3", "import_tasks": null, @@ -83,7 +104,7 @@ Feature: Dataset API ], "limit": 20, "offset": 0, - "total_count": 3 + "total_count": 4 } """ @@ -266,7 +287,42 @@ Feature: Dataset API And I set the "If-Match" header to "wrongValue" When I GET "/instances/test-item-1" Then the HTTP status code should be "409" - And I should receive the following response: + And I should receive the following response: """ instance does not match the expected eTag """ + Scenario: Updating instance with is_area_type explicitly false + Given private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + When I PUT "/instances" + """ + { + "id": "test-item-4", + "dimensions":[ + { + "name": "foo", + "is_area_type": false, + } + ] + } + """ + + Then the instance in the database for id "test-item-4" should be: + """ + { + "id": "test-item-4", + "state": "created", + "links": { + "dataset": { + "id": "other" + } + }, + "dimensions":[ + { + "name": "foo", + "is_area_type": false + } + ] + } + """ \ No newline at end of file diff --git a/features/steps/steps.go b/features/steps/steps.go index cb0d8977..02f0945c 100644 --- a/features/steps/steps.go +++ b/features/steps/steps.go @@ -31,6 +31,7 @@ func init() { func (c *DatasetComponent) RegisterSteps(ctx *godog.ScenarioContext) { ctx.Step(`^private endpoints are enabled$`, c.privateEndpointsAreEnabled) ctx.Step(`^the document in the database for id "([^"]*)" should be:$`, c.theDocumentInTheDatabaseForIdShouldBe) + ctx.Step(`^the instance in the database for id "([^"]*)" should be:$`, c.theInstanceInTheDatabaseForIdShouldBe) ctx.Step(`^there are no datasets$`, c.thereAreNoDatasets) ctx.Step(`^I have these datasets:$`, c.iHaveTheseDatasets) ctx.Step(`^I have these "([^"]*)" datasets:$`, c.iHaveTheseConditionalDatasets) @@ -77,6 +78,25 @@ func (c *DatasetComponent) theDocumentInTheDatabaseForIdShouldBe(documentId stri return c.ErrorFeature.StepError() } +func (c *DatasetComponent) theInstanceInTheDatabaseForIdShouldBe(id string, body *godog.DocString) error { + var expected models.Instance + + if err := json.Unmarshal([]byte(body.Content), &expected); err != nil { + return fmt.Errorf("failed to unmarshal body: %w", err) + } + + collectionName := c.MongoClient.ActualCollectionName(config.InstanceCollection) + var got models.Instance + + if err := c.MongoClient.Connection.Collection(collectionName).FindOne(context.Background(), bson.M{"_id": id}, &got); err != nil { + return fmt.Errorf("failed to get instance from collection: %w", err) + } + + assert.Equal(&c.ErrorFeature, expected, got) + + return nil +} + func (c *DatasetComponent) iHaveARealKafkaContainerWithTopic(topic string) error { c.setConsumer(topic) c.setInitialiserRealKafka() From e306710cf16de292bf0fe60433f32f97ae577708 Mon Sep 17 00:00:00 2001 From: Valentina Date: Tue, 20 Sep 2022 12:47:56 +0100 Subject: [PATCH 04/24] adds dataset topic tag fields --- models/dataset.go | 7 +++++++ mongo/dataset_store.go | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/models/dataset.go b/models/dataset.go index 33a91233..0488a0dd 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -110,6 +110,13 @@ type Dataset struct { Type string `bson:"type,omitempty" json:"type,omitempty"` NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` + CanonicalTopic *TopicTagObject `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` + SubTopics []TopicTagObject `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` +} + +type TopicTagObject struct { + ID string `bson:"id,omitempty" json:"id,omitempty"` + Title string `bson:"title,omitempty" json:"title,omitempty"` } // DatasetLinks represents a list of specific links related to the dataset resource diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index c7a00bc4..e149298d 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -414,6 +414,19 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } + if dataset.CanonicalTopic != nil { + if dataset.CanonicalTopic.ID != "" { + updates["next.canonical_topic.id"] = dataset.CanonicalTopic.ID + } + if dataset.CanonicalTopic.Title != "" { + updates["next.canonical_topic.title"] = dataset.CanonicalTopic.Title + } + } + + if dataset.SubTopics != nil { + updates["next.sub_topics"] = dataset.SubTopics + } + log.Info(ctx, "built update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) return updates From 71cbbb85b7c63cc21f386273779dc854f3767c73 Mon Sep 17 00:00:00 2001 From: Valentina Date: Wed, 21 Sep 2022 17:30:37 +0100 Subject: [PATCH 05/24] allows for the topic tag fields to be removed after added to the dataset --- mongo/dataset_store.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index e149298d..83877d17 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -414,21 +414,10 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - if dataset.CanonicalTopic != nil { - if dataset.CanonicalTopic.ID != "" { - updates["next.canonical_topic.id"] = dataset.CanonicalTopic.ID - } - if dataset.CanonicalTopic.Title != "" { - updates["next.canonical_topic.title"] = dataset.CanonicalTopic.Title - } - } - - if dataset.SubTopics != nil { - updates["next.sub_topics"] = dataset.SubTopics - } + updates["next.canonical_topic"] = dataset.CanonicalTopic + updates["next.sub_topics"] = dataset.SubTopics log.Info(ctx, "built update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) - return updates } From 5fed79feed67e5c6b8c56456e3f377cf0dca3fdf Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Fri, 23 Sep 2022 15:17:20 +0100 Subject: [PATCH 06/24] tests dataset topic fields model and update query --- models/dataset_test.go | 2 ++ models/test_data.go | 9 +++++++++ mongo/dataset_test.go | 30 ++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/models/dataset_test.go b/models/dataset_test.go index 92a41fbc..5566e40d 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -176,6 +176,8 @@ func TestCreateDataset(t *testing.T) { So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") So(dataset.Type, ShouldEqual, "filterable") So(dataset.NomisReferenceURL, ShouldEqual, "") + So(dataset.CanonicalTopic, ShouldResemble, &topic) + So(dataset.SubTopics[0], ShouldResemble, topic) }) }) diff --git a/models/test_data.go b/models/test_data.go index c6f72baa..c5fef9bc 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -43,6 +43,11 @@ var relatedDatasets = GeneralDetails{ Title: "Census Age", } +var topic = TopicTagObject{ + ID: "topidID", + Title: "Topic title", +} + // Create a fully populated dataset object to use in testing. func createTestDataset() *Dataset { return &Dataset{ @@ -80,6 +85,8 @@ func createTestDataset() *Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", + CanonicalTopic: &topic, + SubTopics: []TopicTagObject{topic}, } } @@ -116,6 +123,8 @@ func expectedDataset() Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", + CanonicalTopic: &topic, + SubTopics: []TopicTagObject{topic}, } } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 0a259fb5..23feb3f1 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -179,6 +179,19 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "some dataset title", } + canonicalTopic := models.TopicTagObject{ + ID: "canonicalTopicID", + Title: "Canonical topic title", + } + + subTopics := []models.TopicTagObject{{ + ID: "secondaryTopic1ID", + Title: "Secondary topic 1 title", + }, { + ID: "secondaryTopic2ID", + Title: "Secondary topic 2 title", + }} + var methodologies, publications, relatedDatasets []models.GeneralDetails methodologies = append(methodologies, methodology) publications = append(publications, publication) @@ -209,6 +222,8 @@ func TestDatasetUpdateQuery(t *testing.T) { "next.uri": "http://ons.gov.uk/datasets/123/landing-page", "next.type": "nomis", "next.nomis_reference_url": "https://www.nomisweb.co.uk/census/2011/ks106ew", + "next.canonical_topic": &canonicalTopic, + "next.sub_topics": subTopics, } dataset := &models.Dataset{ @@ -239,6 +254,8 @@ func TestDatasetUpdateQuery(t *testing.T) { URI: "http://ons.gov.uk/datasets/123/landing-page", Type: "nomis", NomisReferenceURL: "https://www.nomisweb.co.uk/census/2011/ks106ew", + CanonicalTopic: &canonicalTopic, + SubTopics: subTopics, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) @@ -251,20 +268,29 @@ func TestDatasetUpdateQuery(t *testing.T) { dataset := &models.Dataset{ NationalStatistic: &nationalStatistic, } + expectedUpdate := bson.M{ "next.national_statistic": &nationalStatistic, + "next.canonical_topic": dataset.CanonicalTopic, + "next.sub_topics": dataset.SubTopics, } + selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) So(selector, ShouldNotBeNil) So(selector, ShouldResemble, expectedUpdate) }) - Convey("When national statistic is not set", t, func() { + Convey("When no properties are set update includes nil values for topic fields", t, func() { dataset := &models.Dataset{} + expectedUpdate := bson.M{ + "next.canonical_topic": dataset.CanonicalTopic, + "next.sub_topics": dataset.SubTopics, + } + selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) So(selector, ShouldNotBeNil) - So(selector, ShouldResemble, bson.M{}) + So(selector, ShouldResemble, expectedUpdate) }) } From 56d014471dc17528edee2ecb228c88d50d25e9e1 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 28 Sep 2022 14:13:43 +0100 Subject: [PATCH 07/24] go fmt --- models/dataset.go | 4 ++-- mongo/dataset_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 0488a0dd..c82f0f9b 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -110,8 +110,8 @@ type Dataset struct { Type string `bson:"type,omitempty" json:"type,omitempty"` NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` - CanonicalTopic *TopicTagObject `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` - SubTopics []TopicTagObject `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` + CanonicalTopic *TopicTagObject `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` + SubTopics []TopicTagObject `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` } type TopicTagObject struct { diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 23feb3f1..83fb01bb 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -271,8 +271,8 @@ func TestDatasetUpdateQuery(t *testing.T) { expectedUpdate := bson.M{ "next.national_statistic": &nationalStatistic, - "next.canonical_topic": dataset.CanonicalTopic, - "next.sub_topics": dataset.SubTopics, + "next.canonical_topic": dataset.CanonicalTopic, + "next.sub_topics": dataset.SubTopics, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) @@ -285,7 +285,7 @@ func TestDatasetUpdateQuery(t *testing.T) { expectedUpdate := bson.M{ "next.canonical_topic": dataset.CanonicalTopic, - "next.sub_topics": dataset.SubTopics, + "next.sub_topics": dataset.SubTopics, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) From 8523698e04fb67eefc7374b626b4cb2f5a7555b7 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Thu, 29 Sep 2022 08:28:29 +0100 Subject: [PATCH 08/24] renames topic struct --- models/dataset.go | 6 +++--- models/test_data.go | 6 +++--- mongo/dataset_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index c82f0f9b..c3d10b90 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -110,11 +110,11 @@ type Dataset struct { Type string `bson:"type,omitempty" json:"type,omitempty"` NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` - CanonicalTopic *TopicTagObject `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` - SubTopics []TopicTagObject `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` + CanonicalTopic *Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` + SubTopics []Topic `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` } -type TopicTagObject struct { +type Topic struct { ID string `bson:"id,omitempty" json:"id,omitempty"` Title string `bson:"title,omitempty" json:"title,omitempty"` } diff --git a/models/test_data.go b/models/test_data.go index c5fef9bc..0c958cca 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -43,7 +43,7 @@ var relatedDatasets = GeneralDetails{ Title: "Census Age", } -var topic = TopicTagObject{ +var topic = Topic{ ID: "topidID", Title: "Topic title", } @@ -86,7 +86,7 @@ func createTestDataset() *Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: &topic, - SubTopics: []TopicTagObject{topic}, + SubTopics: []Topic{topic}, } } @@ -124,7 +124,7 @@ func expectedDataset() Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: &topic, - SubTopics: []TopicTagObject{topic}, + SubTopics: []Topic{topic}, } } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 83fb01bb..a3ce11f0 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -179,12 +179,12 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "some dataset title", } - canonicalTopic := models.TopicTagObject{ + canonicalTopic := models.Topic{ ID: "canonicalTopicID", Title: "Canonical topic title", } - subTopics := []models.TopicTagObject{{ + subTopics := []models.Topic{{ ID: "secondaryTopic1ID", Title: "Secondary topic 1 title", }, { From 122eef9848839ab11729f3f683c8a4d327812d29 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Thu, 29 Sep 2022 08:35:48 +0100 Subject: [PATCH 09/24] refactors test topic mocks --- models/dataset_test.go | 4 ++-- models/test_data.go | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/models/dataset_test.go b/models/dataset_test.go index 5566e40d..6ab90e48 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -176,8 +176,8 @@ func TestCreateDataset(t *testing.T) { So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") So(dataset.Type, ShouldEqual, "filterable") So(dataset.NomisReferenceURL, ShouldEqual, "") - So(dataset.CanonicalTopic, ShouldResemble, &topic) - So(dataset.SubTopics[0], ShouldResemble, topic) + So(dataset.CanonicalTopic, ShouldResemble, &canonicalTopic) + So(dataset.SubTopics[0], ShouldResemble, subtopic) }) }) diff --git a/models/test_data.go b/models/test_data.go index 0c958cca..48437915 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -43,9 +43,14 @@ var relatedDatasets = GeneralDetails{ Title: "Census Age", } -var topic = Topic{ - ID: "topidID", - Title: "Topic title", +var canonicalTopic = Topic{ + ID: "canonicalTopicID", + Title: "Canonical topic title", +} + +var subtopic = Topic{ + ID: "subtopicID", + Title: "Subtopic title", } // Create a fully populated dataset object to use in testing. @@ -85,8 +90,8 @@ func createTestDataset() *Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", - CanonicalTopic: &topic, - SubTopics: []Topic{topic}, + CanonicalTopic: &canonicalTopic, + SubTopics: []Topic{subtopic}, } } @@ -123,8 +128,8 @@ func expectedDataset() Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", - CanonicalTopic: &topic, - SubTopics: []Topic{topic}, + CanonicalTopic: &canonicalTopic, + SubTopics: []Topic{subtopic}, } } From 6ad3ae7981a56b74243cd82ff7aea45b4da02958 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Thu, 29 Sep 2022 16:06:47 +0100 Subject: [PATCH 10/24] adds component tests for dataset topic fields --- features/private_datasets.feature | 79 ++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index fc6c0412..ec6434be 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -44,6 +44,30 @@ Feature: Private Dataset API forbidden - dataset already exists """ + Scenario: Adding canonical and subtopic fields to a dataset + Given I have these datasets: + """ + [ + { + "id": "population-estimates" + } + ] + """ + When I PUT "/datasets/population-estimates" + """ + { + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + } + """ + Then the HTTP status code should be "200" + Scenario: GET /datasets Given I have these datasets: """ @@ -73,4 +97,57 @@ Feature: Private Dataset API "offset": 0, "total_count": 1 } - """ \ No newline at end of file + """ + + Scenario: GET /datasets with topics included + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + } + ] + """ + When I GET "/datasets" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [{ + "id": "population-estimates", + "next": { + "id": "population-estimates", + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + }, + "current": { + "id": "population-estimates", + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + } + }], + "limit": 20, + "offset": 0, + "total_count": 1 + } + """ \ No newline at end of file From 8d5ae6ab73a0b0112fea474f861da17484e18d6c Mon Sep 17 00:00:00 2001 From: PatrickConnollyONS Date: Fri, 30 Sep 2022 12:56:34 +0100 Subject: [PATCH 11/24] Extra comp tests --- features/private_datasets.feature | 14 ++++++++++++++ features/public_datasets.feature | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index ec6434be..245774ae 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -67,6 +67,20 @@ Feature: Private Dataset API } """ Then the HTTP status code should be "200" + And the document in the database for id "population-estimates" should be: + """ + { + "id": "population-estimates", + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + } + """ Scenario: GET /datasets Given I have these datasets: diff --git a/features/public_datasets.feature b/features/public_datasets.feature index bc6382ba..e7c7113f 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -44,3 +44,27 @@ Feature: Dataset API "id": "income-by-age" } """ + + Scenario: Adding canonical and subtopic fields to a dataset + Given I have these datasets: + """ + [ + { + "id": "population-estimates" + } + ] + """ + When I PUT "/datasets/population-estimates" + """ + { + "canonical_topic": { + "id": "canonical-topic-ID", + "title": "Canonical topic title" + }, + "sub_topics": [{ + "id": "subtopic-ID", + "title": "Subtopic title" + }] + } + """ + Then the HTTP status code should be "405" From 49f74457a5290c3cb003ca3f2bf8f089aef85aee Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Tue, 4 Oct 2022 08:59:38 +0100 Subject: [PATCH 12/24] adds guards in dataset update query fn & fixes tests --- mongo/dataset_store.go | 9 +++++++-- mongo/dataset_test.go | 15 --------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 83877d17..6fbc294c 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -414,8 +414,13 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - updates["next.canonical_topic"] = dataset.CanonicalTopic - updates["next.sub_topics"] = dataset.SubTopics + if dataset.CanonicalTopic != nil { + updates["next.canonical_topic"] = dataset.CanonicalTopic + } + + if dataset.SubTopics != nil { + updates["next.sub_topics"] = dataset.SubTopics + } log.Info(ctx, "built update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) return updates diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index a3ce11f0..0e38d119 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -271,21 +271,6 @@ func TestDatasetUpdateQuery(t *testing.T) { expectedUpdate := bson.M{ "next.national_statistic": &nationalStatistic, - "next.canonical_topic": dataset.CanonicalTopic, - "next.sub_topics": dataset.SubTopics, - } - - selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) - So(selector, ShouldNotBeNil) - So(selector, ShouldResemble, expectedUpdate) - }) - - Convey("When no properties are set update includes nil values for topic fields", t, func() { - dataset := &models.Dataset{} - - expectedUpdate := bson.M{ - "next.canonical_topic": dataset.CanonicalTopic, - "next.sub_topics": dataset.SubTopics, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) From 18493f5ed42860ef45c5eff76101a5cb8527f001 Mon Sep 17 00:00:00 2001 From: PatrickConnollyONS Date: Tue, 4 Oct 2022 13:05:51 +0100 Subject: [PATCH 13/24] Bump MongoDB library version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 22c27e87..46bda058 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/ONSdigital/dp-graph/v2 v2.15.0 github.com/ONSdigital/dp-healthcheck v1.3.0 github.com/ONSdigital/dp-kafka/v2 v2.5.0 - github.com/ONSdigital/dp-mongodb/v3 v3.0.2 + github.com/ONSdigital/dp-mongodb/v3 v3.2.0 github.com/ONSdigital/dp-net/v2 v2.4.0 github.com/ONSdigital/log.go/v2 v2.2.0 github.com/cucumber/godog v0.12.5 diff --git a/go.sum b/go.sum index 63f15c16..924fd2fc 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9 h1:+WXVfTDyW github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-mongodb-in-memory v1.3.1 h1:E2xSPOBr5B0riB1qN+z+m8ouWymnLWu+XHBUpgYLXWc= github.com/ONSdigital/dp-mongodb-in-memory v1.3.1/go.mod h1:QuDIrDBBsmL9Jh/kmpKI1ENslO8yBCDLlAQ6qL10Pac= -github.com/ONSdigital/dp-mongodb/v3 v3.0.2 h1:CbIGAnhMvqwfKg3Dc3LCP6ALcFV4I9vdWRDOTi/QPdc= -github.com/ONSdigital/dp-mongodb/v3 v3.0.2/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE= +github.com/ONSdigital/dp-mongodb/v3 v3.2.0 h1:Z1lUFuazj1q9bseJZwR6Ac7fYW0QGMx4I04X5bZxXXA= +github.com/ONSdigital/dp-mongodb/v3 v3.2.0/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE= github.com/ONSdigital/dp-net v1.0.5-0.20200805082802-e518bc287596/go.mod h1:wDVhk2pYosQ1q6PXxuFIRYhYk2XX5+1CeRRnXpSczPY= github.com/ONSdigital/dp-net v1.0.5-0.20200805145012-9227a11caddb/go.mod h1:MrSZwDUvp8u1VJEqa+36Gwq4E7/DdceW+BDCvGes6Cs= github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3LB9tedE0tObBwa12dUOt5rvTW4qQkF5rXtt4b6CE= From 43a3f948e126aa8127b01e21ac7e37e747c5d2d0 Mon Sep 17 00:00:00 2001 From: PatrickConnollyONS Date: Wed, 5 Oct 2022 12:17:24 +0100 Subject: [PATCH 14/24] Bump MongoDB library version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 46bda058..f951452f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/ONSdigital/dp-graph/v2 v2.15.0 github.com/ONSdigital/dp-healthcheck v1.3.0 github.com/ONSdigital/dp-kafka/v2 v2.5.0 - github.com/ONSdigital/dp-mongodb/v3 v3.2.0 + github.com/ONSdigital/dp-mongodb/v3 v3.3.0 github.com/ONSdigital/dp-net/v2 v2.4.0 github.com/ONSdigital/log.go/v2 v2.2.0 github.com/cucumber/godog v0.12.5 diff --git a/go.sum b/go.sum index 924fd2fc..5af18026 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9 h1:+WXVfTDyW github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-mongodb-in-memory v1.3.1 h1:E2xSPOBr5B0riB1qN+z+m8ouWymnLWu+XHBUpgYLXWc= github.com/ONSdigital/dp-mongodb-in-memory v1.3.1/go.mod h1:QuDIrDBBsmL9Jh/kmpKI1ENslO8yBCDLlAQ6qL10Pac= -github.com/ONSdigital/dp-mongodb/v3 v3.2.0 h1:Z1lUFuazj1q9bseJZwR6Ac7fYW0QGMx4I04X5bZxXXA= -github.com/ONSdigital/dp-mongodb/v3 v3.2.0/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE= +github.com/ONSdigital/dp-mongodb/v3 v3.3.0 h1:r6zblTi/X69pNwJeTmnun4DW4GIhSaB66eT2Le1aVR0= +github.com/ONSdigital/dp-mongodb/v3 v3.3.0/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE= github.com/ONSdigital/dp-net v1.0.5-0.20200805082802-e518bc287596/go.mod h1:wDVhk2pYosQ1q6PXxuFIRYhYk2XX5+1CeRRnXpSczPY= github.com/ONSdigital/dp-net v1.0.5-0.20200805145012-9227a11caddb/go.mod h1:MrSZwDUvp8u1VJEqa+36Gwq4E7/DdceW+BDCvGes6Cs= github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3LB9tedE0tObBwa12dUOt5rvTW4qQkF5rXtt4b6CE= From 62671a411772907172cbb6863ddb075e5ea1a370 Mon Sep 17 00:00:00 2001 From: red Date: Mon, 17 Oct 2022 20:06:01 +0100 Subject: [PATCH 15/24] Fix audit, up go to version 19 --- ci/build.yml | 2 +- ci/unit.yml | 2 +- go.mod | 5 ++++- go.sum | 18 +++++++----------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index e95ac0b3..1ba6dcb8 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -6,7 +6,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.18.3 + tag: 1.19.2 inputs: - name: dp-dataset-api diff --git a/ci/unit.yml b/ci/unit.yml index 2c6dcff8..cb8a4391 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -6,7 +6,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.18.3 + tag: 1.19.2 inputs: - name: dp-dataset-api diff --git a/go.mod b/go.mod index f951452f..0bd32029 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ONSdigital/dp-dataset-api -go 1.18 +go 1.19 replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible @@ -9,6 +9,9 @@ replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible // - sonatype-2021-1401 # pkg:golang/github.com/miekg/dns@v1.0.14 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 +// to avoid [CVE-2022-32149] CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion') +replace golang.org/x/text => golang.org/x/text v0.3.8 + // to avoid 'sonatype-2021-4899' non-CVE Vulnerability exclude github.com/gorilla/sessions v1.2.1 diff --git a/go.sum b/go.sum index 5af18026..d3e7d53a 100644 --- a/go.sum +++ b/go.sum @@ -476,6 +476,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.10.0 h1:UtV6N5k14upNp4LTduX0QCufG124fSu25Wz9tu94GLg= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -500,6 +501,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -538,6 +540,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -578,6 +581,7 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.0.0-20220728153142-1f511ac62c11 h1:BZ+7NKCw5UIzmPP4GFz9VcjTVpN9mswsWRDmJ2tWKAs= golang.org/x/net v0.0.0-20220728153142-1f511ac62c11/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -666,20 +670,11 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -728,6 +723,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d996221e914f5d9e7450280f90d5c0af1940cea2 Mon Sep 17 00:00:00 2001 From: red Date: Wed, 19 Oct 2022 11:58:53 +0100 Subject: [PATCH 16/24] Improve go.mod file --- go.mod | 5 +---- go.sum | 15 ++++++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0bd32029..de546b33 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,6 @@ replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible // - sonatype-2021-1401 # pkg:golang/github.com/miekg/dns@v1.0.14 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 -// to avoid [CVE-2022-32149] CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion') -replace golang.org/x/text => golang.org/x/text v0.3.8 - // to avoid 'sonatype-2021-4899' non-CVE Vulnerability exclude github.com/gorilla/sessions v1.2.1 @@ -104,6 +101,6 @@ require ( golang.org/x/net v0.0.0-20220728153142-1f511ac62c11 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/text v0.3.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d3e7d53a..65505295 100644 --- a/go.sum +++ b/go.sum @@ -476,7 +476,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.10.0 h1:UtV6N5k14upNp4LTduX0QCufG124fSu25Wz9tu94GLg= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -501,7 +500,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -540,7 +538,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -581,7 +578,6 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.0.0-20220728153142-1f511ac62c11 h1:BZ+7NKCw5UIzmPP4GFz9VcjTVpN9mswsWRDmJ2tWKAs= golang.org/x/net v0.0.0-20220728153142-1f511ac62c11/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -670,11 +666,21 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -723,7 +729,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 827a33c45e8c3199add6dad58e13c0bd53658ae7 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 5 Oct 2022 16:04:27 +0100 Subject: [PATCH 17/24] sets up survey field and functionality --- models/dataset.go | 1 + models/dataset_test.go | 1 + models/test_data.go | 4 ++++ mongo/dataset_store.go | 4 ++++ mongo/dataset_test.go | 4 ++++ 5 files changed, 14 insertions(+) diff --git a/models/dataset.go b/models/dataset.go index c3d10b90..47492f66 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -112,6 +112,7 @@ type Dataset struct { IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` CanonicalTopic *Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` SubTopics []Topic `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` + Survey []string `bson:"survey,omitempty" json:"survey,omitempty"` } type Topic struct { diff --git a/models/dataset_test.go b/models/dataset_test.go index 6ab90e48..56ef5bda 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -178,6 +178,7 @@ func TestCreateDataset(t *testing.T) { So(dataset.NomisReferenceURL, ShouldEqual, "") So(dataset.CanonicalTopic, ShouldResemble, &canonicalTopic) So(dataset.SubTopics[0], ShouldResemble, subtopic) + So(dataset.Survey, ShouldResemble, survey) }) }) diff --git a/models/test_data.go b/models/test_data.go index 48437915..19f73394 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -53,6 +53,8 @@ var subtopic = Topic{ Title: "Subtopic title", } +var survey = []string{"mockSurvey1", "mockSurvey2"} + // Create a fully populated dataset object to use in testing. func createTestDataset() *Dataset { return &Dataset{ @@ -92,6 +94,7 @@ func createTestDataset() *Dataset { NomisReferenceURL: "", CanonicalTopic: &canonicalTopic, SubTopics: []Topic{subtopic}, + Survey: survey, } } @@ -130,6 +133,7 @@ func expectedDataset() Dataset { NomisReferenceURL: "", CanonicalTopic: &canonicalTopic, SubTopics: []Topic{subtopic}, + Survey: survey, } } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 6fbc294c..3fcb239a 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -422,6 +422,10 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.sub_topics"] = dataset.SubTopics } + if dataset.Survey != nil { + updates["next.survey"] = dataset.Survey + } + log.Info(ctx, "built update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) return updates } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 0e38d119..c1758eed 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -192,6 +192,8 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "Secondary topic 2 title", }} + survey := []string{"mockSurvey1", "mockSurvey2"} + var methodologies, publications, relatedDatasets []models.GeneralDetails methodologies = append(methodologies, methodology) publications = append(publications, publication) @@ -224,6 +226,7 @@ func TestDatasetUpdateQuery(t *testing.T) { "next.nomis_reference_url": "https://www.nomisweb.co.uk/census/2011/ks106ew", "next.canonical_topic": &canonicalTopic, "next.sub_topics": subTopics, + "next.survey": survey, } dataset := &models.Dataset{ @@ -256,6 +259,7 @@ func TestDatasetUpdateQuery(t *testing.T) { NomisReferenceURL: "https://www.nomisweb.co.uk/census/2011/ks106ew", CanonicalTopic: &canonicalTopic, SubTopics: subTopics, + Survey: survey, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) From bc6bc3d43c8691e7b4c7a8e1e220757a147c0a15 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Thu, 6 Oct 2022 14:33:06 +0100 Subject: [PATCH 18/24] converts survey field from slice to string --- models/dataset.go | 2 +- models/dataset_test.go | 2 +- models/test_data.go | 2 +- mongo/dataset_store.go | 2 +- mongo/dataset_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 47492f66..52a71fb3 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -112,7 +112,7 @@ type Dataset struct { IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` CanonicalTopic *Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` SubTopics []Topic `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` - Survey []string `bson:"survey,omitempty" json:"survey,omitempty"` + Survey string `bson:"survey,omitempty" json:"survey,omitempty"` } type Topic struct { diff --git a/models/dataset_test.go b/models/dataset_test.go index 56ef5bda..993c0d28 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -178,7 +178,7 @@ func TestCreateDataset(t *testing.T) { So(dataset.NomisReferenceURL, ShouldEqual, "") So(dataset.CanonicalTopic, ShouldResemble, &canonicalTopic) So(dataset.SubTopics[0], ShouldResemble, subtopic) - So(dataset.Survey, ShouldResemble, survey) + So(dataset.Survey, ShouldEqual, survey) }) }) diff --git a/models/test_data.go b/models/test_data.go index 19f73394..62527a9d 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -53,7 +53,7 @@ var subtopic = Topic{ Title: "Subtopic title", } -var survey = []string{"mockSurvey1", "mockSurvey2"} +var survey = "mockSurvey" // Create a fully populated dataset object to use in testing. func createTestDataset() *Dataset { diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 3fcb239a..e5b06a01 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -422,7 +422,7 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.sub_topics"] = dataset.SubTopics } - if dataset.Survey != nil { + if dataset.Survey != "" { updates["next.survey"] = dataset.Survey } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index c1758eed..ad3ab12c 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -192,7 +192,7 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "Secondary topic 2 title", }} - survey := []string{"mockSurvey1", "mockSurvey2"} + survey := "mockSurvey" var methodologies, publications, relatedDatasets []models.GeneralDetails methodologies = append(methodologies, methodology) From 88fed1ee2e123579a451029c464c58dcd9d15867 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Thu, 6 Oct 2022 16:04:33 +0100 Subject: [PATCH 19/24] updates component tests --- features/private_datasets.feature | 8 +++++--- features/public_datasets.feature | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index 245774ae..efd8cdb2 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -44,7 +44,7 @@ Feature: Private Dataset API forbidden - dataset already exists """ - Scenario: Adding canonical and subtopic fields to a dataset + Scenario: Adding topic and survey fields to a dataset Given I have these datasets: """ [ @@ -63,7 +63,8 @@ Feature: Private Dataset API "sub_topics": [{ "id": "subtopic-ID", "title": "Subtopic title" - }] + }], + "survey": "mockSurvey" } """ Then the HTTP status code should be "200" @@ -78,7 +79,8 @@ Feature: Private Dataset API "sub_topics": [{ "id": "subtopic-ID", "title": "Subtopic title" - }] + }], + "survey": "mockSurvey" } """ diff --git a/features/public_datasets.feature b/features/public_datasets.feature index e7c7113f..7dcd088d 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -45,7 +45,7 @@ Feature: Dataset API } """ - Scenario: Adding canonical and subtopic fields to a dataset + Scenario: Adding topic and survey fields to a dataset Given I have these datasets: """ [ @@ -64,7 +64,8 @@ Feature: Dataset API "sub_topics": [{ "id": "subtopic-ID", "title": "Subtopic title" - }] + }], + "survey": "mockSurvey" } """ Then the HTTP status code should be "405" From dd2b7e7a424314fe7cbe86cb2e5e77e6d012ab0f Mon Sep 17 00:00:00 2001 From: Valentina Date: Fri, 7 Oct 2022 18:45:17 +0100 Subject: [PATCH 20/24] reverts Topic and Survey fields from pointers and fixes failing tests --- api/dataset_test.go | 2 +- features/private_datasets.feature | 23 +++++++++++++++++++++++ models/dataset.go | 4 ++-- models/dataset_test.go | 2 +- models/test_data.go | 4 ++-- mongo/dataset_store.go | 4 ++-- mongo/dataset_test.go | 4 ++-- 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index a97a651c..669d619f 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -740,7 +740,7 @@ func TestPostDatasetReturnsError(t *testing.T) { Convey("When the request body has an empty type field it should create a dataset with type defaulted to filterable", t, func() { b := `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":""}` - res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable"}}` + res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable","canonical_topic":{}}}` r := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b)) w := httptest.NewRecorder() diff --git a/features/private_datasets.feature b/features/private_datasets.feature index efd8cdb2..d1d949ef 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -84,6 +84,29 @@ Feature: Private Dataset API } """ + Scenario: Removing a survey from a dataset + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "survey": "mockSurvey" + } + ] + """ + When I PUT "/datasets/population-estimates" + """ + { + "survey": "" + } + """ + Then the document in the database for id "population-estimates" should be: + """ + { + "id": "population-estimates" + } + """ + Scenario: GET /datasets Given I have these datasets: """ diff --git a/models/dataset.go b/models/dataset.go index 52a71fb3..b282647f 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -110,9 +110,9 @@ type Dataset struct { Type string `bson:"type,omitempty" json:"type,omitempty"` NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` - CanonicalTopic *Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` + CanonicalTopic Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` SubTopics []Topic `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` - Survey string `bson:"survey,omitempty" json:"survey,omitempty"` + Survey string `bson:"survey,omitempty" json:"survey,omitempty"` } type Topic struct { diff --git a/models/dataset_test.go b/models/dataset_test.go index 993c0d28..95b19792 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -176,7 +176,7 @@ func TestCreateDataset(t *testing.T) { So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") So(dataset.Type, ShouldEqual, "filterable") So(dataset.NomisReferenceURL, ShouldEqual, "") - So(dataset.CanonicalTopic, ShouldResemble, &canonicalTopic) + So(dataset.CanonicalTopic, ShouldResemble, canonicalTopic) So(dataset.SubTopics[0], ShouldResemble, subtopic) So(dataset.Survey, ShouldEqual, survey) }) diff --git a/models/test_data.go b/models/test_data.go index 62527a9d..1689e2b4 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -92,7 +92,7 @@ func createTestDataset() *Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", - CanonicalTopic: &canonicalTopic, + CanonicalTopic: canonicalTopic, SubTopics: []Topic{subtopic}, Survey: survey, } @@ -131,7 +131,7 @@ func expectedDataset() Dataset { URI: "http://localhost:22000/datasets/123/breadcrumbs", Type: "filterable", NomisReferenceURL: "", - CanonicalTopic: &canonicalTopic, + CanonicalTopic: canonicalTopic, SubTopics: []Topic{subtopic}, Survey: survey, } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index e5b06a01..7b37eb7e 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -414,11 +414,11 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - if dataset.CanonicalTopic != nil { + if dataset.CanonicalTopic != (models.Topic{}) { updates["next.canonical_topic"] = dataset.CanonicalTopic } - if dataset.SubTopics != nil { + if len(dataset.SubTopics) > 0 { updates["next.sub_topics"] = dataset.SubTopics } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index ad3ab12c..8f1bfd5e 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -224,7 +224,7 @@ func TestDatasetUpdateQuery(t *testing.T) { "next.uri": "http://ons.gov.uk/datasets/123/landing-page", "next.type": "nomis", "next.nomis_reference_url": "https://www.nomisweb.co.uk/census/2011/ks106ew", - "next.canonical_topic": &canonicalTopic, + "next.canonical_topic": canonicalTopic, "next.sub_topics": subTopics, "next.survey": survey, } @@ -257,7 +257,7 @@ func TestDatasetUpdateQuery(t *testing.T) { URI: "http://ons.gov.uk/datasets/123/landing-page", Type: "nomis", NomisReferenceURL: "https://www.nomisweb.co.uk/census/2011/ks106ew", - CanonicalTopic: &canonicalTopic, + CanonicalTopic: canonicalTopic, SubTopics: subTopics, Survey: survey, } From 8bd125c3375fa6f48ffec803f5ab8414a3f6b080 Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 19 Oct 2022 14:09:49 +0100 Subject: [PATCH 21/24] removes Topic struct and reverts topics to string --- api/dataset_test.go | 2 +- features/private_datasets.feature | 50 +++++++------------------------ features/public_datasets.feature | 10 ++----- models/dataset.go | 9 ++---- models/test_data.go | 14 +++------ mongo/dataset_store.go | 2 +- mongo/dataset_test.go | 13 ++------ 7 files changed, 22 insertions(+), 78 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index 669d619f..a97a651c 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -740,7 +740,7 @@ func TestPostDatasetReturnsError(t *testing.T) { Convey("When the request body has an empty type field it should create a dataset with type defaulted to filterable", t, func() { b := `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":""}` - res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable","canonical_topic":{}}}` + res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable"}}` r := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b)) w := httptest.NewRecorder() diff --git a/features/private_datasets.feature b/features/private_datasets.feature index d1d949ef..25d8d8f8 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -56,14 +56,8 @@ Feature: Private Dataset API When I PUT "/datasets/population-estimates" """ { - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }], + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"], "survey": "mockSurvey" } """ @@ -72,14 +66,8 @@ Feature: Private Dataset API """ { "id": "population-estimates", - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }], + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"], "survey": "mockSurvey" } """ @@ -144,14 +132,8 @@ Feature: Private Dataset API [ { "id": "population-estimates", - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }] + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"] } ] """ @@ -164,25 +146,13 @@ Feature: Private Dataset API "id": "population-estimates", "next": { "id": "population-estimates", - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }] + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"] }, "current": { "id": "population-estimates", - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }] + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"] } }], "limit": 20, diff --git a/features/public_datasets.feature b/features/public_datasets.feature index 7dcd088d..627c774d 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -57,14 +57,8 @@ Feature: Dataset API When I PUT "/datasets/population-estimates" """ { - "canonical_topic": { - "id": "canonical-topic-ID", - "title": "Canonical topic title" - }, - "sub_topics": [{ - "id": "subtopic-ID", - "title": "Subtopic title" - }], + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"], "survey": "mockSurvey" } """ diff --git a/models/dataset.go b/models/dataset.go index b282647f..e0922f06 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -110,16 +110,11 @@ type Dataset struct { Type string `bson:"type,omitempty" json:"type,omitempty"` NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` - CanonicalTopic Topic `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` - SubTopics []Topic `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` + CanonicalTopic string `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` + SubTopics []string `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` Survey string `bson:"survey,omitempty" json:"survey,omitempty"` } -type Topic struct { - ID string `bson:"id,omitempty" json:"id,omitempty"` - Title string `bson:"title,omitempty" json:"title,omitempty"` -} - // DatasetLinks represents a list of specific links related to the dataset resource type DatasetLinks struct { AccessRights *LinkObject `bson:"access_rights,omitempty" json:"access_rights,omitempty"` diff --git a/models/test_data.go b/models/test_data.go index 1689e2b4..54dc8734 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -43,15 +43,9 @@ var relatedDatasets = GeneralDetails{ Title: "Census Age", } -var canonicalTopic = Topic{ - ID: "canonicalTopicID", - Title: "Canonical topic title", -} +var canonicalTopic = "canonicalTopicID" -var subtopic = Topic{ - ID: "subtopicID", - Title: "Subtopic title", -} +var subtopic = "subtopicID" var survey = "mockSurvey" @@ -93,7 +87,7 @@ func createTestDataset() *Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: canonicalTopic, - SubTopics: []Topic{subtopic}, + SubTopics: []string{subtopic}, Survey: survey, } } @@ -132,7 +126,7 @@ func expectedDataset() Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: canonicalTopic, - SubTopics: []Topic{subtopic}, + SubTopics: []string{subtopic}, Survey: survey, } } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 7b37eb7e..5fc1b306 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -414,7 +414,7 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - if dataset.CanonicalTopic != (models.Topic{}) { + if dataset.CanonicalTopic != "" { updates["next.canonical_topic"] = dataset.CanonicalTopic } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 8f1bfd5e..20002941 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -179,18 +179,9 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "some dataset title", } - canonicalTopic := models.Topic{ - ID: "canonicalTopicID", - Title: "Canonical topic title", - } + canonicalTopic := "canonicalTopicID" - subTopics := []models.Topic{{ - ID: "secondaryTopic1ID", - Title: "Secondary topic 1 title", - }, { - ID: "secondaryTopic2ID", - Title: "Secondary topic 2 title", - }} + subTopics := []string{"secondaryTopic1ID", "secondaryTopic2ID"} survey := "mockSurvey" From e91a047eb151f2cbb124633b325761f841c7528f Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 19 Oct 2022 14:21:41 +0100 Subject: [PATCH 22/24] separates survey and topics component tests --- features/private_datasets.feature | 32 ++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index 25d8d8f8..da9bea71 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -44,7 +44,7 @@ Feature: Private Dataset API forbidden - dataset already exists """ - Scenario: Adding topic and survey fields to a dataset + Scenario: Adding survey field to a dataset Given I have these datasets: """ [ @@ -56,8 +56,6 @@ Feature: Private Dataset API When I PUT "/datasets/population-estimates" """ { - "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"], "survey": "mockSurvey" } """ @@ -66,12 +64,36 @@ Feature: Private Dataset API """ { "id": "population-estimates", - "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"], "survey": "mockSurvey" } """ + Scenario: Adding topic fields to a dataset + Given I have these datasets: + """ + [ + { + "id": "population-estimates" + } + ] + """ + When I PUT "/datasets/population-estimates" + """ + { + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"] + } + """ + Then the HTTP status code should be "200" + And the document in the database for id "population-estimates" should be: + """ + { + "id": "population-estimates", + "canonical_topic": "canonical-topic-ID", + "sub_topics": ["subtopic-ID"] + } + """ + Scenario: Removing a survey from a dataset Given I have these datasets: """ From 70394723c7c128185c949d7db054845a3dff8f4d Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 19 Oct 2022 15:26:04 +0100 Subject: [PATCH 23/24] updates go version for Docker --- Dockerfile-local | 2 +- Dockerfile.local | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile-local b/Dockerfile-local index 49162000..b6dd77fe 100644 --- a/Dockerfile-local +++ b/Dockerfile-local @@ -1,4 +1,4 @@ -FROM golang:1.17-stretch as build +FROM golang:1.18-stretch as build RUN apt-get update && apt-get upgrade -y diff --git a/Dockerfile.local b/Dockerfile.local index ba4ec51b..a270516e 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,8 +1,8 @@ -FROM golang:1.17-stretch AS base +FROM golang:1.18-stretch AS base ENV GOCACHE=/go/.go/cache GOPATH=/go/.go/path TZ=Europe/London -RUN GOBIN=/bin go get github.com/cespare/reflex +RUN GOBIN=/bin go install github.com/cespare/reflex@latest # Map between the working directories of dev and live RUN ln -s /go /dp-dataset-api From f705a337fe29436f21a0c703c67395f56f15d9bc Mon Sep 17 00:00:00 2001 From: Patrick Fleming Date: Wed, 19 Oct 2022 16:19:33 +0100 Subject: [PATCH 24/24] changes subtopics casing --- features/private_datasets.feature | 10 +++++----- features/public_datasets.feature | 2 +- models/dataset.go | 2 +- models/dataset_test.go | 2 +- models/test_data.go | 4 ++-- mongo/dataset_store.go | 4 ++-- mongo/dataset_test.go | 6 +++--- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/features/private_datasets.feature b/features/private_datasets.feature index da9bea71..d58b7157 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -81,7 +81,7 @@ Feature: Private Dataset API """ { "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"] + "subtopics": ["subtopic-ID"] } """ Then the HTTP status code should be "200" @@ -90,7 +90,7 @@ Feature: Private Dataset API { "id": "population-estimates", "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"] + "subtopics": ["subtopic-ID"] } """ @@ -155,7 +155,7 @@ Feature: Private Dataset API { "id": "population-estimates", "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"] + "subtopics": ["subtopic-ID"] } ] """ @@ -169,12 +169,12 @@ Feature: Private Dataset API "next": { "id": "population-estimates", "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"] + "subtopics": ["subtopic-ID"] }, "current": { "id": "population-estimates", "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"] + "subtopics": ["subtopic-ID"] } }], "limit": 20, diff --git a/features/public_datasets.feature b/features/public_datasets.feature index 627c774d..0a3b1b80 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -58,7 +58,7 @@ Feature: Dataset API """ { "canonical_topic": "canonical-topic-ID", - "sub_topics": ["subtopic-ID"], + "subtopics": ["subtopic-ID"], "survey": "mockSurvey" } """ diff --git a/models/dataset.go b/models/dataset.go index e0922f06..ffd96bc0 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -111,7 +111,7 @@ type Dataset struct { NomisReferenceURL string `bson:"nomis_reference_url,omitempty" json:"nomis_reference_url,omitempty"` IsBasedOn *IsBasedOn `bson:"is_based_on,omitempty" json:"is_based_on,omitempty"` CanonicalTopic string `bson:"canonical_topic,omitempty" json:"canonical_topic,omitempty"` - SubTopics []string `bson:"sub_topics,omitempty" json:"sub_topics,omitempty"` + Subtopics []string `bson:"subtopics,omitempty" json:"subtopics,omitempty"` Survey string `bson:"survey,omitempty" json:"survey,omitempty"` } diff --git a/models/dataset_test.go b/models/dataset_test.go index 95b19792..5c9b1d60 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -177,7 +177,7 @@ func TestCreateDataset(t *testing.T) { So(dataset.Type, ShouldEqual, "filterable") So(dataset.NomisReferenceURL, ShouldEqual, "") So(dataset.CanonicalTopic, ShouldResemble, canonicalTopic) - So(dataset.SubTopics[0], ShouldResemble, subtopic) + So(dataset.Subtopics[0], ShouldResemble, subtopic) So(dataset.Survey, ShouldEqual, survey) }) }) diff --git a/models/test_data.go b/models/test_data.go index 54dc8734..feabe38b 100644 --- a/models/test_data.go +++ b/models/test_data.go @@ -87,7 +87,7 @@ func createTestDataset() *Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: canonicalTopic, - SubTopics: []string{subtopic}, + Subtopics: []string{subtopic}, Survey: survey, } } @@ -126,7 +126,7 @@ func expectedDataset() Dataset { Type: "filterable", NomisReferenceURL: "", CanonicalTopic: canonicalTopic, - SubTopics: []string{subtopic}, + Subtopics: []string{subtopic}, Survey: survey, } } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 5fc1b306..1a307035 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -418,8 +418,8 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.canonical_topic"] = dataset.CanonicalTopic } - if len(dataset.SubTopics) > 0 { - updates["next.sub_topics"] = dataset.SubTopics + if len(dataset.Subtopics) > 0 { + updates["next.subtopics"] = dataset.Subtopics } if dataset.Survey != "" { diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 20002941..9ad330ad 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -181,7 +181,7 @@ func TestDatasetUpdateQuery(t *testing.T) { canonicalTopic := "canonicalTopicID" - subTopics := []string{"secondaryTopic1ID", "secondaryTopic2ID"} + subtopics := []string{"secondaryTopic1ID", "secondaryTopic2ID"} survey := "mockSurvey" @@ -216,7 +216,7 @@ func TestDatasetUpdateQuery(t *testing.T) { "next.type": "nomis", "next.nomis_reference_url": "https://www.nomisweb.co.uk/census/2011/ks106ew", "next.canonical_topic": canonicalTopic, - "next.sub_topics": subTopics, + "next.subtopics": subtopics, "next.survey": survey, } @@ -249,7 +249,7 @@ func TestDatasetUpdateQuery(t *testing.T) { Type: "nomis", NomisReferenceURL: "https://www.nomisweb.co.uk/census/2011/ks106ew", CanonicalTopic: canonicalTopic, - SubTopics: subTopics, + Subtopics: subtopics, Survey: survey, }