Merge branch 'release/1.48.0'

Dockerfile-local

@@ -1,4 +1,4 @@
-FROM golang:1.17-stretch as build
+FROM golang:1.18-stretch as build
 
 RUN apt-get update && apt-get upgrade -y
 

Dockerfile.local

@@ -1,8 +1,8 @@
-FROM golang:1.17-stretch AS base
+FROM golang:1.18-stretch AS base
 
 ENV GOCACHE=/go/.go/cache GOPATH=/go/.go/path TZ=Europe/London
 
-RUN GOBIN=/bin go get github.com/cespare/reflex
+RUN GOBIN=/bin go install github.com/cespare/reflex@latest
 
 # Map between the working directories of dev and live
 RUN ln -s /go /dp-dataset-api

api/metadata.go

@@ -92,7 +92,7 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) {
 
 		var metaDataDoc *models.Metadata
 
-		if t == models.CantabularBlob || t == models.CantabularTable || t == models.CantabularFlexibleTable {
+		if t == models.CantabularBlob || t == models.CantabularTable || t == models.CantabularFlexibleTable || t == models.CantabularMultivariateTable {
 			metaDataDoc = models.CreateCantabularMetaDataDoc(doc, versionDoc, api.urlBuilder)
 		} else {
 			// combine version and dataset metadata

ci/build.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.2
 
 inputs:
   - name: dp-dataset-api

ci/unit.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.2
 
 inputs:
   - name: dp-dataset-api

features/instances.feature

@@ -1,4 +1,4 @@
-Feature: Dataset API
+    Feature: Dataset API
 
     Background: we have instances
         Given I have these instances:
@@ -30,6 +30,15 @@ Feature: Dataset API
                             "id": "income"
                         }
                     }
+                },
+                {
+                    "id": "test-item-4",
+                    "state": "created",
+                    "links": {
+                        "dataset": {
+                            "id": "other"
+                        }
+                    }
                 }
             ]
             """
@@ -42,8 +51,20 @@ Feature: Dataset API
         Then I should receive the following JSON response with status "200":
             """
             {
-                "count": 3,
+                "count": 4,
                 "items": [
+                      {
+                        "id": "test-item-4",
+                        "import_tasks": null,
+                        "last_updated": "2021-01-01T00:00:03Z",
+                        "links": {
+                            "dataset": {
+                                "id": "other"
+                            },
+                            "job": null
+                        },
+                        "state": "created"
+                    },
                     {
                         "id": "test-item-3",
                         "import_tasks": null,
@@ -83,7 +104,7 @@ Feature: Dataset API
                 ],
                 "limit": 20,
                 "offset": 0,
-                "total_count": 3
+                "total_count": 4
             }
             """
 
@@ -266,7 +287,42 @@ Feature: Dataset API
         And I set the "If-Match" header to "wrongValue"
         When I GET "/instances/test-item-1"
         Then the HTTP status code should be "409"
-                And I should receive the following response:
+        And I should receive the following response:
             """
             instance does not match the expected eTag
             """
+    Scenario: Updating instance with is_area_type explicitly false
+        Given private endpoints are enabled
+        And I am identified as "[email protected]"
+        And I am authorised
+        When I PUT "/instances"
+        """
+        {
+            "id": "test-item-4",
+            "dimensions":[
+                {
+                    "name": "foo",
+                    "is_area_type": false,
+                }
+            ]
+        }
+        """
+
+        Then the instance in the database for id "test-item-4" should be:
+        """
+        {
+            "id": "test-item-4",
+            "state": "created",
+            "links": {
+                "dataset": {
+                    "id": "other"
+                }
+            },
+            "dimensions":[
+                {
+                    "name": "foo",
+                    "is_area_type": false
+                }
+            ]
+        }
+        """

features/private_datasets.feature

@@ -44,6 +44,79 @@ Feature: Private Dataset API
             forbidden - dataset already exists
             """
 
+    Scenario: Adding survey field to a dataset
+        Given I have these datasets:
+            """
+            [
+                {
+                    "id": "population-estimates"
+                }
+            ]
+            """
+        When I PUT "/datasets/population-estimates"
+            """
+            {
+                    "survey": "mockSurvey"
+            }
+            """
+        Then the HTTP status code should be "200"
+        And the document in the database for id "population-estimates" should be:
+        """
+            {
+                "id": "population-estimates",
+                "survey": "mockSurvey"
+            }
+        """
+
+    Scenario: Adding topic fields to a dataset
+        Given I have these datasets:
+            """
+            [
+                {
+                    "id": "population-estimates"
+                }
+            ]
+            """
+        When I PUT "/datasets/population-estimates"
+            """
+            {
+                    "canonical_topic": "canonical-topic-ID",
+                    "subtopics": ["subtopic-ID"]
+            }
+            """
+        Then the HTTP status code should be "200"
+        And the document in the database for id "population-estimates" should be:
+        """
+            {
+                "id": "population-estimates",
+                "canonical_topic": "canonical-topic-ID",
+                "subtopics": ["subtopic-ID"]
+            }
+        """
+
+    Scenario: Removing a survey from a dataset
+        Given I have these datasets:
+            """
+            [
+                {
+                    "id": "population-estimates",
+                    "survey": "mockSurvey"
+                }
+            ]
+            """
+        When I PUT "/datasets/population-estimates"
+            """
+            {
+                "survey": ""
+            }
+            """
+        Then the document in the database for id "population-estimates" should be:
+            """
+            {
+                "id": "population-estimates"
+            }
+            """
+
     Scenario: GET /datasets
         Given I have these datasets:
             """
@@ -73,4 +146,39 @@ Feature: Private Dataset API
                 "offset": 0,
                 "total_count": 1
             }
-            """
+            """
+
+    Scenario: GET /datasets with topics included
+        Given I have these datasets:
+            """
+            [
+                {
+                    "id": "population-estimates",
+                    "canonical_topic": "canonical-topic-ID",
+                    "subtopics": ["subtopic-ID"]
+                }
+            ]
+            """
+        When I GET "/datasets"
+        Then I should receive the following JSON response with status "200":
+            """
+            {
+            	"count": 1,
+            	"items": [{
+            		"id": "population-estimates",
+            		"next": {
+            			"id": "population-estimates",
+            			"canonical_topic": "canonical-topic-ID",
+            			"subtopics": ["subtopic-ID"]
+            		},
+                    "current": {
+                        "id": "population-estimates",
+            			"canonical_topic": "canonical-topic-ID",
+            			"subtopics": ["subtopic-ID"]
+                    }
+            	}],
+            	"limit": 20,
+            	"offset": 0,
+            	"total_count": 1
+            }
+        """

features/public_datasets.feature

@@ -44,3 +44,22 @@ Feature: Dataset API
                 "id": "income-by-age"
             }
             """
+
+    Scenario: Adding topic and survey fields to a dataset
+        Given I have these datasets:
+            """
+            [
+                {
+                    "id": "population-estimates"
+                }
+            ]
+            """
+        When I PUT "/datasets/population-estimates"
+            """
+            {
+                    "canonical_topic": "canonical-topic-ID",
+                    "subtopics": ["subtopic-ID"],
+                    "survey": "mockSurvey"
+            }
+            """
+        Then the HTTP status code should be "405"

features/steps/steps.go

@@ -31,6 +31,7 @@ func init() {
 func (c *DatasetComponent) RegisterSteps(ctx *godog.ScenarioContext) {
 	ctx.Step(`^private endpoints are enabled$`, c.privateEndpointsAreEnabled)
 	ctx.Step(`^the document in the database for id "([^"]*)" should be:$`, c.theDocumentInTheDatabaseForIdShouldBe)
+	ctx.Step(`^the instance in the database for id "([^"]*)" should be:$`, c.theInstanceInTheDatabaseForIdShouldBe)
 	ctx.Step(`^there are no datasets$`, c.thereAreNoDatasets)
 	ctx.Step(`^I have these datasets:$`, c.iHaveTheseDatasets)
 	ctx.Step(`^I have these "([^"]*)" datasets:$`, c.iHaveTheseConditionalDatasets)
@@ -77,6 +78,25 @@ func (c *DatasetComponent) theDocumentInTheDatabaseForIdShouldBe(documentId stri
 	return c.ErrorFeature.StepError()
 }
 
+func (c *DatasetComponent) theInstanceInTheDatabaseForIdShouldBe(id string, body *godog.DocString) error {
+	var expected models.Instance
+
+	if err := json.Unmarshal([]byte(body.Content), &expected); err != nil {
+		return fmt.Errorf("failed to unmarshal body: %w", err)
+	}
+
+	collectionName := c.MongoClient.ActualCollectionName(config.InstanceCollection)
+	var got models.Instance
+
+	if err := c.MongoClient.Connection.Collection(collectionName).FindOne(context.Background(), bson.M{"_id": id}, &got); err != nil {
+		return fmt.Errorf("failed to get instance from collection: %w", err)
+	}
+
+	assert.Equal(&c.ErrorFeature, expected, got)
+
+	return nil
+}
+
 func (c *DatasetComponent) iHaveARealKafkaContainerWithTopic(topic string) error {
 	c.setConsumer(topic)
 	c.setInitialiserRealKafka()

go.mod

@@ -1,6 +1,6 @@
 module github.com/ONSdigital/dp-dataset-api
 
-go 1.18
+go 1.19
 
 replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible
 
@@ -20,7 +20,7 @@ require (
 	github.com/ONSdigital/dp-graph/v2 v2.15.0
 	github.com/ONSdigital/dp-healthcheck v1.3.0
 	github.com/ONSdigital/dp-kafka/v2 v2.5.0
-	github.com/ONSdigital/dp-mongodb/v3 v3.0.2
+	github.com/ONSdigital/dp-mongodb/v3 v3.3.0
 	github.com/ONSdigital/dp-net/v2 v2.4.0
 	github.com/ONSdigital/log.go/v2 v2.2.0
 	github.com/cucumber/godog v0.12.5
@@ -101,6 +101,6 @@ require (
 	golang.org/x/net v0.0.0-20220728153142-1f511ac62c11 // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -68,8 +68,8 @@ github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9 h1:+WXVfTDyW
 github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk=
 github.com/ONSdigital/dp-mongodb-in-memory v1.3.1 h1:E2xSPOBr5B0riB1qN+z+m8ouWymnLWu+XHBUpgYLXWc=
 github.com/ONSdigital/dp-mongodb-in-memory v1.3.1/go.mod h1:QuDIrDBBsmL9Jh/kmpKI1ENslO8yBCDLlAQ6qL10Pac=
-github.com/ONSdigital/dp-mongodb/v3 v3.0.2 h1:CbIGAnhMvqwfKg3Dc3LCP6ALcFV4I9vdWRDOTi/QPdc=
-github.com/ONSdigital/dp-mongodb/v3 v3.0.2/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE=
+github.com/ONSdigital/dp-mongodb/v3 v3.3.0 h1:r6zblTi/X69pNwJeTmnun4DW4GIhSaB66eT2Le1aVR0=
+github.com/ONSdigital/dp-mongodb/v3 v3.3.0/go.mod h1:5EtdWLz6X+S6mtFLvmsYqaP/uzGCFLmEbzjbepZwHgE=
 github.com/ONSdigital/dp-net v1.0.5-0.20200805082802-e518bc287596/go.mod h1:wDVhk2pYosQ1q6PXxuFIRYhYk2XX5+1CeRRnXpSczPY=
 github.com/ONSdigital/dp-net v1.0.5-0.20200805145012-9227a11caddb/go.mod h1:MrSZwDUvp8u1VJEqa+36Gwq4E7/DdceW+BDCvGes6Cs=
 github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3LB9tedE0tObBwa12dUOt5rvTW4qQkF5rXtt4b6CE=
@@ -674,8 +674,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

instance/instance.go

@@ -268,7 +268,7 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) {
 		}
 
 		// update dp-graph instance node (only for non-cantabular types)
-		if currentInstance.Type == models.CantabularBlob.String() || currentInstance.Type == models.CantabularTable.String() || currentInstance.Type == models.CantabularFlexibleTable.String() {
+		if currentInstance.Type == models.CantabularBlob.String() || currentInstance.Type == models.CantabularTable.String() || currentInstance.Type == models.CantabularFlexibleTable.String() || currentInstance.Type == models.CantabularMultivariateTable.String() {
 			editionLogData["instance_type"] = instance.Type
 			log.Info(ctx, "skipping dp-graph instance update because it is not required by instance type", editionLogData)
 		} else {