diff --git a/.gitignore b/.gitignore index d258b9e6..5303470a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ debug build .idea .vscode -def.sdmx.json \ No newline at end of file +def.sdmx.json +vendor/ \ No newline at end of file diff --git a/Makefile b/Makefile index 260d2879..018b953c 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test: .PHONY: test-component test-component: - go test -race -cover -component + go test -race -cover -coverpkg=github.com/ONSdigital/dp-dataset-api/... -component .PHONY: nomis diff --git a/api/api.go b/api/api.go index 8997ea47..8d348c2d 100644 --- a/api/api.go +++ b/api/api.go @@ -11,6 +11,7 @@ import ( "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/dimension" "github.com/ONSdigital/dp-dataset-api/instance" + "github.com/ONSdigital/dp-dataset-api/pagination" "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/dp-dataset-api/url" dphandlers "github.com/ONSdigital/dp-net/handlers" @@ -91,6 +92,8 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d maxLimit: cfg.DefaultMaxLimit, } + paginator := pagination.NewPaginator(cfg.DefaultLimit, cfg.DefaultOffset, cfg.DefaultMaxLimit) + if api.enablePrivateEndpoints { log.Event(ctx, "enabling private endpoints for dataset api", log.INFO) @@ -114,21 +117,21 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d Storer: api.dataStore.Backend, } - api.enablePrivateDatasetEndpoints(ctx) + api.enablePrivateDatasetEndpoints(ctx, paginator) api.enablePrivateInstancesEndpoints(instanceAPI) api.enablePrivateDimensionsEndpoints(dimensionAPI) } else { log.Event(ctx, "enabling only public endpoints for dataset api", log.INFO) - api.enablePublicEndpoints(ctx) + api.enablePublicEndpoints(ctx, paginator) } return api } // enablePublicEndpoints register only the public GET endpoints. -func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) { - api.get("/datasets", api.getDatasets) +func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context, paginator *pagination.Paginator) { + api.get("/datasets", paginator.Paginate(api.getDatasets)) api.get("/datasets/{dataset_id}", api.getDataset) - api.get("/datasets/{dataset_id}/editions", api.getEditions) + api.get("/datasets/{dataset_id}/editions", paginator.Paginate(api.getEditions)) api.get("/datasets/{dataset_id}/editions/{edition}", api.getEdition) api.get("/datasets/{dataset_id}/editions/{edition}/versions", api.getVersions) api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}", api.getVersion) @@ -140,10 +143,10 @@ func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) { // enablePrivateDatasetEndpoints register the datasets endpoints with the appropriate authentication and authorisation // checks required when running the dataset API in publishing (private) mode. -func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) { +func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context, paginator *pagination.Paginator) { api.get( "/datasets", - api.isAuthorised(readPermission, api.getDatasets), + api.isAuthorised(readPermission, paginator.Paginate(api.getDatasets)), ) api.get( @@ -154,7 +157,7 @@ func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) { api.get( "/datasets/{dataset_id}/editions", - api.isAuthorisedForDatasets(readPermission, api.getEditions), + api.isAuthorisedForDatasets(readPermission, paginator.Paginate(api.getEditions)), ) api.get( @@ -316,6 +319,14 @@ func (api *DatasetAPI) enablePrivateDimensionsEndpoints(dimensionAPI *dimension. dimensionAPI.GetUniqueDimensionAndOptionsHandler)), ) + api.patch( + "/instances/{instance_id}/dimensions/{dimension}/options/{option}", + api.isAuthenticated( + api.isAuthorised(updatePermission, + api.isInstancePublished(dimensionAPI.PatchOptionHandler))), + ) + + // Deprecated api.put( "/instances/{instance_id}/dimensions/{dimension}/options/{option}/node_id/{node_id}", api.isAuthenticated( @@ -358,24 +369,29 @@ func (api *DatasetAPI) isVersionPublished(action string, handler http.HandlerFun return api.versionPublishedChecker.Check(handler, action) } -// get register a GET http.HandlerFunc. +// get registers a GET http.HandlerFunc. func (api *DatasetAPI) get(path string, handler http.HandlerFunc) { - api.Router.HandleFunc(path, handler).Methods("GET") + api.Router.HandleFunc(path, handler).Methods(http.MethodGet) } -// get register a PUT http.HandlerFunc. +// put registers a PUT http.HandlerFunc. func (api *DatasetAPI) put(path string, handler http.HandlerFunc) { - api.Router.HandleFunc(path, handler).Methods("PUT") + api.Router.HandleFunc(path, handler).Methods(http.MethodPut) +} + +// patch registers a PATCH http.HandlerFunc +func (api *DatasetAPI) patch(path string, handler http.HandlerFunc) { + api.Router.HandleFunc(path, handler).Methods(http.MethodPatch) } -// get register a POST http.HandlerFunc. +// post registers a POST http.HandlerFunc. func (api *DatasetAPI) post(path string, handler http.HandlerFunc) { - api.Router.HandleFunc(path, handler).Methods("POST") + api.Router.HandleFunc(path, handler).Methods(http.MethodPost) } -// get register a DELETE http.HandlerFunc. +// delete registers a DELETE http.HandlerFunc. func (api *DatasetAPI) delete(path string, handler http.HandlerFunc) { - api.Router.HandleFunc(path, handler).Methods("DELETE") + api.Router.HandleFunc(path, handler).Methods(http.MethodDelete) } func (api *DatasetAPI) authenticate(r *http.Request, logData log.Data) bool { diff --git a/api/dataset.go b/api/dataset.go index d5b4e73a..1139894e 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -5,11 +5,11 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "time" errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-dataset-api/utils" dphttp "github.com/ONSdigital/dp-net/http" "github.com/ONSdigital/log.go/log" "github.com/gorilla/mux" @@ -42,96 +42,20 @@ var ( } ) -func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request) { +func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) { ctx := r.Context() logData := log.Data{} - offsetParameter := r.URL.Query().Get("offset") - limitParameter := r.URL.Query().Get("limit") - - offset := api.defaultOffset - limit := api.defaultLimit - - var err error - - if offsetParameter != "" { - logData["offset"] = offsetParameter - offset, err = utils.ValidatePositiveInt(offsetParameter) - if err != nil { - log.Event(ctx, "invalid query parameter: offset", log.ERROR, log.Error(err), logData) - handleDatasetAPIErr(ctx, err, w, nil) - return - } - } - - if limitParameter != "" { - logData["limit"] = limitParameter - limit, err = utils.ValidatePositiveInt(limitParameter) - if err != nil { - log.Event(ctx, "invalid query parameter: limit", log.ERROR, log.Error(err), logData) - handleDatasetAPIErr(ctx, err, w, nil) - return - } - } - - if limit > api.maxLimit { - logData["max_limit"] = api.maxLimit - err = errs.ErrInvalidQueryParameter - log.Event(ctx, "limit is greater than the maximum allowed", log.ERROR, logData) - handleDatasetAPIErr(ctx, err, w, nil) - return - } - - b, err := func() ([]byte, error) { - - logData := log.Data{} - - authorised := api.authenticate(r, logData) - - datasets, err := api.dataStore.Backend.GetDatasets(ctx, offset, limit, authorised) - if err != nil { - log.Event(ctx, "api endpoint getDatasets datastore.GetDatasets returned an error", log.ERROR, log.Error(err)) - return nil, err - } - - var b []byte - - var datasetsResponse interface{} - - if authorised { - datasetsResponse = datasets - } else { - datasetsResponse = &models.DatasetResults{ - Items: mapResults(datasets.Items), - Offset: offset, - Limit: limit, - Count: datasets.Count, - TotalCount: datasets.TotalCount, - } - - } - - b, err = json.Marshal(datasetsResponse) - - if err != nil { - log.Event(ctx, "api endpoint getDatasets failed to marshal dataset resource into bytes", log.ERROR, log.Error(err), logData) - return nil, err - } - - return b, nil - }() - + authorised := api.authenticate(r, logData) + datasets, totalCount, err := api.dataStore.Backend.GetDatasets(ctx, offset, limit, authorised) if err != nil { - handleDatasetAPIErr(ctx, err, w, nil) - return + log.Event(ctx, "api endpoint getDatasets datastore.GetDatasets returned an error", log.ERROR, log.Error(err)) + handleDatasetAPIErr(ctx, err, w, logData) + return nil, 0, err } - - setJSONContentType(w) - if _, err = w.Write(b); err != nil { - log.Event(ctx, "api endpoint getDatasets error writing response body", log.ERROR, log.Error(err)) - handleDatasetAPIErr(ctx, err, w, nil) - return + if authorised { + return datasets, totalCount, nil } - log.Event(ctx, "api endpoint getDatasets request successful", log.INFO) + return mapResults(datasets), totalCount, nil } func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { @@ -236,6 +160,13 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { return nil, err } + models.CleanDataset(dataset) + + if err = models.ValidateDataset(dataset); err != nil { + log.Event(ctx, "addDataset endpoint: dataset failed validation checks", log.ERROR, log.Error(err)) + return nil, err + } + dataset.Type = datasetType dataset.State = models.CreatedState dataset.ID = datasetID @@ -321,20 +252,19 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { return err } + models.CleanDataset(dataset) + + if err = models.ValidateDataset(dataset); err != nil { + log.Event(ctx, "putDataset endpoint: failed validation check to update dataset", log.ERROR, log.Error(err), data) + return err + } + if dataset.State == models.PublishedState { if err := api.publishDataset(ctx, currentDataset, nil); err != nil { log.Event(ctx, "putDataset endpoint: failed to update dataset document to published", log.ERROR, log.Error(err), data) return err } } else { - if err := models.CleanDataset(dataset); err != nil { - log.Event(ctx, "could not clean dataset", log.ERROR, log.Error(err)) - return nil - } - if err := models.ValidateDataset(dataset); err != nil { - log.Event(ctx, "failed validation check to update dataset", log.ERROR, log.Error(err)) - return nil - } if err := api.dataStore.Backend.UpdateDataset(ctx, datasetID, dataset, currentDataset.Next.State); err != nil { log.Event(ctx, "putDataset endpoint: failed to update dataset resource", log.ERROR, log.Error(err), data) return err @@ -410,15 +340,15 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { } // Find any editions associated with this dataset - editionDocs, err := api.dataStore.Backend.GetEditions(ctx, currentDataset.ID, "", 0, 0, true) + editionDocs, _, err := api.dataStore.Backend.GetEditions(ctx, currentDataset.ID, "", 0, 0, true) if err != nil { log.Event(ctx, "unable to find the dataset editions", log.ERROR, log.Error(errs.ErrEditionsNotFound), logData) return errs.ErrEditionsNotFound } // Then delete them - for i := range editionDocs.Items { - if err := api.dataStore.Backend.DeleteEdition(editionDocs.Items[i].ID); err != nil { + for i := range editionDocs { + if err := api.dataStore.Backend.DeleteEdition(editionDocs[i].ID); err != nil { log.Event(ctx, "failed to delete edition", log.ERROR, log.Error(err), logData) return err } @@ -455,7 +385,7 @@ func slice(full []string, offset, limit int) (sliced []string) { return full[offset:end] } -func mapResults(results []models.DatasetUpdate) []*models.Dataset { +func mapResults(results []*models.DatasetUpdate) []*models.Dataset { items := []*models.Dataset{} for _, item := range results { if item.Current == nil { @@ -478,7 +408,7 @@ func handleDatasetAPIErr(ctx context.Context, err error, w http.ResponseWriter, status = http.StatusForbidden case datasetsNoContent[err]: status = http.StatusNoContent - case datasetsBadRequest[err]: + case datasetsBadRequest[err], strings.HasPrefix(err.Error(), "invalid fields:"): status = http.StatusBadRequest case resourcesNotFound[err]: status = http.StatusNotFound diff --git a/api/dataset_test.go b/api/dataset_test.go index c8de92e1..9c3d6788 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -3,12 +3,10 @@ package api import ( "bytes" "context" - "encoding/json" "errors" "io" "net/http" "net/http/httptest" - "strings" "sync" "testing" "time" @@ -71,144 +69,56 @@ func createRequestWithAuth(method, URL string, body io.Reader) (*http.Request, e func TestGetDatasetsReturnsOK(t *testing.T) { t.Parallel() - Convey("A successful request to get dataset returns 200 OK response", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { - return &models.DatasetUpdateResults{}, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) - So(datasetPermissions.Required.Calls, ShouldEqual, 0) - So(permissions.Required.Calls, ShouldEqual, 1) - }) - - // func to unmarshal and validate body bytes - validateBody := func(bytes []byte, expected models.DatasetUpdateResults) { - var response models.DatasetUpdateResults - err := json.Unmarshal(bytes, &response) - So(err, ShouldBeNil) - So(response, ShouldResemble, expected) - } - Convey("When valid limit and offset query parameters are provided, then return datasets information according to the offset and limit", t, func() { - - r := httptest.NewRequest("GET", "http://localhost:22000/datasets?offset=2&limit=2", nil) + Convey("A successful request to get dataset returns 200 OK response, and limit and offset are delegated to the datastore", t, func() { + r := &http.Request{} w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { - return &models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{}, - Count: 2, - Offset: offset, - Limit: limit, - TotalCount: 5, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - Convey("Then the call succeeds with 200 OK code, expected body and calls", func() { - expectedResponse := models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{}, - Count: 2, - Offset: 2, - Limit: 2, - TotalCount: 5, - } - - So(w.Code, ShouldEqual, http.StatusOK) - validateBody(w.Body.Bytes(), expectedResponse) - }) - }) - - Convey("When valid limit above maximum and offset query parameters are provided, then return datasets information according to the offset and limit", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets?offset=2&limit=7", nil) - w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { - return &models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{}, - Count: 2, - Offset: offset, - Limit: limit, - TotalCount: 5, - }, nil + GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { + return []*models.DatasetUpdate{}, 15, nil }, } datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - Convey("Then the call succeeds with 200 OK code, expected body and calls", func() { - expectedResponse := models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{}, - Count: 2, - Offset: 2, - Limit: 7, - TotalCount: 5, - } - - So(w.Code, ShouldEqual, http.StatusOK) - validateBody(w.Body.Bytes(), expectedResponse) - }) - }) + actualResponse, actualTotalCount, err := api.getDatasets(w, r, 11, 12) + So(actualResponse, ShouldResemble, []*models.Dataset{}) + So(actualTotalCount, ShouldEqual, 15) + So(err, ShouldEqual, nil) + So(mockedDataStore.GetDatasetsCalls()[0].Limit, ShouldEqual, 11) + So(mockedDataStore.GetDatasetsCalls()[0].Offset, ShouldEqual, 12) + }) } func TestGetDatasetsReturnsError(t *testing.T) { t.Parallel() Convey("When the api cannot connect to datastore return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets", nil) + r := &http.Request{} w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { - return nil, errs.ErrInternalServer + GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { + return nil, 0, errs.ErrInternalServer }, } datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) + actualResponse, actualTotalCount, err := api.getDatasets(w, r, 6, 7) assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) So(datasetPermissions.Required.Calls, ShouldEqual, 0) - So(permissions.Required.Calls, ShouldEqual, 1) - }) - - Convey("When a negative limit and offset query parameters are provided, then return a 400 error", t, func() { - - r := httptest.NewRequest("GET", "http://localhost:22000/datasets?offset=-2&limit=-7", nil) - w := httptest.NewRecorder() - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(&storetest.StorerMock{}, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(datasetPermissions.Required.Calls, ShouldEqual, 0) - So(permissions.Required.Calls, ShouldEqual, 1) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrInvalidQueryParameter.Error()) - + So(permissions.Required.Calls, ShouldEqual, 0) + So(actualResponse, ShouldResemble, nil) + So(actualTotalCount, ShouldEqual, 0) + So(err, ShouldEqual, errs.ErrInternalServer) + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(string(w.Body.Bytes()), ShouldEqual, "internal error\n") }) } @@ -324,7 +234,7 @@ func TestGetDatasetReturnsError(t *testing.T) { func TestPostDatasetsReturnsCreated(t *testing.T) { t.Parallel() - Convey("A successful request to post dataset returns 200 OK response", t, func() { + Convey("A successful request to post dataset returns 201 OK response", t, func() { var b string b = datasetPayload r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) @@ -357,6 +267,181 @@ func TestPostDatasetsReturnsCreated(t *testing.T) { So(err, ShouldEqual, io.EOF) }) }) + + Convey("When creating the dataset with an empty QMI url returns 201 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(id string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(datasetPermissions.Required.Calls, ShouldEqual, 1) + So(permissions.Required.Calls, ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When creating the dataset with a valid QMI url (path in appropriate url format) returns 201 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://domain.com/path", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(id string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(datasetPermissions.Required.Calls, ShouldEqual, 1) + So(permissions.Required.Calls, ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When creating the dataset with a valid QMI url (relative path) returns 201 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "/path", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(id string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(datasetPermissions.Required.Calls, ShouldEqual, 1) + So(permissions.Required.Calls, ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When creating the dataset with a valid QMI url (valid host but an empty path) returns 201 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://domain.com/", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(id string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(datasetPermissions.Required.Calls, ShouldEqual, 1) + So(permissions.Required.Calls, ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When creating the dataset with a valid QMI url (only a valid domain) returns 201 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "domain.com", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(id string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(datasetPermissions.Required.Calls, ShouldEqual, 1) + So(permissions.Required.Calls, ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) } func TestPostDatasetReturnsError(t *testing.T) { @@ -530,9 +615,9 @@ func TestPostDatasetReturnsError(t *testing.T) { }) }) - Convey("When the request has an invalid datatype it should return invalid type errorq", t, func() { + Convey("When creating the dataset with invalid QMI url (invalid character) returns bad request", t, func() { var b string - b = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"nomis_filterable"}` + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": ":not a link", "title": "test"}}` r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) So(err, ShouldBeNil) @@ -552,9 +637,9 @@ func TestPostDatasetReturnsError(t *testing.T) { api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "invalid dataset type\n") + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(w.Code, ShouldEqual, http.StatusBadRequest) So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0) Convey("then the request body has been drained", func() { @@ -563,11 +648,11 @@ func TestPostDatasetReturnsError(t *testing.T) { }) }) - Convey("When the request body has an empty type field it should create a dataset with type defaulted to filterable", t, func() { + Convey("When creating the dataset with invalid QMI url (scheme only) returns bad request", t, func() { var b string - b = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":""}` - res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable"}}` - r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b)) + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) So(err, ShouldBeNil) w := httptest.NewRecorder() @@ -582,14 +667,13 @@ func TestPostDatasetReturnsError(t *testing.T) { datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() - mockedDataStore.UpsertDataset("123123", &models.DatasetUpdate{Next: &models.Dataset{}}) api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusCreated) - So(w.Body.String(), ShouldContainSubstring, res) + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) - So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 2) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0) Convey("then the request body has been drained", func() { _, err = r.Body.Read(make([]byte, 1)) @@ -597,38 +681,138 @@ func TestPostDatasetReturnsError(t *testing.T) { }) }) -} - -func TestPutDatasetReturnsSuccessfully(t *testing.T) { - t.Parallel() - Convey("A successful request to put dataset returns 200 OK response", t, func() { + Convey("When creating the dataset with invalid QMI url (scheme and path only) returns bad request", t, func() { var b string - b = datasetPayload - r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http:///path", "title": "test"}}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + return nil, errs.ErrDatasetNotFound }, - UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { return nil }, } datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() - - dataset := &models.Dataset{ - Title: "CPI", - } - mockedDataStore.UpdateDataset(testContext, "123", dataset, models.CreatedState) - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusOK) + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When the request has an invalid datatype it should return invalid type errorq", t, func() { + var b string + b = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"nomis_filterable"}` + + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldResemble, "invalid dataset type\n") + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When the request body has an empty type field it should create a dataset with type defaulted to filterable", t, func() { + var b string + b = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":""}` + res := `{"id":"123123","next":{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable"}}` + r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + mockedDataStore.UpsertDataset("123123", &models.DatasetUpdate{Next: &models.Dataset{}}) + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(w.Body.String(), ShouldContainSubstring, res) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 2) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + +} + +func TestPutDatasetReturnsSuccessfully(t *testing.T) { + t.Parallel() + Convey("A successful request to put dataset returns 200 OK response", t, func() { + var b string + b = datasetPayload + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + dataset := &models.Dataset{ + Title: "CPI", + } + mockedDataStore.UpdateDataset(testContext, "123", dataset, models.CreatedState) + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) So(datasetPermissions.Required.Calls, ShouldEqual, 1) So(permissions.Required.Calls, ShouldEqual, 0) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) @@ -674,6 +858,176 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(err, ShouldEqual, io.EOF) }) }) + + Convey("When updating the dataset with an empty QMI url returns 200 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with a valid QMI url (path in appropriate url format) returns 200 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://domain.com/path", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with a valid QMI url (relative path) returns 200 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "/path", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with a valid QMI url (valid host but an empty path) returns 200 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://domain.com/", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with a valid QMI url (only a valid domain) returns 200 success", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "domain.com", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) } func TestPutDatasetReturnsError(t *testing.T) { @@ -826,6 +1180,111 @@ func TestPutDatasetReturnsError(t *testing.T) { }) }) + Convey("When updating the dataset with invalid QMI url (invalid character) returns bad request", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": ":not a link", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with invalid QMI url (scheme only) returns bad request", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http://", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + + Convey("When updating the dataset with invalid QMI url (scheme and path only) returns bad request", t, func() { + var b string + b = `{"contacts": [{"email": "testing@hotmail.com", "name": "John Cox", "telephone": "01623 456789"}], "description": "census", "links": {"access_rights": {"href": "http://ons.gov.uk/accessrights"}}, "title": "CensusEthnicity", "theme": "population", "state": "completed", "next_release": "2016-04-04", "publisher": {"name": "The office of national statistics", "type": "government department", "url": "https://www.ons.gov.uk/"}, "type": "nomis", "nomis_reference_url": "https://www.nomis.co.uk", "qmi": {"href": "http:///path", "title": "test"}}` + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil + }, + UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error { + return nil + }, + } + + datasetPermissions := getAuthorisationHandlerMock() + permissions := getAuthorisationHandlerMock() + + api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldResemble, "invalid fields: [QMI]\n") + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) + + Convey("then the request body has been drained", func() { + _, err = r.Body.Read(make([]byte, 1)) + So(err, ShouldEqual, io.EOF) + }) + }) + Convey("When the request is not authorised to update dataset return status unauthorised", t, func() { var b string b = "{\"edition\":\"2017\",\"state\":\"created\",\"license\":\"ONS\",\"release_date\":\"2017-04-04\",\"version\":\"1\"}" @@ -873,8 +1332,8 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, DeleteDatasetFunc: func(string) error { return nil @@ -903,10 +1362,10 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { var items []*models.EditionUpdate items = append(items, &models.EditionUpdate{}) - return &models.EditionUpdateResults{Items: items}, nil + return items, 0, nil }, DeleteEditionFunc: func(ID string) error { return nil @@ -942,8 +1401,8 @@ func TestDeleteDatasetReturnsError(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, DeleteDatasetFunc: func(string) error { return nil @@ -974,8 +1433,8 @@ func TestDeleteDatasetReturnsError(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, DeleteDatasetFunc: func(string) error { return errs.ErrInternalServer @@ -1005,8 +1464,8 @@ func TestDeleteDatasetReturnsError(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return nil, errs.ErrDatasetNotFound }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, DeleteDatasetFunc: func(string) error { return nil @@ -1037,8 +1496,8 @@ func TestDeleteDatasetReturnsError(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return nil, errors.New("database is broken") }, - GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, ID string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, DeleteDatasetFunc: func(string) error { return nil diff --git a/api/dimensions.go b/api/dimensions.go index 616f7b5a..615b3b2d 100644 --- a/api/dimensions.go +++ b/api/dimensions.go @@ -7,6 +7,7 @@ import ( "net/http" "sort" + "github.com/ONSdigital/dp-dataset-api/apierrors" errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/utils" @@ -302,22 +303,31 @@ func handleDimensionsErr(ctx context.Context, w http.ResponseWriter, msg string, data = log.Data{} } - switch { - case errs.BadRequestMap[err]: + // Switch by error type + switch err.(type) { + case apierrors.ErrInvalidPatch: data["response_status"] = http.StatusBadRequest data["user_error"] = err.Error() log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, data) http.Error(w, err.Error(), http.StatusBadRequest) - case errs.NotFoundMap[err]: - data["response_status"] = http.StatusNotFound - data["user_error"] = err.Error() - log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, data) - http.Error(w, err.Error(), http.StatusNotFound) default: - // a stack trace is added for Non User errors - data["response_status"] = http.StatusInternalServerError - log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, log.Error(err), data) - http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + // Switch by error message + switch { + case errs.BadRequestMap[err]: + data["response_status"] = http.StatusBadRequest + data["user_error"] = err.Error() + log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, data) + http.Error(w, err.Error(), http.StatusBadRequest) + case errs.NotFoundMap[err]: + data["response_status"] = http.StatusNotFound + data["user_error"] = err.Error() + log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, data) + http.Error(w, err.Error(), http.StatusNotFound) + default: + // a stack trace is added for Non User errors + data["response_status"] = http.StatusInternalServerError + log.Event(ctx, fmt.Sprintf("request unsuccessful: %s", msg), log.ERROR, log.Error(err), data) + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + } } - } diff --git a/api/editions.go b/api/editions.go index 9da3f022..9b34da41 100644 --- a/api/editions.go +++ b/api/editions.go @@ -6,123 +6,57 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-dataset-api/utils" "github.com/ONSdigital/log.go/log" "github.com/gorilla/mux" ) -func (api *DatasetAPI) getEditions(w http.ResponseWriter, r *http.Request) { +func (api *DatasetAPI) getEditions(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) { ctx := r.Context() vars := mux.Vars(r) datasetID := vars["dataset_id"] logData := log.Data{"dataset_id": datasetID} - offsetParameter := r.URL.Query().Get("offset") - limitParameter := r.URL.Query().Get("limit") - var err error - offset := api.defaultOffset - limit := api.defaultLimit + authorised := api.authenticate(r, logData) - if offsetParameter != "" { - logData["offset"] = offsetParameter - offset, err = utils.ValidatePositiveInt(offsetParameter) - if err != nil { - log.Event(ctx, "invalid query parameter: offset", log.ERROR, log.Error(err), logData) - handleDatasetAPIErr(ctx, err, w, nil) - return - } + var state string + if !authorised { + state = models.PublishedState } - if limitParameter != "" { - logData["limit"] = limitParameter - limit, err = utils.ValidatePositiveInt(limitParameter) - if err != nil { - log.Event(ctx, "invalid query parameter: limit", log.ERROR, log.Error(err), logData) - handleDatasetAPIErr(ctx, err, w, nil) - return - } - } - - if limit > api.maxLimit { - logData["max_limit"] = api.maxLimit - err = errs.ErrInvalidQueryParameter - log.Event(ctx, "limit is greater than the maximum allowed", log.ERROR, logData) - handleDimensionsErr(ctx, w, "unpublished version has an invalid state", err, logData) - return - } - - b, err := func() ([]byte, error) { - authorised := api.authenticate(r, logData) - - var state string - if !authorised { - state = models.PublishedState - } - - logData["state"] = state - - if err := api.dataStore.Backend.CheckDatasetExists(datasetID, state); err != nil { - log.Event(ctx, "getEditions endpoint: unable to find dataset", log.ERROR, log.Error(err), logData) - return nil, err - } - - results, err := api.dataStore.Backend.GetEditions(ctx, datasetID, state, offset, limit, authorised) - if err != nil { - log.Event(ctx, "getEditions endpoint: unable to find editions for dataset", log.ERROR, log.Error(err), logData) - return nil, err - } - - var editionBytes []byte - - if authorised { - - // User has valid authentication to get raw edition document - editionBytes, err = json.Marshal(results) - if err != nil { - log.Event(ctx, "getEditions endpoint: failed to marshal a list of edition resources into bytes", log.ERROR, log.Error(err), logData) - return nil, err - } - log.Event(ctx, "getEditions endpoint: get all edition with auth", log.INFO, logData) + logData["state"] = state + if err := api.dataStore.Backend.CheckDatasetExists(datasetID, state); err != nil { + log.Event(ctx, "getEditions endpoint: unable to find dataset", log.ERROR, log.Error(err), logData) + if err == errs.ErrDatasetNotFound { + http.Error(w, err.Error(), http.StatusNotFound) } else { - // User is not authenticated and hence has only access to current sub document - var publicResults []*models.Edition - for i := range results.Items { - publicResults = append(publicResults, results.Items[i].Current) - } - - editionBytes, err = json.Marshal(&models.EditionResults{ - Items: publicResults, - Offset: offset, - Limit: limit, - Count: results.Count, - TotalCount: results.TotalCount, - }) - if err != nil { - log.Event(ctx, "getEditions endpoint: failed to marshal a list of edition resources into bytes", log.ERROR, log.Error(err), logData) - return nil, err - } - log.Event(ctx, "getEditions endpoint: get all edition without auth", log.INFO, logData) + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) } - return editionBytes, nil - }() + return nil, 0, err + } + results, totalCount, err := api.dataStore.Backend.GetEditions(ctx, datasetID, state, offset, limit, authorised) if err != nil { - if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound { + log.Event(ctx, "getEditions endpoint: unable to find editions for dataset", log.ERROR, log.Error(err), logData) + if err == errs.ErrEditionNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) } - return + return nil, 0, err } - setJSONContentType(w) - _, err = w.Write(b) - if err != nil { - log.Event(ctx, "getEditions endpoint: failed writing bytes to response", log.ERROR, log.Error(err), logData) - http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + if authorised { + log.Event(ctx, "getEditions endpoint: get all edition with auth", log.INFO, logData) + return results, totalCount, nil + } + + var publicResults []*models.Edition + for i := range results { + publicResults = append(publicResults, results[i].Current) } - log.Event(ctx, "getEditions endpoint: request successful", log.INFO, logData) + log.Event(ctx, "getEditions endpoint: get all edition without auth", log.INFO, logData) + return publicResults, totalCount, nil } func (api *DatasetAPI) getEdition(w http.ResponseWriter, r *http.Request) { diff --git a/api/editions_test.go b/api/editions_test.go index d74d8539..9c518b10 100644 --- a/api/editions_test.go +++ b/api/editions_test.go @@ -28,8 +28,8 @@ func TestGetEditionsReturnsOK(t *testing.T) { CheckDatasetExistsFunc: func(datasetID, state string) error { return nil }, - GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{}, nil + GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{}, 0, nil }, } @@ -61,19 +61,14 @@ func TestGetEditionsReturnsOK(t *testing.T) { CheckDatasetExistsFunc: func(datasetID, state string) error { return nil }, - GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{ - Items: []*models.EditionUpdate{ - {ID: "id1", - Current: &models.Edition{ - ID: "id2", - }}, + GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{ + {ID: "id1", + Current: &models.Edition{ + ID: "id2", + }, }, - Count: 1, - Offset: offset, - Limit: limit, - TotalCount: 3, - }, nil + }, 3, nil }, } @@ -106,19 +101,13 @@ func TestGetEditionsReturnsOK(t *testing.T) { CheckDatasetExistsFunc: func(datasetID, state string) error { return nil }, - GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return &models.EditionUpdateResults{ - Items: []*models.EditionUpdate{ - {ID: "id1", - Current: &models.Edition{ - ID: "id2", - }}, - }, - Count: 1, - Offset: offset, - Limit: limit, - TotalCount: 3, - }, nil + GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return []*models.EditionUpdate{ + {ID: "id1", + Current: &models.Edition{ + ID: "id2", + }}, + }, 3, nil }, } @@ -217,8 +206,8 @@ func TestGetEditionsReturnsError(t *testing.T) { CheckDatasetExistsFunc: func(datasetID, state string) error { return nil }, - GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return nil, errs.ErrEditionNotFound + GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return nil, 0, errs.ErrEditionNotFound }, } @@ -242,8 +231,8 @@ func TestGetEditionsReturnsError(t *testing.T) { CheckDatasetExistsFunc: func(datasetID, state string) error { return nil }, - GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { - return nil, errs.ErrEditionNotFound + GetEditionsFunc: func(ctx context.Context, id string, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { + return nil, 0, errs.ErrEditionNotFound }, } diff --git a/api/publish_state_checker.go b/api/publish_state_checker.go index f5311300..4d6f89f5 100644 --- a/api/publish_state_checker.go +++ b/api/publish_state_checker.go @@ -55,7 +55,7 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request), ac // TODO Logic here might require it's own endpoint, // possibly /datasets/.../versions//downloads if action == updateVersionAction { - versionDoc, err := models.CreateVersion(r.Body) + versionDoc, err := models.CreateVersion(r.Body, datasetID) if err != nil { log.Event(ctx, "failed to model version resource based on request", log.ERROR, log.Error(err), data) dphttp.DrainBody(r) diff --git a/api/versions.go b/api/versions.go index 81f81fd8..15411150 100644 --- a/api/versions.go +++ b/api/versions.go @@ -393,7 +393,7 @@ func (api *DatasetAPI) updateVersion(ctx context.Context, body io.ReadCloser, ve // attempt to update the version currentDataset, currentVersion, versionUpdate, err := func() (*models.DatasetUpdate, *models.Version, *models.Version, error) { - versionUpdate, err := models.CreateVersion(body) + versionUpdate, err := models.CreateVersion(body, versionDetails.datasetID) if err != nil { log.Event(ctx, "putVersion endpoint: failed to model version resource based on request", log.ERROR, log.Error(err), data) return nil, nil, nil, errs.ErrUnableToParseJSON diff --git a/api/webendpoints_test.go b/api/webendpoints_test.go index 825ade5b..844ecb2a 100644 --- a/api/webendpoints_test.go +++ b/api/webendpoints_test.go @@ -34,14 +34,13 @@ func TestWebSubnetDatasetsEndpoint(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { - return &models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{ - { - Current: current, - Next: next, - }}, - }, nil + GetDatasetsFunc: func(ctx context.Context, offset, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { + return []*models.DatasetUpdate{ + { + Current: current, + Next: next, + }, + }, 0, nil }, } Convey("Calling the datasets endpoint should allow only published items", func() { @@ -98,7 +97,7 @@ func TestWebSubnetEditionsEndpoint(t *testing.T) { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions", nil) So(err, ShouldBeNil) - edition := &models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} + edition := models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} var editionSearchState, datasetSearchState string w := httptest.NewRecorder() @@ -107,11 +106,9 @@ func TestWebSubnetEditionsEndpoint(t *testing.T) { datasetSearchState = state return nil }, - GetEditionsFunc: func(ctx context.Context, ID, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { + GetEditionsFunc: func(ctx context.Context, ID, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { editionSearchState = state - return &models.EditionUpdateResults{ - Items: []*models.EditionUpdate{edition}, - }, nil + return []*models.EditionUpdate{&edition}, 0, nil }, } Convey("Calling the editions endpoint should allow only published items", func() { diff --git a/apierrors/errors.go b/apierrors/errors.go index 8c69a6d1..b1c3ef60 100644 --- a/apierrors/errors.go +++ b/apierrors/errors.go @@ -4,6 +4,15 @@ import ( "errors" ) +// ErrInvalidPatch represents an error due to an invalid HTTP PATCH request +type ErrInvalidPatch struct { + Msg string +} + +func (e ErrInvalidPatch) Error() string { + return e.Msg +} + // A list of error messages for Dataset API var ( ErrAddDatasetAlreadyExists = errors.New("forbidden - dataset already exists") @@ -26,6 +35,7 @@ var ( ErrInternalServer = errors.New("internal error") ErrInsertedObservationsInvalidSyntax = errors.New("inserted observation request parameter not an integer") ErrInvalidQueryParameter = errors.New("invalid query parameter") + ErrInvalidBody = errors.New("invalid request body") ErrTooManyQueryParameters = errors.New("too many query parameters have been provided") ErrMetadataVersionNotFound = errors.New("version not found") ErrMissingJobProperties = errors.New("missing job properties") @@ -63,6 +73,7 @@ var ( BadRequestMap = map[error]bool{ ErrInsertedObservationsInvalidSyntax: true, + ErrInvalidBody: true, ErrInvalidQueryParameter: true, ErrTooManyQueryParameters: true, ErrMissingJobProperties: true, diff --git a/dimension/dimension.go b/dimension/dimension.go index d9b639fa..e1b10c23 100644 --- a/dimension/dimension.go +++ b/dimension/dimension.go @@ -3,11 +3,16 @@ package dimension import ( "context" "encoding/json" + "fmt" + "io" + "io/ioutil" "net/http" + "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" dphttp "github.com/ONSdigital/dp-net/http" + dprequest "github.com/ONSdigital/dp-net/request" "github.com/ONSdigital/log.go/log" "github.com/gorilla/mux" ) @@ -38,6 +43,7 @@ func (s *Store) GetDimensionsHandler(w http.ResponseWriter, r *http.Request) { handleDimensionErr(ctx, w, err, logData) return } + setJSONContentType(w) writeBody(ctx, w, b, logData) log.Event(ctx, "successfully get dimensions for an instance resource", log.INFO, logData) } @@ -85,6 +91,7 @@ func (s *Store) GetUniqueDimensionAndOptionsHandler(w http.ResponseWriter, r *ht handleDimensionErr(ctx, w, err, logData) return } + setJSONContentType(w) writeBody(ctx, w, b, logData) log.Event(ctx, "successfully get unique dimension options for an instance resource", log.INFO, logData) } @@ -168,9 +175,104 @@ func (s *Store) add(ctx context.Context, instanceID string, option *models.Cache return nil } +// createPatches manages the creation of an array of patch structs from the provided reader, and validates them +func createPatches(reader io.Reader) ([]dprequest.Patch, error) { + patches := []dprequest.Patch{} + + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return []dprequest.Patch{}, apierrors.ErrInvalidBody + } + + err = json.Unmarshal(bytes, &patches) + if err != nil { + return []dprequest.Patch{}, apierrors.ErrInvalidBody + } + + for _, patch := range patches { + if err := patch.Validate(dprequest.OpAdd); err != nil { + return []dprequest.Patch{}, err + } + } + return patches, nil +} + +// PatchOptionHandler updates a dimension option according to the provided patch array body +func (s *Store) PatchOptionHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + instanceID := vars["instance_id"] + dimensionName := vars["dimension"] + option := vars["option"] + logData := log.Data{"instance_id": instanceID, "dimension": dimensionName, "option": option} + + // unmarshal and validate the patch array + patches, err := createPatches(r.Body) + if err != nil { + log.Event(ctx, "error obtaining patch from request body", log.ERROR, log.Error(err), logData) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + logData["patch_list"] = patches + + // apply the patches to the dimension option + successfulPatches, err := s.patchOption(ctx, instanceID, dimensionName, option, patches, logData) + if err != nil { + logData["successful_patches"] = successfulPatches + handleDimensionErr(ctx, w, err, logData) + return + } + + // Marshal provided model + b, err := json.Marshal(successfulPatches) + if err != nil { + handleDimensionErr(ctx, w, err, logData) + return + } + + // set content type and write response body + setJSONPatchContentType(w) + writeBody(ctx, w, b, logData) + log.Event(ctx, "successfully patched dimension option of an instance resource", log.INFO, logData) +} + +func (s *Store) patchOption(ctx context.Context, instanceID, dimensionName, option string, patches []dprequest.Patch, logData log.Data) (successful []dprequest.Patch, err error) { + // apply patch operations sequentially, stop processing if one patch fails, and return a list of successful patches operations + for _, patch := range patches { + dimOption := models.DimensionOption{Name: dimensionName, Option: option, InstanceID: instanceID} + + // populate the field from the patch path + switch patch.Path { + case "/node_id": + val, ok := patch.Value.(string) + if !ok { + return successful, apierrors.ErrInvalidPatch{Msg: "wrong value type for /node_id, expected string"} + } + dimOption.NodeID = val + case "/order": + // json numeric values are always float64 + v, ok := patch.Value.(float64) + if !ok { + return successful, apierrors.ErrInvalidPatch{Msg: "wrong value type for /order, expected numeric value (float64)"} + } + val := int(v) + dimOption.Order = &val + default: + return successful, apierrors.ErrInvalidPatch{Msg: fmt.Sprintf("wrong path: %s", patch.Path)} + } + + // update values in database + if err := s.updateOption(ctx, dimOption, logData); err != nil { + return successful, err + } + successful = append(successful, patch) + } + return successful, nil +} + // AddNodeIDHandler against a specific option for dimension +// Deprecated: this method is superseded by PatchOptionHandler func (s *Store) AddNodeIDHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() vars := mux.Vars(r) instanceID := vars["instance_id"] @@ -179,9 +281,9 @@ func (s *Store) AddNodeIDHandler(w http.ResponseWriter, r *http.Request) { nodeID := vars["node_id"] logData := log.Data{"instance_id": instanceID, "dimension": dimensionName, "option": option, "node_id": nodeID, "action": UpdateNodeIDAction} - dim := models.DimensionOption{Name: dimensionName, Option: option, NodeID: nodeID, InstanceID: instanceID} + dimOption := models.DimensionOption{Name: dimensionName, Option: option, NodeID: nodeID, InstanceID: instanceID} - if err := s.addNodeID(ctx, dim, logData); err != nil { + if err := s.updateOption(ctx, dimOption, logData); err != nil { handleDimensionErr(ctx, w, err, logData) return } @@ -190,9 +292,11 @@ func (s *Store) AddNodeIDHandler(w http.ResponseWriter, r *http.Request) { log.Event(ctx, "added node id to dimension of an instance resource", log.INFO, logData) } -func (s *Store) addNodeID(ctx context.Context, dim models.DimensionOption, logData log.Data) error { +// updateOption checks that the instance is in a valid state +// and then updates nodeID and order (if provided) to the provided dimension option +func (s *Store) updateOption(ctx context.Context, dimOption models.DimensionOption, logData log.Data) error { // Get instance - instance, err := s.GetInstance(dim.InstanceID) + instance, err := s.GetInstance(dimOption.InstanceID) if err != nil { log.Event(ctx, "failed to get instance", log.ERROR, log.Error(err), logData) return err @@ -205,7 +309,7 @@ func (s *Store) addNodeID(ctx context.Context, dim models.DimensionOption, logDa return err } - if err := s.UpdateDimensionNodeID(&dim); err != nil { + if err := s.UpdateDimensionNodeIDAndOrder(&dimOption); err != nil { log.Event(ctx, "failed to update a dimension of that instance", log.ERROR, log.Error(err), logData) return err } @@ -213,9 +317,15 @@ func (s *Store) addNodeID(ctx context.Context, dim models.DimensionOption, logDa return nil } -func writeBody(ctx context.Context, w http.ResponseWriter, b []byte, data log.Data) { - +func setJSONContentType(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") +} + +func setJSONPatchContentType(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json-patch+json") +} + +func writeBody(ctx context.Context, w http.ResponseWriter, b []byte, data log.Data) { if _, err := w.Write(b); err != nil { log.Event(ctx, "failed to write response body", log.ERROR, log.Error(err), data) http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/dimension/dimension_test.go b/dimension/dimension_test.go index 16fcf9ba..adbf9a8b 100644 --- a/dimension/dimension_test.go +++ b/dimension/dimension_test.go @@ -36,6 +36,18 @@ func createRequestWithToken(method, url string, body io.Reader) (*http.Request, return r, err } +func validateDimensionUpdate(mockedDataStore *storetest.StorerMock, expected *models.DimensionOption) { + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(mockedDataStore.GetInstanceCalls(), ShouldHaveLength, 2) + So(mockedDataStore.GetInstanceCalls()[0].ID, ShouldEqual, expected.InstanceID) + So(mockedDataStore.GetInstanceCalls()[1].ID, ShouldEqual, expected.InstanceID) + + So(mockedDataStore.UpdateDimensionNodeIDAndOrderCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpdateDimensionNodeIDAndOrderCalls()[0].Dimension, ShouldResemble, expected) +} + +// Deprecated func TestAddNodeIDToDimensionReturnsOK(t *testing.T) { t.Parallel() Convey("Add node id to a dimension returns ok", t, func() { @@ -48,7 +60,7 @@ func TestAddNodeIDToDimensionReturnsOK(t *testing.T) { GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - UpdateDimensionNodeIDFunc: func(event *models.DimensionOption) error { + UpdateDimensionNodeIDAndOrderFunc: func(event *models.DimensionOption) error { return nil }, } @@ -57,41 +69,226 @@ func TestAddNodeIDToDimensionReturnsOK(t *testing.T) { datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - // Gets called twice as there is a check wrapper around this route which - // checks the instance is not published before entering handler - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 1) + Convey("And the expected database calls are performed to update nodeID and order", func() { + validateDimensionUpdate(mockedDataStore, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + NodeID: "11", + Option: "55", + Order: nil, + }) + }) }) } -func TestAddNodeIDToDimensionReturnsBadRequest(t *testing.T) { +func TestPatchOptionReturnsOK(t *testing.T) { t.Parallel() - Convey("Add node id to a dimension returns bad request", t, func() { - r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) - So(err, ShouldBeNil) + Convey("Given a Dataset API instance with a mocked data store", t, func() { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - UpdateDimensionNodeIDFunc: func(event *models.DimensionOption) error { + UpdateDimensionNodeIDAndOrderFunc: func(event *models.DimensionOption) error { + return nil + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + + Convey("Then patch dimension option with a valid node_id returns ok", func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + + Convey("And the expected database calls are performed to update node_id", func() { + validateDimensionUpdate(mockedDataStore, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + NodeID: "11", + Option: "55", + }) + }) + }) + + Convey("Then patch dimension option with a valid order returns ok", func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/order", "value": 0} + ]`) + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + + Convey("And the expected database calls are performed to update order", func() { + expectedOrder := 0 + validateDimensionUpdate(mockedDataStore, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + Option: "55", + Order: &expectedOrder, + }) + }) + }) + + Convey("Then patch dimension option with a valid order and node_id returns ok", func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/order", "value": 0}, + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + + Convey("And the expected database calls are performed to update node_id and order", func() { + So(mockedDataStore.GetInstanceCalls(), ShouldHaveLength, 3) + So(mockedDataStore.GetInstanceCalls()[0].ID, ShouldEqual, "123") + So(mockedDataStore.GetInstanceCalls()[1].ID, ShouldEqual, "123") + So(mockedDataStore.GetInstanceCalls()[2].ID, ShouldEqual, "123") + + expectedOrder := 0 + So(mockedDataStore.UpdateDimensionNodeIDAndOrderCalls(), ShouldHaveLength, 2) + So(mockedDataStore.UpdateDimensionNodeIDAndOrderCalls()[0].Dimension, ShouldResemble, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + Option: "55", + Order: &expectedOrder, + }) + So(mockedDataStore.UpdateDimensionNodeIDAndOrderCalls()[1].Dimension, ShouldResemble, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + NodeID: "11", + Option: "55", + }) + }) + }) + }) +} + +// Deprecated +func TestAddNodeIDToDimensionReturnsNotFound(t *testing.T) { + t.Parallel() + + Convey("Given a mocked Dataset API that fails to update dimension node ID due to DimensionNodeNotFound error", t, func() { + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateDimensionNodeIDAndOrderFunc: func(event *models.DimensionOption) error { return errs.ErrDimensionNodeNotFound }, } datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) - datasetAPI.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusNotFound) - // Gets called twice as there is a check wrapper around this route which - // checks the instance is not published before entering handler - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 1) + Convey("Add node id to a dimension returns status not found", func() { + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) + + Convey("And the expected database calls are performed to update nodeID", func() { + validateDimensionUpdate(mockedDataStore, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + NodeID: "11", + Option: "55", + Order: nil, + }) + }) + }) + }) +} + +func TestPatchOptionReturnsNotFound(t *testing.T) { + t.Parallel() + + Convey("Given a Dataset API instance with a mocked data store that fails to update dimension node ID due to DimensionNodeNotFound error", t, func() { + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateDimensionNodeIDAndOrderFunc: func(event *models.DimensionOption) error { + return errs.ErrDimensionNodeNotFound + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + + Convey("Then patch dimension option returns status not found", func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) + + Convey("And the expected database calls are performed to update nodeID", func() { + validateDimensionUpdate(mockedDataStore, &models.DimensionOption{ + InstanceID: "123", + Name: "age", + NodeID: "11", + Option: "55", + Order: nil, + }) + }) + }) }) } +func TestPatchOptionReturnsBadRequest(t *testing.T) { + t.Parallel() + + Convey("Given a Dataset API instance with a mocked datastore GetInstance", t, func() { + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + + bodies := map[string]io.Reader{ + "Then patch dimension option with an invalid body returns bad request": strings.NewReader(`wrong`), + "Then patch dimension option with a patch containing an unsupported method returns bad request": strings.NewReader(`[{"op": "remove", "path": "/node_id"}]`), + "Then patch dimension option with an unexpected path returns bad request": strings.NewReader(`[{"op": "add", "path": "unexpected", "value": "11"}]`), + "Then patch dimension option with an unexpected value type for /node_id path returns bad request": strings.NewReader(`[{"op": "add", "path": "/node_id", "value": 123.321}]`), + "Then patch dimension option with an unexpected value type for /order path returns bad request": strings.NewReader(`[{"op": "add", "path": "/order", "value": "notAnOrder"}]`), + } + + for msg, body := range bodies { + Convey(msg, func() { + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + datasetAPI.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(mockedDataStore.GetInstanceCalls(), ShouldHaveLength, 1) + }) + } + }) +} + +// Deprecated func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { @@ -111,7 +308,6 @@ func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { @@ -133,10 +329,81 @@ func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { // Gets called twice as there is a check wrapper around this route which // checks the instance is not published before entering handler So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) }) } +func TestPatchOptionReturnsInternalError(t *testing.T) { + t.Parallel() + + body := strings.NewReader(`[ + {"op": "add", "path": "/order", "value": 0}, + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + + Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return nil, errs.ErrInternalServer + }, + } + + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + }) + + Convey("Given instance state is invalid, then response returns an internal error", t, func() { + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: "gobbledygook"}, nil + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + }) + + Convey("Given an internal error is returned from mongo GetInstance on the second call, then response returns an internal error", t, func() { + mockedDataStore := &storetest.StorerMock{} + mockedDataStore.GetInstanceFunc = func(ID string) (*models.Instance, error) { + if len(mockedDataStore.GetInstanceCalls()) == 1 { + return &models.Instance{State: models.CreatedState}, nil + } + return nil, errs.ErrInternalServer + } + + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + }) + +} + +// Deprecated func TestAddNodeIDToDimensionReturnsForbidden(t *testing.T) { t.Parallel() Convey("Add node id to a dimension of a published instance returns forbidden", t, func() { @@ -156,10 +423,36 @@ func TestAddNodeIDToDimensionReturnsForbidden(t *testing.T) { So(w.Code, ShouldEqual, http.StatusForbidden) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) }) } +func TestPatchOptionReturnsForbidden(t *testing.T) { + t.Parallel() + Convey("Patch dimension option of a published instance returns forbidden", t, func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/order", "value": 0}, + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + r, err := createRequestWithToken(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + }) +} + +// Deprecated func TestAddNodeIDToDimensionReturnsUnauthorized(t *testing.T) { t.Parallel() Convey("Add node id to a dimension of an instance returns unauthorized", t, func() { @@ -179,14 +472,39 @@ func TestAddNodeIDToDimensionReturnsUnauthorized(t *testing.T) { So(w.Code, ShouldEqual, http.StatusUnauthorized) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) - So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) + }) +} + +func TestPatchOptionReturnsUnauthorized(t *testing.T) { + t.Parallel() + Convey("Patch option of an instance returns unauthorized", t, func() { + body := strings.NewReader(`[ + {"op": "add", "path": "/order", "value": 0}, + {"op": "add", "path": "/node_id", "value": "11"} + ]`) + r, err := http.NewRequest(http.MethodPatch, "http://localhost:21800/instances/123/dimensions/age/options/55", body) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + datasetAPI := getAPIWithMocks(testContext, mockedDataStore, &mocks.DownloadsGeneratorMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusUnauthorized) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) }) } func TestAddDimensionToInstanceReturnsOk(t *testing.T) { t.Parallel() Convey("Add a dimension to an instance returns ok", t, func() { - json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test", "order": 1}`) r, err := createRequestWithToken("POST", "http://localhost:22000/instances/123/dimensions", json) So(err, ShouldBeNil) @@ -207,7 +525,13 @@ func TestAddDimensionToInstanceReturnsOk(t *testing.T) { // Gets called twice as there is a check wrapper around this route which // checks the instance is not published before entering handler So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(mockedDataStore.GetInstanceCalls()[0].ID, ShouldEqual, "123") + So(mockedDataStore.GetInstanceCalls()[1].ID, ShouldEqual, "123") + So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 1) + So(mockedDataStore.AddDimensionToInstanceCalls()[0].Dimension.CodeList, ShouldEqual, "123-456") + So(mockedDataStore.AddDimensionToInstanceCalls()[0].Dimension.Name, ShouldEqual, "test") + So(*mockedDataStore.AddDimensionToInstanceCalls()[0].Dimension.Order, ShouldEqual, 1) }) } @@ -295,7 +619,8 @@ func TestAddDimensionToInstanceReturnsUnauthorized(t *testing.T) { func TestAddDimensionToInstanceReturnsInternalError(t *testing.T) { t.Parallel() - Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { + + Convey("Given an internal error is returned from mongo GetInstance, then response returns an internal error", t, func() { json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) So(err, ShouldBeNil) diff --git a/dimension/helpers.go b/dimension/helpers.go index 7631f01a..caf7172a 100644 --- a/dimension/helpers.go +++ b/dimension/helpers.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" + "github.com/ONSdigital/dp-dataset-api/apierrors" errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" dprequest "github.com/ONSdigital/dp-net/request" @@ -44,14 +45,22 @@ func handleDimensionErr(ctx context.Context, w http.ResponseWriter, err error, d } var status int - switch { - case errs.NotFoundMap[err]: - status = http.StatusNotFound - case errs.BadRequestMap[err]: + + // Switch by error type + switch err.(type) { + case apierrors.ErrInvalidPatch: status = http.StatusBadRequest default: - status = http.StatusInternalServerError - err = errs.ErrInternalServer + // Switch by error message + switch { + case errs.NotFoundMap[err]: + status = http.StatusNotFound + case errs.BadRequestMap[err]: + status = http.StatusBadRequest + default: + status = http.StatusInternalServerError + err = errs.ErrInternalServer + } } data["response_status"] = status diff --git a/features/editions.feature b/features/editions.feature new file mode 100644 index 00000000..15debf9d --- /dev/null +++ b/features/editions.feature @@ -0,0 +1,120 @@ +Feature: Dataset API + + Scenario: GET /datasets/{id}/editions + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "state": "published" + } + ] + """ + And I have these editions: + """ + [ + { + "id": "population-estimates", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ] + """ + When I GET "/datasets/population-estimates/editions" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [ + { + "id": "population-estimates", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ], + "limit": 20, + "offset": 0, + "total_count": 1 + } + """ + + Scenario: GET a dataset with two editions + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "state": "published" + } + ] + """ + And I have these editions: + """ + [ + { + "id": "1", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + }, + { + "id": "2", + "edition": "time-series", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ] + """ + When I GET "/datasets/population-estimates/editions" + Then I should receive the following JSON response with status "200": + """ + { + "count": 2, + "items": [ + { + "id": "1", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + }, + { + "id": "2", + "edition": "time-series", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ], + "limit": 20, + "offset": 0, + "total_count": 2 + } + """ + + + diff --git a/features/pagination.feature b/features/pagination.feature index b22c6ca2..0a89a969 100644 --- a/features/pagination.feature +++ b/features/pagination.feature @@ -1,5 +1,5 @@ Feature: Dataset Pagination - Background: + Background: Given I have these datasets: """ [ @@ -14,13 +14,13 @@ Feature: Dataset Pagination } ] """ - + Scenario: Offset skips first result of datasets when set to 1 When I GET "/datasets?offset=1" Then I should receive the following JSON response with status "200": """ { - "count":2, + "count": 2, "items": [ { "id": "income" @@ -29,26 +29,26 @@ Feature: Dataset Pagination "id": "age" } ], - "limit":20, - "offset":1, - "total_count":3 + "limit": 20, + "offset": 1, + "total_count": 3 } """ - + Scenario: Results limited to 1 when limit set to 1 When I GET "/datasets?offset=0&limit=1" Then I should receive the following JSON response with status "200": """ { - "count":1, + "count": 1, "items": [ { "id": "population-estimates" } ], - "limit":1, - "offset":0, - "total_count":3 + "limit": 1, + "offset": 0, + "total_count": 3 } """ Scenario: Second dataset returned when offset and limit set to 1 @@ -56,15 +56,28 @@ Feature: Dataset Pagination Then I should receive the following JSON response with status "200": """ { - "count":1, + "count": 1, "items": [ { "id": "income" } ], - "limit":1, - "offset":1, - "total_count":3 + "limit": 1, + "offset": 1, + "total_count": 3 + } + """ + + Scenario: No datasets returned when limit set to 0 + When I GET "/datasets?limit=0" + Then I should receive the following JSON response with status "200": + """ + { + "count": 0, + "items": [], + "limit": 0, + "offset": 0, + "total_count": 3 } """ @@ -73,11 +86,11 @@ Feature: Dataset Pagination Then I should receive the following JSON response with status "200": """ { - "count":0, + "count": 0, "items": [], - "limit":1, - "offset":4, - "total_count":3 + "limit": 1, + "offset": 4, + "total_count": 3 } """ @@ -105,18 +118,133 @@ Feature: Dataset Pagination """ invalid query parameter """ - + Scenario: Returning metadata when there are no datasets Given there are no datasets When I GET "/datasets?offset=1&limit=1" Then I should receive the following JSON response with status "200": """ { - "count":0, + "count": 0, "items": [], - "limit":1, - "offset":1, - "total_count":0 + "limit": 1, + "offset": 1, + "total_count": 0 } """ - \ No newline at end of file + + Scenario: GET a dataset with two editions with offset + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "state": "published" + } + ] + """ + And I have these editions: + """ + [ + { + "id": "1", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + }, + { + "id": "2", + "edition": "time-series", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ] + """ + When I GET "/datasets/population-estimates/editions?offset=1" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [ + { + "id": "2", + "edition": "time-series", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ], + "limit": 20, + "offset": 1, + "total_count": 2 + } + """ + + Scenario: GET a dataset with two editions with limit + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "state": "published" + } + ] + """ + And I have these editions: + """ + [ + { + "id": "1", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + }, + { + "id": "2", + "edition": "time-series", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ] + """ + When I GET "/datasets/population-estimates/editions?limit=1" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [ + { + "id": "1", + "edition": "2019", + "state": "published", + "links": { + "dataset": { + "id": "population-estimates" + } + } + } + ], + "limit": 1, + "offset": 0, + "total_count": 2 + } + """ \ No newline at end of file diff --git a/features/private_datasets.feature b/features/private_datasets.feature index 4f1b6fcf..fc6c0412 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -42,4 +42,35 @@ Feature: Private Dataset API And I should receive the following response: """ forbidden - dataset already exists + """ + + Scenario: GET /datasets + Given I have these datasets: + """ + [ + { + "id": "population-estimates" + } + ] + """ + When I GET "/datasets" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [ + { + "id": "population-estimates", + "next": { + "id": "population-estimates" + }, + "current": { + "id": "population-estimates" + } + } + ], + "limit": 20, + "offset": 0, + "total_count": 1 + } """ \ No newline at end of file diff --git a/features/steps/dataset_feature.go b/features/steps/dataset_feature.go index 2dd359e2..c67deb3c 100644 --- a/features/steps/dataset_feature.go +++ b/features/steps/dataset_feature.go @@ -87,6 +87,7 @@ func (f *DatasetFeature) RegisterSteps(ctx *godog.ScenarioContext) { ctx.Step(`^I have these datasets:$`, f.iHaveTheseDatasets) ctx.Step(`^the document in the database for id "([^"]*)" should be:$`, f.theDocumentInTheDatabaseForIdShouldBe) ctx.Step(`^there are no datasets$`, f.thereAreNoDatasets) + ctx.Step(`^I have these editions:$`, f.iHaveTheseEditions) } func (f *DatasetFeature) Reset() *DatasetFeature { @@ -226,3 +227,39 @@ func (f *DatasetFeature) theDocumentInTheDatabaseForIdShouldBe(documentId string return f.ErrorFeature.StepError() } + +func (f *DatasetFeature) iHaveTheseEditions(editionsJson *godog.DocString) error { + + editions := []models.Edition{} + m := f.MongoClient + + err := json.Unmarshal([]byte(editionsJson.Content), &editions) + if err != nil { + return err + } + s := m.Session.Copy() + defer s.Close() + + for _, editionDoc := range editions { + editionID := editionDoc.ID + + editionUp := models.EditionUpdate{ + ID: editionID, + Next: &editionDoc, + Current: &editionDoc, + } + + update := bson.M{ + "$set": editionUp, + "$setOnInsert": bson.M{ + "last_updated": time.Now(), + }, + } + _, err := s.DB(f.MongoClient.Database).C("editions").UpsertId(editionID, update) + if err != nil { + return err + } + } + + return nil +} diff --git a/go.mod b/go.mod index e9b2979c..b0526e7b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ONSdigital/dp-healthcheck v1.0.5 github.com/ONSdigital/dp-kafka/v2 v2.1.2 github.com/ONSdigital/dp-mongodb v1.5.0 - github.com/ONSdigital/dp-net v1.0.11 + github.com/ONSdigital/dp-net v1.0.12 github.com/ONSdigital/go-ns v0.0.0-20200902154605-290c8b5ba5eb github.com/ONSdigital/log.go v1.0.1 github.com/benweissmann/memongo v0.1.1 diff --git a/go.sum b/go.sum index 7196db71..59f2f02d 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3L github.com/ONSdigital/dp-net v1.0.7/go.mod h1:1QFzx32FwPKD2lgZI6MtcsUXritsBdJihlzIWDrQ/gc= github.com/ONSdigital/dp-net v1.0.9/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= github.com/ONSdigital/dp-net v1.0.10/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= -github.com/ONSdigital/dp-net v1.0.11 h1:BJi+e21NuwEaqANDhEzWeaQgPuoSWkQS49mJALgZJKs= -github.com/ONSdigital/dp-net v1.0.11/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= +github.com/ONSdigital/dp-net v1.0.12 h1:Vd06ia1FXKR9uyhzWykQ52b1LTp4N0VOLnrF7KOeP78= +github.com/ONSdigital/dp-net v1.0.12/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= github.com/ONSdigital/dp-rchttp v0.0.0-20190919143000-bb5699e6fd59/go.mod h1:KkW68U3FPuivW4ogi9L8CPKNj9ZxGko4qcUY7KoAAkQ= github.com/ONSdigital/dp-rchttp v0.0.0-20200114090501-463a529590e8/go.mod h1:821jZtK0oBsV8hjIkNr8vhAWuv0FxJBPJuAHa2B70Gk= github.com/ONSdigital/dp-rchttp v1.0.0 h1:K/1/gDtfMZCX1Mbmq80nZxzDirzneqA1c89ea26FqP4= @@ -206,7 +206,6 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/main_test.go b/main_test.go index c5d9c7ea..17c5f301 100644 --- a/main_test.go +++ b/main_test.go @@ -2,6 +2,7 @@ package main import ( "flag" + "fmt" "os" "testing" @@ -76,6 +77,10 @@ func TestMain(t *testing.T) { Options: &opts, }.Run() + fmt.Println("=================================") + fmt.Printf("Component test coverage: %.2f%%\n", testing.Coverage()*100) + fmt.Println("=================================") + os.Exit(status) } else { t.Skip("component flag required to run component tests") diff --git a/mocks/generate_downloads_mocks.go b/mocks/generate_downloads_mocks.go index 4edbab0e..0c49a1fa 100755 --- a/mocks/generate_downloads_mocks.go +++ b/mocks/generate_downloads_mocks.go @@ -7,6 +7,10 @@ import ( "sync" ) +var ( + lockKafkaProducerMockOutput sync.RWMutex +) + // KafkaProducerMock is a mock implementation of download.KafkaProducer. // // func TestSomethingThatUsesKafkaProducer(t *testing.T) { @@ -32,7 +36,6 @@ type KafkaProducerMock struct { Output []struct { } } - lockOutput sync.RWMutex } // Output calls OutputFunc. @@ -42,9 +45,9 @@ func (mock *KafkaProducerMock) Output() chan []byte { } callInfo := struct { }{} - mock.lockOutput.Lock() + lockKafkaProducerMockOutput.Lock() mock.calls.Output = append(mock.calls.Output, callInfo) - mock.lockOutput.Unlock() + lockKafkaProducerMockOutput.Unlock() return mock.OutputFunc() } @@ -55,12 +58,16 @@ func (mock *KafkaProducerMock) OutputCalls() []struct { } { var calls []struct { } - mock.lockOutput.RLock() + lockKafkaProducerMockOutput.RLock() calls = mock.calls.Output - mock.lockOutput.RUnlock() + lockKafkaProducerMockOutput.RUnlock() return calls } +var ( + lockGenerateDownloadsEventMockMarshal sync.RWMutex +) + // GenerateDownloadsEventMock is a mock implementation of download.GenerateDownloadsEvent. // // func TestSomethingThatUsesGenerateDownloadsEvent(t *testing.T) { @@ -88,7 +95,6 @@ type GenerateDownloadsEventMock struct { S interface{} } } - lockMarshal sync.RWMutex } // Marshal calls MarshalFunc. @@ -101,9 +107,9 @@ func (mock *GenerateDownloadsEventMock) Marshal(s interface{}) ([]byte, error) { }{ S: s, } - mock.lockMarshal.Lock() + lockGenerateDownloadsEventMockMarshal.Lock() mock.calls.Marshal = append(mock.calls.Marshal, callInfo) - mock.lockMarshal.Unlock() + lockGenerateDownloadsEventMockMarshal.Unlock() return mock.MarshalFunc(s) } @@ -116,8 +122,8 @@ func (mock *GenerateDownloadsEventMock) MarshalCalls() []struct { var calls []struct { S interface{} } - mock.lockMarshal.RLock() + lockGenerateDownloadsEventMockMarshal.RLock() calls = mock.calls.Marshal - mock.lockMarshal.RUnlock() + lockGenerateDownloadsEventMockMarshal.RUnlock() return calls } diff --git a/mocks/mocks.go b/mocks/mocks.go index aef0ba64..2badf57c 100755 --- a/mocks/mocks.go +++ b/mocks/mocks.go @@ -8,6 +8,10 @@ import ( "sync" ) +var ( + lockDownloadsGeneratorMockGenerate sync.RWMutex +) + // DownloadsGeneratorMock is a mock implementation of api.DownloadsGenerator. // // func TestSomethingThatUsesDownloadsGenerator(t *testing.T) { @@ -43,7 +47,6 @@ type DownloadsGeneratorMock struct { Version string } } - lockGenerate sync.RWMutex } // Generate calls GenerateFunc. @@ -64,9 +67,9 @@ func (mock *DownloadsGeneratorMock) Generate(ctx context.Context, datasetID stri Edition: edition, Version: version, } - mock.lockGenerate.Lock() + lockDownloadsGeneratorMockGenerate.Lock() mock.calls.Generate = append(mock.calls.Generate, callInfo) - mock.lockGenerate.Unlock() + lockDownloadsGeneratorMockGenerate.Unlock() return mock.GenerateFunc(ctx, datasetID, instanceID, edition, version) } @@ -87,8 +90,8 @@ func (mock *DownloadsGeneratorMock) GenerateCalls() []struct { Edition string Version string } - mock.lockGenerate.RLock() + lockDownloadsGeneratorMockGenerate.RLock() calls = mock.calls.Generate - mock.lockGenerate.RUnlock() + lockDownloadsGeneratorMockGenerate.RUnlock() return calls } diff --git a/models/dataset.go b/models/dataset.go index 20c2efc4..aa74c904 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -62,16 +62,6 @@ type DatasetResults struct { TotalCount int `json:"total_count"` } -// DatasetUpdateResults represents a structure for a list of evolving dataset -// with the current dataset and the updated dataset -type DatasetUpdateResults struct { - Items []DatasetUpdate `json:"items"` - Count int `json:"count"` - Offset int `json:"offset"` - Limit int `json:"limit"` - TotalCount int `json:"total_count"` -} - // EditionResults represents a structure for a list of editions for a dataset type EditionResults struct { Items []*Edition `json:"items"` @@ -207,6 +197,7 @@ type Publisher struct { type Version struct { Alerts *[]Alert `bson:"alerts,omitempty" json:"alerts,omitempty"` CollectionID string `bson:"collection_id,omitempty" json:"collection_id,omitempty"` + DatasetID string `bson:"-" json:"dataset_id,omitempty"` Dimensions []Dimension `bson:"dimensions,omitempty" json:"dimensions,omitempty"` Downloads *DownloadList `bson:"downloads,omitempty" json:"downloads,omitempty"` Edition string `bson:"edition,omitempty" json:"edition,omitempty"` @@ -296,17 +287,15 @@ func CreateDataset(reader io.Reader) (*Dataset, error) { } // CreateVersion manages the creation of a version from a reader -func CreateVersion(reader io.Reader) (*Version, error) { +func CreateVersion(reader io.Reader, datasetID string) (*Version, error) { b, err := ioutil.ReadAll(reader) if err != nil { return nil, errs.ErrUnableToReadMessage } - // Create unique id - id := uuid.NewV4() - var version Version - version.ID = id.String() + version.ID = uuid.NewV4().String() + version.DatasetID = datasetID err = json.Unmarshal(b, &version) if err != nil { @@ -453,12 +442,17 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL } // CleanDataset trims URI and any hrefs contained in the database -func CleanDataset(dataset *Dataset) error { - if dataset == nil { - return errors.New("clean dataset called without a valid dataset") - } +func CleanDataset(dataset *Dataset) { dataset.URI = strings.TrimSpace(dataset.URI) + if dataset.QMI != nil { + dataset.QMI.HRef = strings.TrimSpace(dataset.QMI.HRef) + } + + if dataset.Publisher != nil { + dataset.Publisher.HRef = strings.TrimSpace(dataset.Publisher.HRef) + } + for i := range dataset.Publications { dataset.Publications[i].HRef = strings.TrimSpace(dataset.Publications[i].HRef) } @@ -470,18 +464,21 @@ func CleanDataset(dataset *Dataset) error { for i := range dataset.RelatedDatasets { dataset.RelatedDatasets[i].HRef = strings.TrimSpace(dataset.RelatedDatasets[i].HRef) } - return nil } // ValidateDataset checks the dataset has invalid fields func ValidateDataset(dataset *Dataset) error { - var invalidFields []string if dataset.URI != "" { - _, err := url.Parse(dataset.URI) - if err != nil { - invalidFields = append(invalidFields, "URI") - } + invalidFields = append(invalidFields, validateUrlString(dataset.URI, "URI")...) + } + + if dataset.QMI != nil && dataset.QMI.HRef != "" { + invalidFields = append(invalidFields, validateUrlString(dataset.QMI.HRef, "QMI")...) + } + + if dataset.Publisher != nil && dataset.Publisher.HRef != "" { + invalidFields = append(invalidFields, validateUrlString(dataset.Publisher.HRef, "Publisher")...) } invalidFields = append(invalidFields, validateGeneralDetails(dataset.Publications, "Publications")...) @@ -490,20 +487,24 @@ func ValidateDataset(dataset *Dataset) error { invalidFields = append(invalidFields, validateGeneralDetails(dataset.Methodologies, "Methodologies")...) - if invalidFields != nil { + if len(invalidFields) > 0 { return fmt.Errorf("invalid fields: %v", invalidFields) } return nil - } func validateGeneralDetails(generalDetails []GeneralDetails, identifier string) (invalidFields []string) { for i, gd := range generalDetails { - _, err := url.Parse(gd.HRef) - if err != nil { - invalidFields = append(invalidFields, fmt.Sprintf("%s[%d].HRef", identifier, i)) - } + invalidFields = append(invalidFields, validateUrlString(gd.HRef, fmt.Sprintf("%s[%d].HRef", identifier, i))...) + } + return +} + +func validateUrlString(urlString string, identifier string) (invalidFields []string) { + url, err := url.Parse(urlString) + if err != nil || (url.Scheme != "" && url.Host == "" && url.Path == "") || (url.Scheme != "" && url.Host == "" && url.Path != "") { + invalidFields = append(invalidFields, identifier) } return } diff --git a/models/dataset_test.go b/models/dataset_test.go index 6eefce56..f8387ddb 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -15,7 +15,7 @@ import ( const ( validURI = "http://localhost:22000/datasets/123" - validHref = "http://localhost:22000//datasets/href" + validHref = "http://localhost:22000/datasets/href" invalidHref = ":invalid" ) @@ -23,6 +23,14 @@ func createDataset() Dataset { return Dataset{ ID: "123", URI: validURI, + QMI: &GeneralDetails { + Description: "some qmi description", + HRef: validHref, + Title: "some qmi title", + }, + Publisher: &Publisher { + HRef: validHref, + }, Publications: []GeneralDetails{{ Description: "some publication description", HRef: validHref, @@ -182,44 +190,47 @@ func TestCreateDataset(t *testing.T) { } func TestCreateVersion(t *testing.T) { - t.Parallel() - Convey("Successfully return without any errors", t, func() { - Convey("when the version has all fields", func() { - b, err := json.Marshal(associatedVersion) + t.Parallel() + Convey("Successfully return without any errors", t, func() { + Convey("when the version has all fields", func() { + testDatasetID := "test-dataset-id" + b, err := json.Marshal(associatedVersion) + if err != nil { + t.Logf("failed to marshal test data into bytes, error: %v", err) + t.FailNow() + } + r := bytes.NewReader(b) + version, err := CreateVersion(r, testDatasetID) + So(err, ShouldBeNil) + So(version.CollectionID, ShouldEqual, collectionID) + So(version.Dimensions, ShouldResemble, []Dimension{dimension}) + So(version.DatasetID, ShouldEqual, testDatasetID) + So(version.Downloads, ShouldResemble, &downloads) + So(version.Edition, ShouldEqual, "2017") + So(version.ID, ShouldNotBeNil) + So(version.ReleaseDate, ShouldEqual, "2017-10-12") + So(version.LatestChanges, ShouldResemble, &[]LatestChange{latestChange}) + So(version.Links.Spatial.HRef, ShouldEqual, "http://ons.gov.uk/geographylist") + So(version.State, ShouldEqual, AssociatedState) + So(version.Temporal, ShouldResemble, &[]TemporalFrequency{temporal}) + So(version.Version, ShouldEqual, 1) + }) + }) + + Convey("Return with error when the request body contains the correct fields but of the wrong type", t, func() { + testDatasetID := "test-dataset-id" + b, err := json.Marshal(badInputData) if err != nil { t.Logf("failed to marshal test data into bytes, error: %v", err) t.FailNow() } r := bytes.NewReader(b) - version, err := CreateVersion(r) - So(err, ShouldBeNil) - So(version.CollectionID, ShouldEqual, collectionID) - So(version.Dimensions, ShouldResemble, []Dimension{dimension}) - So(version.Downloads, ShouldResemble, &downloads) - So(version.Edition, ShouldEqual, "2017") - So(version.ID, ShouldNotBeNil) - So(version.ReleaseDate, ShouldEqual, "2017-10-12") - So(version.LatestChanges, ShouldResemble, &[]LatestChange{latestChange}) - So(version.Links.Spatial.HRef, ShouldEqual, "http://ons.gov.uk/geographylist") - So(version.State, ShouldEqual, AssociatedState) - So(version.Temporal, ShouldResemble, &[]TemporalFrequency{temporal}) - So(version.Version, ShouldEqual, 1) + version, err := CreateVersion(r, testDatasetID) + So(version, ShouldBeNil) + So(err, ShouldNotBeNil) + So(err, ShouldResemble, errs.ErrUnableToParseJSON) }) - }) - - Convey("Return with error when the request body contains the correct fields but of the wrong type", t, func() { - b, err := json.Marshal(badInputData) - if err != nil { - t.Logf("failed to marshal test data into bytes, error: %v", err) - t.FailNow() - } - r := bytes.NewReader(b) - version, err := CreateVersion(r) - So(version, ShouldBeNil) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errs.ErrUnableToParseJSON) - }) -} + } func TestCleanDataset(t *testing.T) { t.Parallel() @@ -227,8 +238,7 @@ func TestCleanDataset(t *testing.T) { Convey("A clean dataset stays unmodified", t, func() { Convey("When a clean dataset is cleaned, the URI and hrefs stay the same", func() { dataset := createDataset() - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.URI, ShouldEqual, validURI) So(dataset.Publications, ShouldHaveLength, 1) So(dataset.Publications[0].HRef, ShouldEqual, validHref) @@ -239,24 +249,35 @@ func TestCleanDataset(t *testing.T) { Convey("When a dataset URI has leading space it is trimmed", func() { dataset := createDataset() dataset.URI = " " + validURI - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.URI, ShouldEqual, validURI) }) Convey("When a dataset URI has trailing space it is trimmed", func() { dataset := createDataset() dataset.URI = validURI + " " - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.URI, ShouldEqual, validURI) }) + Convey("When a QMI HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.QMI.HRef = " " + validHref + CleanDataset(&dataset) + So(dataset.QMI.HRef, ShouldEqual, validHref) + }) + + Convey("When a Publisher HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.Publisher.HRef = " " + validHref + CleanDataset(&dataset) + So(dataset.Publisher.HRef, ShouldEqual, validHref) + }) + Convey("When a Publications HRef has whitespace it is trimmed", func() { dataset := createDataset() dataset.Publications[0].HRef = " " + validHref - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.Publications, ShouldHaveLength, 1) So(dataset.Publications[0].HRef, ShouldEqual, validHref) }) @@ -265,8 +286,7 @@ func TestCleanDataset(t *testing.T) { dataset := createDataset() dataset.Publications[0].HRef = " " + validHref dataset.Publications = append(dataset.Publications, GeneralDetails{HRef: validHref + " "}) - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.Publications, ShouldHaveLength, 2) So(dataset.Publications[0].HRef, ShouldEqual, validHref) So(dataset.Publications[1].HRef, ShouldEqual, validHref) @@ -275,8 +295,7 @@ func TestCleanDataset(t *testing.T) { Convey("When a Methodologies HRef has whitespace it is trimmed", func() { dataset := createDataset() dataset.Methodologies[0].HRef = " " + validHref - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.Methodologies, ShouldHaveLength, 1) So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) }) @@ -285,8 +304,7 @@ func TestCleanDataset(t *testing.T) { dataset := createDataset() dataset.Methodologies[0].HRef = " " + validHref dataset.Methodologies = append(dataset.Methodologies, GeneralDetails{HRef: validHref + " "}) - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.Methodologies, ShouldHaveLength, 2) So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) So(dataset.Methodologies[1].HRef, ShouldEqual, validHref) @@ -295,8 +313,7 @@ func TestCleanDataset(t *testing.T) { Convey("When a RelatedDatasets HRef has whitespace it is trimmed", func() { dataset := createDataset() dataset.RelatedDatasets[0].HRef = " " + validHref - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.RelatedDatasets, ShouldHaveLength, 1) So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) }) @@ -305,22 +322,13 @@ func TestCleanDataset(t *testing.T) { dataset := createDataset() dataset.RelatedDatasets[0].HRef = " " + validHref dataset.RelatedDatasets = append(dataset.RelatedDatasets, GeneralDetails{HRef: validHref + " "}) - cleanErr := CleanDataset(&dataset) - So(cleanErr, ShouldBeNil) + CleanDataset(&dataset) So(dataset.RelatedDatasets, ShouldHaveLength, 2) So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) So(dataset.RelatedDatasets[1].HRef, ShouldEqual, validHref) }) }) - - Convey("A nil dataset returns an error when cleaned", t, func() { - Convey("A nil dataset returns an error", func() { - cleanErr := CleanDataset(nil) - So(cleanErr, ShouldNotBeNil) - So(cleanErr.Error(), ShouldResemble, errors.New("clean dataset called without a valid dataset").Error()) - }) - }) } func TestValidateDataset(t *testing.T) { @@ -328,8 +336,36 @@ func TestValidateDataset(t *testing.T) { Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI contains its path in appropriate url format ", func() { + Convey("when dataset.URI contains its path in appropriate url format", func() { + dataset := createDataset() + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldBeNil) + }) + + Convey("when dataset.URI is empty", func() { dataset := createDataset() + dataset.URI ="" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldBeNil) + }) + + Convey("when dataset.URI is a relative path", func() { + dataset := createDataset() + dataset.URI = "/relative_path" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldBeNil) + }) + + Convey("when dataset.URI has a valid host but an empty path", func() { + dataset := createDataset() + dataset.URI = "http://domain.com/" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldBeNil) + }) + + Convey("when dataset.URI is only a valid domain", func() { + dataset := createDataset() + dataset.URI = "domain.com" validationErr := ValidateDataset(&dataset) So(validationErr, ShouldBeNil) }) @@ -340,12 +376,43 @@ func TestValidateDataset(t *testing.T) { Convey("when dataset.URI is unable to be parsed into url format", func() { dataset := createDataset() dataset.URI = ":foo" - fmt.Println(dataset.URI) validationErr := ValidateDataset(&dataset) So(validationErr, ShouldNotBeNil) So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) }) + Convey("when dataset.URI has an empty host and path", func() { + dataset := createDataset() + dataset.URI = "http://" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + }) + + Convey("when dataset.URI has an empty host but a non empty path", func() { + dataset := createDataset() + dataset.URI = "http:///path" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + }) + + Convey("when dataset.QMI.Href is unable to be parsed into url format", func() { + dataset := createDataset() + dataset.QMI.HRef = ":foo" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [QMI]").Error()) + }) + + Convey("when dataset.Publisher.Href is unable to be parsed into url format", func() { + dataset := createDataset() + dataset.Publisher.HRef = ":foo" + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [Publisher]").Error()) + }) + Convey("when Publications href is unable to be parsed into url format", func() { dataset := createDataset() dataset.Publications[0].HRef = invalidHref diff --git a/models/dimension.go b/models/dimension.go index ed318e7e..38977f4c 100644 --- a/models/dimension.go +++ b/models/dimension.go @@ -48,6 +48,7 @@ type CachedDimensionOption struct { Name string `bson:"name,omitempty" json:"dimension"` NodeID string `bson:"node_id,omitempty" json:"node_id"` Option string `bson:"option,omitempty" json:"option"` + Order *int `bson:"order,omitempty" json:"order"` } // DimensionOption contains unique information and metadata used when processing the data @@ -59,6 +60,7 @@ type DimensionOption struct { Name string `bson:"name,omitempty" json:"dimension"` NodeID string `bson:"node_id,omitempty" json:"node_id"` Option string `bson:"option,omitempty" json:"option"` + Order *int `bson:"order,omitempty" json:"order"` } // PublicDimensionOption hides values which are only used by interval services diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index cb184ffb..fc024d0e 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -74,7 +74,7 @@ func (m *Mongo) Checker(ctx context.Context, state *healthcheck.CheckState) erro } // GetDatasets retrieves all dataset documents -func (m *Mongo) GetDatasets(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) { +func (m *Mongo) GetDatasets(ctx context.Context, offset, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { s := m.Session.Copy() defer s.Close() @@ -90,27 +90,12 @@ func (m *Mongo) GetDatasets(ctx context.Context, offset, limit int, authorised b if err != nil { log.Event(ctx, "error counting items", log.ERROR, log.Error(err)) if err == mgo.ErrNotFound { - return &models.DatasetUpdateResults{ - Items: []models.DatasetUpdate{}, - Count: 0, - TotalCount: 0, - Offset: offset, - Limit: limit, - }, nil + return []*models.DatasetUpdate{}, totalCount, nil } - return nil, err + return nil, 0, err } - iter := q.Sort().Skip(offset).Limit(limit).Iter() - - defer func() { - err := iter.Close() - if err != nil { - log.Event(ctx, "error closing iterator", log.ERROR, log.Error(err)) - } - }() - - values := []models.DatasetUpdate{} + values := []*models.DatasetUpdate{} if limit > 0 { iter := q.Sort().Skip(offset).Limit(limit).Iter() @@ -123,25 +108,13 @@ func (m *Mongo) GetDatasets(ctx context.Context, offset, limit int, authorised b if err := iter.All(&values); err != nil { if err == mgo.ErrNotFound { - return &models.DatasetUpdateResults{ - Items: values, - Count: 0, - TotalCount: totalCount, - Offset: offset, - Limit: limit, - }, nil + return values, totalCount, nil } - return nil, err + return nil, 0, err } } - return &models.DatasetUpdateResults{ - Items: values, - Count: len(values), - TotalCount: totalCount, - Offset: offset, - Limit: limit, - }, nil + return values, totalCount, nil } // GetDataset retrieves a dataset document @@ -161,7 +134,7 @@ func (m *Mongo) GetDataset(id string) (*models.DatasetUpdate, error) { } // GetEditions retrieves all edition documents for a dataset -func (m *Mongo) GetEditions(ctx context.Context, id, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) { +func (m *Mongo) GetEditions(ctx context.Context, id, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { s := m.Session.Copy() defer s.Close() @@ -172,19 +145,13 @@ func (m *Mongo) GetEditions(ctx context.Context, id, state string, offset, limit if err != nil { log.Event(ctx, "error counting items", log.ERROR, log.Error(err)) if err == mgo.ErrNotFound { - return &models.EditionUpdateResults{ - Items: []*models.EditionUpdate{}, - Count: 0, - TotalCount: 0, - Offset: offset, - Limit: limit, - }, nil + return []*models.EditionUpdate{}, 0, nil } - return nil, err + return nil, 0, err } if totalCount < 1 { - return nil, errs.ErrEditionNotFound + return nil, 0, errs.ErrEditionNotFound } var results []*models.EditionUpdate @@ -200,25 +167,13 @@ func (m *Mongo) GetEditions(ctx context.Context, id, state string, offset, limit if err := iter.All(&results); err != nil { if err == mgo.ErrNotFound { - return &models.EditionUpdateResults{ - Items: []*models.EditionUpdate{}, - Count: 0, - TotalCount: totalCount, - Offset: offset, - Limit: limit, - }, nil + return []*models.EditionUpdate{}, 0, err } - return nil, err + return nil, 0, err } } - return &models.EditionUpdateResults{ - Items: results, - Count: len(results), - TotalCount: totalCount, - Offset: offset, - Limit: limit, - }, nil + return results, totalCount, nil } func buildEditionsQuery(id, state string, authorised bool) bson.M { @@ -304,13 +259,13 @@ func (m *Mongo) GetNextVersion(datasetID, edition string) (int, error) { } // GetVersions retrieves all version documents for a dataset edition -func (m *Mongo) GetVersions(ctx context.Context, id, editionID, state string, offset, limit int) (*models.VersionResults, error) { +func (m *Mongo) GetVersions(ctx context.Context, datasetID, editionID, state string, offset, limit int) (*models.VersionResults, error) { s := m.Session.Copy() defer s.Close() var q *mgo.Query - selector := buildVersionsQuery(id, editionID, state) + selector := buildVersionsQuery(datasetID, editionID, state) q = s.DB(m.Database).C("instances").Find(selector) totalCount, err := q.Count() @@ -359,6 +314,7 @@ func (m *Mongo) GetVersions(ctx context.Context, id, editionID, state string, of for i := 0; i < len(results); i++ { results[i].Links.Self.HRef = results[i].Links.Version.HRef + results[i].DatasetID = datasetID } return &models.VersionResults{ @@ -370,11 +326,11 @@ func (m *Mongo) GetVersions(ctx context.Context, id, editionID, state string, of }, nil } -func buildVersionsQuery(id, editionID, state string) bson.M { +func buildVersionsQuery(datasetID, editionID, state string) bson.M { var selector bson.M if state == "" { selector = bson.M{ - "links.dataset.id": id, + "links.dataset.id": datasetID, "edition": editionID, "$or": []interface{}{ bson.M{"state": models.EditionConfirmedState}, @@ -384,7 +340,7 @@ func buildVersionsQuery(id, editionID, state string) bson.M { } } else { selector = bson.M{ - "links.dataset.id": id, + "links.dataset.id": datasetID, "edition": editionID, "state": state, } diff --git a/mongo/dimension_store.go b/mongo/dimension_store.go index 233d1060..876009c6 100644 --- a/mongo/dimension_store.go +++ b/mongo/dimension_store.go @@ -7,6 +7,7 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" ) @@ -34,8 +35,13 @@ func (m *Mongo) GetUniqueDimensionAndOptions(id, dimension string) (*models.Dime s := m.Session.Copy() defer s.Close() + q, err := m.sortedQuery(s, bson.M{"instance_id": id, "name": dimension}) + if err != nil { + return nil, err + } + var values []string - err := s.DB(m.Database).C(dimensionOptions).Find(bson.M{"instance_id": id, "name": dimension}).Distinct("option", &values) + err = q.Distinct("option", &values) if err != nil { return nil, err } @@ -53,6 +59,7 @@ func (m *Mongo) AddDimensionToInstance(opt *models.CachedDimensionOption) error defer s.Close() option := models.DimensionOption{InstanceID: opt.InstanceID, Option: opt.Option, Name: opt.Name, Label: opt.Label} + option.Order = opt.Order option.Links.CodeList = models.LinkObject{ID: opt.CodeList, HRef: fmt.Sprintf("%s/code-lists/%s", m.CodeListURL, opt.CodeList)} option.Links.Code = models.LinkObject{ID: opt.Code, HRef: fmt.Sprintf("%s/code-lists/%s/codes/%s", m.CodeListURL, opt.CodeList, opt.Code)} @@ -97,20 +104,32 @@ func (m *Mongo) GetDimensionOptions(version *models.Version, dimension string, o s := m.Session.Copy() defer s.Close() - q := s.DB(m.Database).C(dimensionOptions).Find(bson.M{"instance_id": version.ID, "name": dimension}) - totalCount, err := q.Count() + // define selector to obtain all the dimension options for an instance + selector := bson.M{"instance_id": version.ID, "name": dimension} + + // get total count of items + totalCount, err := s.DB(m.Database).C(dimensionOptions).Find(selector).Count() if err != nil { return nil, err } var values []models.PublicDimensionOption - if limit > 0 { - iter := q.Sort("option").Skip(offset).Limit(limit).Iter() + if limit > 0 && totalCount > 0 { + + // obtain query defining the order + q, err := m.sortedQuery(s, selector) + if err != nil { + return nil, err + } + + // obtain only the necessary items according to offset and limit + iter := q.Skip(offset).Limit(limit).Iter() if err := iter.All(&values); err != nil { return nil, err } + // update links for returned values for i := 0; i < len(values); i++ { values[i].Links.Version = *version.Links.Self } @@ -138,23 +157,31 @@ func (m *Mongo) GetDimensionOptionsFromIDs(version *models.Version, dimension st selectorInList := bson.M{"instance_id": version.ID, "name": dimension, "option": bson.M{"$in": IDs}} // count total number of options in dimension - q := s.DB(m.Database).C(dimensionOptions).Find(selectorAll) - totalCount, err := q.Count() + totalCount, err := s.DB(m.Database).C(dimensionOptions).Find(selectorAll).Count() if err != nil { return nil, err } - // obtain only options matching the provided IDs - q = s.DB(m.Database).C(dimensionOptions).Find(selectorInList) - var values []models.PublicDimensionOption - iter := q.Sort("option").Iter() - if err := iter.All(&values); err != nil { - return nil, err - } - for i := 0; i < len(values); i++ { - values[i].Links.Version = *version.Links.Self + if totalCount > 0 { + + // obtain query defining the order for the provided IDs only + q, err := m.sortedQuery(s, selectorInList) + if err != nil { + return nil, err + } + + // obtain all required options in order + iter := q.Iter() + if err := iter.All(&values); err != nil { + return nil, err + } + + // update links for returned values + for i := 0; i < len(values); i++ { + values[i].Links.Version = *version.Links.Self + } } return &models.DimensionOptionResults{ @@ -165,3 +192,22 @@ func (m *Mongo) GetDimensionOptionsFromIDs(version *models.Version, dimension st Limit: 0, }, nil } + +// sortedQuery generates a sorted mongoDB query from the provided bson.M selector +// if order property exists, it will be used to determine the order +// otherwise, the items will be sorted alphabetically by option +func (m *Mongo) sortedQuery(s *mgo.Session, selector bson.M) (*mgo.Query, error) { + q := s.DB(m.Database).C(dimensionOptions).Find(selector) + + selector["order"] = bson.M{"$exists": true} + orderCount, err := s.DB(m.Database).C(dimensionOptions).Find(selector).Count() + if err != nil { + return nil, err + } + delete(selector, "order") + + if orderCount > 0 { + return q.Sort("order"), nil + } + return q.Sort("option"), nil +} diff --git a/mongo/instance_store.go b/mongo/instance_store.go index de7c9f4d..8bf9db50 100644 --- a/mongo/instance_store.go +++ b/mongo/instance_store.go @@ -265,13 +265,28 @@ func (m *Mongo) AddEventToInstance(instanceID string, event *models.Event) error return nil } -// UpdateDimensionNodeID to cache the id for other import processes -func (m *Mongo) UpdateDimensionNodeID(dimension *models.DimensionOption) error { +// UpdateDimensionNodeIDAndOrder to cache the id and order (optional) for other import processes +func (m *Mongo) UpdateDimensionNodeIDAndOrder(dimension *models.DimensionOption) error { + + // validate that there is something to update + if dimension.Order == nil && dimension.NodeID == "" { + return nil + } + s := m.Session.Copy() defer s.Close() - err := s.DB(m.Database).C(dimensionOptions).Update(bson.M{"instance_id": dimension.InstanceID, "name": dimension.Name, - "option": dimension.Option}, bson.M{"$set": bson.M{"node_id": &dimension.NodeID, "last_updated": time.Now().UTC()}}) + selector := bson.M{"instance_id": dimension.InstanceID, "name": dimension.Name, "option": dimension.Option} + + update := bson.M{"last_updated": time.Now().UTC()} + if dimension.NodeID != "" { + update["node_id"] = &dimension.NodeID + } + if dimension.Order != nil { + update["order"] = &dimension.Order + } + + err := s.DB(m.Database).C(dimensionOptions).Update(selector, bson.M{"$set": update}) if err == mgo.ErrNotFound { return errs.ErrDimensionOptionNotFound } diff --git a/pagination/pagination.go b/pagination/pagination.go new file mode 100644 index 00000000..f39cb010 --- /dev/null +++ b/pagination/pagination.go @@ -0,0 +1,133 @@ +package pagination + +import ( + "encoding/json" + "errors" + "net/http" + "reflect" + "strconv" + + "github.com/ONSdigital/log.go/log" +) + +// ListFetcher is an interface for an endpoint that returns a list of values that we want to paginate +type ListFetcher func(w http.ResponseWriter, r *http.Request, limit int, offset int) (list interface{}, totalCount int, err error) + +type page struct { + Items interface{} `json:"items"` + Count int `json:"count"` + Offset int `json:"offset"` + Limit int `json:"limit"` + TotalCount int `json:"total_count"` +} + +type Paginator struct { + DefaultLimit int + DefaultOffset int + DefaultMaxLimit int +} + +func NewPaginator(defaultLimit, defaultOffset, defaultMaxLimit int) *Paginator { + + return &Paginator{ + DefaultLimit: defaultLimit, + DefaultOffset: defaultOffset, + DefaultMaxLimit: defaultMaxLimit, + } +} + +func (p *Paginator) getPaginationParameters(w http.ResponseWriter, r *http.Request) (offset int, limit int, err error) { + + logData := log.Data{} + offsetParameter := r.URL.Query().Get("offset") + limitParameter := r.URL.Query().Get("limit") + + offset = p.DefaultOffset + limit = p.DefaultLimit + + if offsetParameter != "" { + logData["offset"] = offsetParameter + offset, err = strconv.Atoi(offsetParameter) + if err != nil || offset < 0 { + err = errors.New("invalid query parameter") + log.Event(r.Context(), "invalid query parameter: offset", log.ERROR, log.Error(err), logData) + return 0, 0, err + } + } + + if limitParameter != "" { + logData["limit"] = limitParameter + limit, err = strconv.Atoi(limitParameter) + if err != nil || limit < 0 { + err = errors.New("invalid query parameter") + log.Event(r.Context(), "invalid query parameter: limit", log.ERROR, log.Error(err), logData) + return 0, 0, err + } + } + + if limit > p.DefaultMaxLimit { + logData["max_limit"] = p.DefaultMaxLimit + err = errors.New("invalid query parameter") + log.Event(r.Context(), "limit is greater than the maximum allowed", log.ERROR, logData) + return 0, 0, err + } + return +} + +func renderPage(list interface{}, offset int, limit int, totalCount int) page { + + return page{ + Items: list, + Count: listLength(list), + Offset: offset, + Limit: limit, + TotalCount: totalCount, + } +} + +func listLength(list interface{}) int { + l := reflect.ValueOf(list) + return l.Len() +} + +// Paginate wraps a http endpoint to return a paginated list from the list returned by the provided function +func (p *Paginator) Paginate(listFetcher ListFetcher) func(w http.ResponseWriter, r *http.Request) { + + return func(w http.ResponseWriter, r *http.Request) { + offset, limit, err := p.getPaginationParameters(w, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + list, totalCount, err := listFetcher(w, r, limit, offset) + if err != nil { + return + } + + page := renderPage(list, offset, limit, totalCount) + + returnPaginatedResults(w, r, page) + } +} + +func returnPaginatedResults(w http.ResponseWriter, r *http.Request, list page) { + + logData := log.Data{"path": r.URL.Path, "method": r.Method} + + b, err := json.Marshal(list) + + if err != nil { + log.Event(r.Context(), "api endpoint failed to marshal resource into bytes", log.ERROR, log.Error(err), logData) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + + if _, err = w.Write(b); err != nil { + log.Event(r.Context(), "api endpoint error writing response body", log.ERROR, log.Error(err), logData) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + log.Event(r.Context(), "api endpoint request successful", log.INFO, logData) +} diff --git a/pagination/pagination_test.go b/pagination/pagination_test.go new file mode 100644 index 00000000..230b3afd --- /dev/null +++ b/pagination/pagination_test.go @@ -0,0 +1,235 @@ +package pagination + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPaginationParametersReturnsErrorWhenOffsetIsNegative(t *testing.T) { + + r := httptest.NewRequest("GET", "/test?offset=-1", nil) + w := httptest.NewRecorder() + paginator := &Paginator{} + + offset, limit, err := paginator.getPaginationParameters(w, r) + + assert.Equal(t, errors.New("invalid query parameter"), err) + assert.Equal(t, 0, offset) + assert.Equal(t, 0, limit) +} + +func TestGetPaginationParametersReturnsErrorWhenLimitIsNegative(t *testing.T) { + + r := httptest.NewRequest("GET", "/test?limit=-1", nil) + w := httptest.NewRecorder() + paginator := &Paginator{} + + offset, limit, err := paginator.getPaginationParameters(w, r) + + assert.Equal(t, errors.New("invalid query parameter"), err) + assert.Equal(t, 0, offset) + assert.Equal(t, 0, limit) +} + +func TestGetPaginationParametersReturnsErrorWhenLimitIsGreaterThanMaxLimit(t *testing.T) { + + r := httptest.NewRequest("GET", "/test?limit=1001", nil) + w := httptest.NewRecorder() + paginator := &Paginator{DefaultMaxLimit: 1000} + + offset, limit, err := paginator.getPaginationParameters(w, r) + + assert.Equal(t, errors.New("invalid query parameter"), err) + assert.Equal(t, 0, offset) + assert.Equal(t, 0, limit) +} + +func TestGetPaginationParametersReturnsLimitAndOffsetProvidedFromQuery(t *testing.T) { + + r := httptest.NewRequest("GET", "/test?limit=10&offset=5", nil) + w := httptest.NewRecorder() + paginator := &Paginator{DefaultMaxLimit: 1000} + + offset, limit, err := paginator.getPaginationParameters(w, r) + + assert.Equal(t, nil, err) + assert.Equal(t, 5, offset) + assert.Equal(t, 10, limit) +} + +func TestGetPaginationParametersReturnsDefaultValuesWhenNotProvided(t *testing.T) { + + r := httptest.NewRequest("GET", "/test", nil) + w := httptest.NewRecorder() + paginator := &Paginator{DefaultLimit: 20, DefaultOffset: 1, DefaultMaxLimit: 1000} + + offset, limit, err := paginator.getPaginationParameters(w, r) + + assert.Equal(t, nil, err) + assert.Equal(t, 1, offset) + assert.Equal(t, 20, limit) +} + +func TestRenderPageReturnsPageStrucWithFilledValues(t *testing.T) { + + expectedPage := page{ + Items: []int{1, 2, 3}, + Count: 3, + Offset: 0, + Limit: 10, + TotalCount: 3, + } + list := []int{1, 2, 3} + actualPage := renderPage(list, 0, 10, 3) + + assert.Equal(t, expectedPage, actualPage) +} + +func TestRenderPageTakesListOfAnyType(t *testing.T) { + + type custom struct { + name string + } + + expectedPage := page{ + Items: []custom{}, + Count: 0, + Offset: 0, + Limit: 20, + TotalCount: 0, + } + list := []custom{} + actualPage := renderPage(list, 0, 20, 0) + + assert.Equal(t, expectedPage, actualPage) +} + +func TestNewPaginatorReturnsPaginatorStructWithFilledValues(t *testing.T) { + + expectedPaginator := &Paginator{ + DefaultLimit: 10, + DefaultOffset: 5, + DefaultMaxLimit: 100, + } + actualPaginator := NewPaginator(10, 5, 100) + + assert.Equal(t, expectedPaginator, actualPaginator) +} + +func TestReturnPaginatedResultsWritesJSONPageToHTTPResponseBody(t *testing.T) { + + r := httptest.NewRequest("GET", "/test", nil) + w := httptest.NewRecorder() + inputPage := page{ + Items: []int{1, 2, 3}, + Count: 3, + Offset: 0, + Limit: 20, + TotalCount: 3, + } + expectedPage := page{ + Items: []int{1, 2, 3}, + Count: 3, + Offset: 0, + Limit: 20, + TotalCount: 3, + } + returnPaginatedResults(w, r, inputPage) + + content, _ := ioutil.ReadAll(w.Body) + expectedContent, _ := json.Marshal(expectedPage) + assert.Equal(t, expectedContent, content) + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + assert.Equal(t, 200, w.Code) +} + +func TestReturnPaginatedResultsReturnsErrorIfCanNotMarshalJSON(t *testing.T) { + + r := httptest.NewRequest("GET", "/test", nil) + w := httptest.NewRecorder() + inputPage := page{ + Items: make(chan int), + Count: 3, + Offset: 0, + Limit: 20, + TotalCount: 3, + } + + returnPaginatedResults(w, r, inputPage) + content, _ := ioutil.ReadAll(w.Body) + + assert.Equal(t, "internal error\n", string(content)) + assert.Equal(t, 500, w.Code) +} + +func TestPaginateFunctionPassesParametersDownToProvidedFunction(t *testing.T) { + r := httptest.NewRequest("GET", "/test?limit=1&offset=2", nil) + w := httptest.NewRecorder() + + fetchListFunc := func(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) { + return []int{limit, offset}, 10, nil + } + + paginator := &Paginator{ + DefaultLimit: 10, + DefaultOffset: 0, + DefaultMaxLimit: 100, + } + + paginatedHandler := paginator.Paginate(fetchListFunc) + + expectedPage := page{ + Items: []int{1, 2}, + Count: 2, + Offset: 2, + Limit: 1, + TotalCount: 10, + } + + paginatedHandler(w, r) + + content, _ := ioutil.ReadAll(w.Body) + expectedContent, _ := json.Marshal(expectedPage) + + assert.Equal(t, string(expectedContent), string(content)) + assert.Equal(t, 200, w.Code) +} + +func TestPaginateFunctionReturnsBadRequestWhenInvalidQueryParametersAreGiven(t *testing.T) { + r := httptest.NewRequest("GET", "/test?limit=-1", nil) + w := httptest.NewRecorder() + fetchListFunc := func(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) { + return []int{}, 0, nil + } + + paginator := &Paginator{} + paginatedHandler := paginator.Paginate(fetchListFunc) + + paginatedHandler(w, r) + content, _ := ioutil.ReadAll(w.Body) + assert.Equal(t, 400, w.Code) + assert.Equal(t, "invalid query parameter\n", string(content)) +} + +func TestPaginateFunctionReturnsListFuncImplementedHttpErrorIfListFuncReturnsAnError(t *testing.T) { + r := httptest.NewRequest("GET", "/test", nil) + w := httptest.NewRecorder() + fetchListFunc := func(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) { + http.Error(w, "internal error", http.StatusInternalServerError) + return 0, 0, errors.New("internal error") + } + + paginator := &Paginator{} + paginatedHandler := paginator.Paginate(fetchListFunc) + + paginatedHandler(w, r) + content, _ := ioutil.ReadAll(w.Body) + assert.Equal(t, 500, w.Code) + assert.Equal(t, "internal error\n", string(content)) +} diff --git a/service/mock/closer.go b/service/mock/closer.go index 84d5601d..7282cbc1 100644 --- a/service/mock/closer.go +++ b/service/mock/closer.go @@ -5,9 +5,18 @@ package mock import ( "context" + "github.com/ONSdigital/dp-dataset-api/service" "sync" ) +var ( + lockCloserMockClose sync.RWMutex +) + +// Ensure, that CloserMock does implement service.Closer. +// If this is not the case, regenerate this file with moq. +var _ service.Closer = &CloserMock{} + // CloserMock is a mock implementation of service.Closer. // // func TestSomethingThatUsesCloser(t *testing.T) { @@ -35,7 +44,6 @@ type CloserMock struct { Ctx context.Context } } - lockClose sync.RWMutex } // Close calls CloseFunc. @@ -48,9 +56,9 @@ func (mock *CloserMock) Close(ctx context.Context) error { }{ Ctx: ctx, } - mock.lockClose.Lock() + lockCloserMockClose.Lock() mock.calls.Close = append(mock.calls.Close, callInfo) - mock.lockClose.Unlock() + lockCloserMockClose.Unlock() return mock.CloseFunc(ctx) } @@ -63,8 +71,8 @@ func (mock *CloserMock) CloseCalls() []struct { var calls []struct { Ctx context.Context } - mock.lockClose.RLock() + lockCloserMockClose.RLock() calls = mock.calls.Close - mock.lockClose.RUnlock() + lockCloserMockClose.RUnlock() return calls } diff --git a/service/mock/healthcheck.go b/service/mock/healthcheck.go index 92c4bda5..f91f78dd 100644 --- a/service/mock/healthcheck.go +++ b/service/mock/healthcheck.go @@ -5,12 +5,23 @@ package mock import ( "context" + "github.com/ONSdigital/dp-dataset-api/service" + "github.com/ONSdigital/dp-healthcheck/healthcheck" "net/http" "sync" +) - "github.com/ONSdigital/dp-healthcheck/healthcheck" +var ( + lockHealthCheckerMockAddCheck sync.RWMutex + lockHealthCheckerMockHandler sync.RWMutex + lockHealthCheckerMockStart sync.RWMutex + lockHealthCheckerMockStop sync.RWMutex ) +// Ensure, that HealthCheckerMock does implement service.HealthChecker. +// If this is not the case, regenerate this file with moq. +var _ service.HealthChecker = &HealthCheckerMock{} + // HealthCheckerMock is a mock implementation of service.HealthChecker. // // func TestSomethingThatUsesHealthChecker(t *testing.T) { @@ -73,10 +84,6 @@ type HealthCheckerMock struct { Stop []struct { } } - lockAddCheck sync.RWMutex - lockHandler sync.RWMutex - lockStart sync.RWMutex - lockStop sync.RWMutex } // AddCheck calls AddCheckFunc. @@ -91,9 +98,9 @@ func (mock *HealthCheckerMock) AddCheck(name string, checker healthcheck.Checker Name: name, Checker: checker, } - mock.lockAddCheck.Lock() + lockHealthCheckerMockAddCheck.Lock() mock.calls.AddCheck = append(mock.calls.AddCheck, callInfo) - mock.lockAddCheck.Unlock() + lockHealthCheckerMockAddCheck.Unlock() return mock.AddCheckFunc(name, checker) } @@ -108,9 +115,9 @@ func (mock *HealthCheckerMock) AddCheckCalls() []struct { Name string Checker healthcheck.Checker } - mock.lockAddCheck.RLock() + lockHealthCheckerMockAddCheck.RLock() calls = mock.calls.AddCheck - mock.lockAddCheck.RUnlock() + lockHealthCheckerMockAddCheck.RUnlock() return calls } @@ -126,9 +133,9 @@ func (mock *HealthCheckerMock) Handler(w http.ResponseWriter, req *http.Request) W: w, Req: req, } - mock.lockHandler.Lock() + lockHealthCheckerMockHandler.Lock() mock.calls.Handler = append(mock.calls.Handler, callInfo) - mock.lockHandler.Unlock() + lockHealthCheckerMockHandler.Unlock() mock.HandlerFunc(w, req) } @@ -143,9 +150,9 @@ func (mock *HealthCheckerMock) HandlerCalls() []struct { W http.ResponseWriter Req *http.Request } - mock.lockHandler.RLock() + lockHealthCheckerMockHandler.RLock() calls = mock.calls.Handler - mock.lockHandler.RUnlock() + lockHealthCheckerMockHandler.RUnlock() return calls } @@ -159,9 +166,9 @@ func (mock *HealthCheckerMock) Start(ctx context.Context) { }{ Ctx: ctx, } - mock.lockStart.Lock() + lockHealthCheckerMockStart.Lock() mock.calls.Start = append(mock.calls.Start, callInfo) - mock.lockStart.Unlock() + lockHealthCheckerMockStart.Unlock() mock.StartFunc(ctx) } @@ -174,9 +181,9 @@ func (mock *HealthCheckerMock) StartCalls() []struct { var calls []struct { Ctx context.Context } - mock.lockStart.RLock() + lockHealthCheckerMockStart.RLock() calls = mock.calls.Start - mock.lockStart.RUnlock() + lockHealthCheckerMockStart.RUnlock() return calls } @@ -187,9 +194,9 @@ func (mock *HealthCheckerMock) Stop() { } callInfo := struct { }{} - mock.lockStop.Lock() + lockHealthCheckerMockStop.Lock() mock.calls.Stop = append(mock.calls.Stop, callInfo) - mock.lockStop.Unlock() + lockHealthCheckerMockStop.Unlock() mock.StopFunc() } @@ -200,8 +207,8 @@ func (mock *HealthCheckerMock) StopCalls() []struct { } { var calls []struct { } - mock.lockStop.RLock() + lockHealthCheckerMockStop.RLock() calls = mock.calls.Stop - mock.lockStop.RUnlock() + lockHealthCheckerMockStop.RUnlock() return calls } diff --git a/service/mock/initialiser.go b/service/mock/initialiser.go index 35ab9b75..f4b5d785 100644 --- a/service/mock/initialiser.go +++ b/service/mock/initialiser.go @@ -5,15 +5,26 @@ package mock import ( "context" - "net/http" - "sync" - "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/service" "github.com/ONSdigital/dp-dataset-api/store" - kafka "github.com/ONSdigital/dp-kafka/v2" + "github.com/ONSdigital/dp-kafka/v2" + "net/http" + "sync" +) + +var ( + lockInitialiserMockDoGetGraphDB sync.RWMutex + lockInitialiserMockDoGetHTTPServer sync.RWMutex + lockInitialiserMockDoGetHealthCheck sync.RWMutex + lockInitialiserMockDoGetKafkaProducer sync.RWMutex + lockInitialiserMockDoGetMongoDB sync.RWMutex ) +// Ensure, that InitialiserMock does implement service.Initialiser. +// If this is not the case, regenerate this file with moq. +var _ service.Initialiser = &InitialiserMock{} + // InitialiserMock is a mock implementation of service.Initialiser. // // func TestSomethingThatUsesInitialiser(t *testing.T) { @@ -97,11 +108,6 @@ type InitialiserMock struct { Cfg *config.Configuration } } - lockDoGetGraphDB sync.RWMutex - lockDoGetHTTPServer sync.RWMutex - lockDoGetHealthCheck sync.RWMutex - lockDoGetKafkaProducer sync.RWMutex - lockDoGetMongoDB sync.RWMutex } // DoGetGraphDB calls DoGetGraphDBFunc. @@ -114,9 +120,9 @@ func (mock *InitialiserMock) DoGetGraphDB(ctx context.Context) (store.GraphDB, s }{ Ctx: ctx, } - mock.lockDoGetGraphDB.Lock() + lockInitialiserMockDoGetGraphDB.Lock() mock.calls.DoGetGraphDB = append(mock.calls.DoGetGraphDB, callInfo) - mock.lockDoGetGraphDB.Unlock() + lockInitialiserMockDoGetGraphDB.Unlock() return mock.DoGetGraphDBFunc(ctx) } @@ -129,9 +135,9 @@ func (mock *InitialiserMock) DoGetGraphDBCalls() []struct { var calls []struct { Ctx context.Context } - mock.lockDoGetGraphDB.RLock() + lockInitialiserMockDoGetGraphDB.RLock() calls = mock.calls.DoGetGraphDB - mock.lockDoGetGraphDB.RUnlock() + lockInitialiserMockDoGetGraphDB.RUnlock() return calls } @@ -147,9 +153,9 @@ func (mock *InitialiserMock) DoGetHTTPServer(bindAddr string, router http.Handle BindAddr: bindAddr, Router: router, } - mock.lockDoGetHTTPServer.Lock() + lockInitialiserMockDoGetHTTPServer.Lock() mock.calls.DoGetHTTPServer = append(mock.calls.DoGetHTTPServer, callInfo) - mock.lockDoGetHTTPServer.Unlock() + lockInitialiserMockDoGetHTTPServer.Unlock() return mock.DoGetHTTPServerFunc(bindAddr, router) } @@ -164,9 +170,9 @@ func (mock *InitialiserMock) DoGetHTTPServerCalls() []struct { BindAddr string Router http.Handler } - mock.lockDoGetHTTPServer.RLock() + lockInitialiserMockDoGetHTTPServer.RLock() calls = mock.calls.DoGetHTTPServer - mock.lockDoGetHTTPServer.RUnlock() + lockInitialiserMockDoGetHTTPServer.RUnlock() return calls } @@ -186,9 +192,9 @@ func (mock *InitialiserMock) DoGetHealthCheck(cfg *config.Configuration, buildTi GitCommit: gitCommit, Version: version, } - mock.lockDoGetHealthCheck.Lock() + lockInitialiserMockDoGetHealthCheck.Lock() mock.calls.DoGetHealthCheck = append(mock.calls.DoGetHealthCheck, callInfo) - mock.lockDoGetHealthCheck.Unlock() + lockInitialiserMockDoGetHealthCheck.Unlock() return mock.DoGetHealthCheckFunc(cfg, buildTime, gitCommit, version) } @@ -207,9 +213,9 @@ func (mock *InitialiserMock) DoGetHealthCheckCalls() []struct { GitCommit string Version string } - mock.lockDoGetHealthCheck.RLock() + lockInitialiserMockDoGetHealthCheck.RLock() calls = mock.calls.DoGetHealthCheck - mock.lockDoGetHealthCheck.RUnlock() + lockInitialiserMockDoGetHealthCheck.RUnlock() return calls } @@ -225,9 +231,9 @@ func (mock *InitialiserMock) DoGetKafkaProducer(ctx context.Context, cfg *config Ctx: ctx, Cfg: cfg, } - mock.lockDoGetKafkaProducer.Lock() + lockInitialiserMockDoGetKafkaProducer.Lock() mock.calls.DoGetKafkaProducer = append(mock.calls.DoGetKafkaProducer, callInfo) - mock.lockDoGetKafkaProducer.Unlock() + lockInitialiserMockDoGetKafkaProducer.Unlock() return mock.DoGetKafkaProducerFunc(ctx, cfg) } @@ -242,9 +248,9 @@ func (mock *InitialiserMock) DoGetKafkaProducerCalls() []struct { Ctx context.Context Cfg *config.Configuration } - mock.lockDoGetKafkaProducer.RLock() + lockInitialiserMockDoGetKafkaProducer.RLock() calls = mock.calls.DoGetKafkaProducer - mock.lockDoGetKafkaProducer.RUnlock() + lockInitialiserMockDoGetKafkaProducer.RUnlock() return calls } @@ -260,9 +266,9 @@ func (mock *InitialiserMock) DoGetMongoDB(ctx context.Context, cfg *config.Confi Ctx: ctx, Cfg: cfg, } - mock.lockDoGetMongoDB.Lock() + lockInitialiserMockDoGetMongoDB.Lock() mock.calls.DoGetMongoDB = append(mock.calls.DoGetMongoDB, callInfo) - mock.lockDoGetMongoDB.Unlock() + lockInitialiserMockDoGetMongoDB.Unlock() return mock.DoGetMongoDBFunc(ctx, cfg) } @@ -277,8 +283,8 @@ func (mock *InitialiserMock) DoGetMongoDBCalls() []struct { Ctx context.Context Cfg *config.Configuration } - mock.lockDoGetMongoDB.RLock() + lockInitialiserMockDoGetMongoDB.RLock() calls = mock.calls.DoGetMongoDB - mock.lockDoGetMongoDB.RUnlock() + lockInitialiserMockDoGetMongoDB.RUnlock() return calls } diff --git a/service/mock/server.go b/service/mock/server.go index 8590dab4..5b0d31e2 100644 --- a/service/mock/server.go +++ b/service/mock/server.go @@ -5,9 +5,19 @@ package mock import ( "context" + "github.com/ONSdigital/dp-dataset-api/service" "sync" ) +var ( + lockHTTPServerMockListenAndServe sync.RWMutex + lockHTTPServerMockShutdown sync.RWMutex +) + +// Ensure, that HTTPServerMock does implement service.HTTPServer. +// If this is not the case, regenerate this file with moq. +var _ service.HTTPServer = &HTTPServerMock{} + // HTTPServerMock is a mock implementation of service.HTTPServer. // // func TestSomethingThatUsesHTTPServer(t *testing.T) { @@ -44,8 +54,6 @@ type HTTPServerMock struct { Ctx context.Context } } - lockListenAndServe sync.RWMutex - lockShutdown sync.RWMutex } // ListenAndServe calls ListenAndServeFunc. @@ -55,9 +63,9 @@ func (mock *HTTPServerMock) ListenAndServe() error { } callInfo := struct { }{} - mock.lockListenAndServe.Lock() + lockHTTPServerMockListenAndServe.Lock() mock.calls.ListenAndServe = append(mock.calls.ListenAndServe, callInfo) - mock.lockListenAndServe.Unlock() + lockHTTPServerMockListenAndServe.Unlock() return mock.ListenAndServeFunc() } @@ -68,9 +76,9 @@ func (mock *HTTPServerMock) ListenAndServeCalls() []struct { } { var calls []struct { } - mock.lockListenAndServe.RLock() + lockHTTPServerMockListenAndServe.RLock() calls = mock.calls.ListenAndServe - mock.lockListenAndServe.RUnlock() + lockHTTPServerMockListenAndServe.RUnlock() return calls } @@ -84,9 +92,9 @@ func (mock *HTTPServerMock) Shutdown(ctx context.Context) error { }{ Ctx: ctx, } - mock.lockShutdown.Lock() + lockHTTPServerMockShutdown.Lock() mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo) - mock.lockShutdown.Unlock() + lockHTTPServerMockShutdown.Unlock() return mock.ShutdownFunc(ctx) } @@ -99,8 +107,8 @@ func (mock *HTTPServerMock) ShutdownCalls() []struct { var calls []struct { Ctx context.Context } - mock.lockShutdown.RLock() + lockHTTPServerMockShutdown.RLock() calls = mock.calls.Shutdown - mock.lockShutdown.RUnlock() + lockHTTPServerMockShutdown.RUnlock() return calls } diff --git a/service/service.go b/service/service.go index 41f344b9..c87e7988 100644 --- a/service/service.go +++ b/service/service.go @@ -109,10 +109,16 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin store := store.DataStore{Backend: DatsetAPIStore{svc.mongoDB, svc.graphDB}} // Get GenerateDownloads Kafka Producer - svc.generateDownloadsProducer, err = svc.serviceList.GetProducer(ctx, svc.config) - if err != nil { - log.Event(ctx, "could not obtain generate downloads producer", log.FATAL, log.Error(err)) - return err + if !svc.config.EnablePrivateEndpoints { + log.Event(ctx, "skipping kafka producer creation, because it is not required by the enabled endpoints", log.INFO, log.Data{ + "EnablePrivateEndpoints": svc.config.EnablePrivateEndpoints, + }) + } else { + svc.generateDownloadsProducer, err = svc.serviceList.GetProducer(ctx, svc.config) + if err != nil { + log.Event(ctx, "could not obtain generate downloads producer", log.FATAL, log.Error(err)) + return err + } } downloadGenerator := &download.Generator{ @@ -148,7 +154,9 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin svc.healthCheck.Start(ctx) // Log kafka producer errors in parallel go-routine - svc.generateDownloadsProducer.Channels().LogErrors(ctx, "generate downloads producer error") + if svc.config.EnablePrivateEndpoints { + svc.generateDownloadsProducer.Channels().LogErrors(ctx, "generate downloads producer error") + } // Run the http server in a new go-routine go func() { @@ -301,15 +309,20 @@ func (svc *Service) registerCheckers(ctx context.Context) (err error) { hasErrors := false if svc.config.EnablePrivateEndpoints { + log.Event(ctx, "adding kafka, zebedee and graph db health check as the private endpoints are enabled", log.INFO) if err = svc.healthCheck.AddCheck("Zebedee", svc.identityClient.Checker); err != nil { hasErrors = true log.Event(ctx, "error adding check for zebedeee", log.ERROR, log.Error(err)) } - } + if err = svc.healthCheck.AddCheck("Kafka Generate Downloads Producer", svc.generateDownloadsProducer.Checker); err != nil { + hasErrors = true + log.Event(ctx, "error adding check for kafka downloads producer", log.ERROR, log.Error(err)) + } - if err = svc.healthCheck.AddCheck("Kafka Generate Downloads Producer", svc.generateDownloadsProducer.Checker); err != nil { - hasErrors = true - log.Event(ctx, "error adding check for kafka downloads producer", log.ERROR, log.Error(err)) + if err = svc.healthCheck.AddCheck("Graph DB", svc.graphDB.Checker); err != nil { + hasErrors = true + log.Event(ctx, "error adding check for graph db", log.ERROR, log.Error(err)) + } } if err = svc.healthCheck.AddCheck("Mongo DB", svc.mongoDB.Checker); err != nil { @@ -317,14 +330,6 @@ func (svc *Service) registerCheckers(ctx context.Context) (err error) { log.Event(ctx, "error adding check for mongo db", log.ERROR, log.Error(err)) } - if svc.config.EnablePrivateEndpoints { - log.Event(ctx, "adding graph db health check as the private endpoints are enabled", log.INFO) - if err = svc.healthCheck.AddCheck("Graph DB", svc.graphDB.Checker); err != nil { - hasErrors = true - log.Event(ctx, "error adding check for graph db", log.ERROR, log.Error(err)) - } - } - if hasErrors { return errors.New("Error(s) registering checkers for healthcheck") } diff --git a/service/service_test.go b/service/service_test.go index 0593b59c..2c697830 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -223,8 +223,8 @@ func TestRun(t *testing.T) { So(len(hcMockAddFail.AddCheckCalls()), ShouldEqual, 4) So(hcMockAddFail.AddCheckCalls()[0].Name, ShouldResemble, "Zebedee") So(hcMockAddFail.AddCheckCalls()[1].Name, ShouldResemble, "Kafka Generate Downloads Producer") - So(hcMockAddFail.AddCheckCalls()[2].Name, ShouldResemble, "Mongo DB") - So(hcMockAddFail.AddCheckCalls()[3].Name, ShouldResemble, "Graph DB") + So(hcMockAddFail.AddCheckCalls()[2].Name, ShouldResemble, "Graph DB") + So(hcMockAddFail.AddCheckCalls()[3].Name, ShouldResemble, "Mongo DB") }) }) @@ -254,8 +254,8 @@ func TestRun(t *testing.T) { So(len(hcMock.AddCheckCalls()), ShouldEqual, 4) So(hcMock.AddCheckCalls()[0].Name, ShouldResemble, "Zebedee") So(hcMock.AddCheckCalls()[1].Name, ShouldResemble, "Kafka Generate Downloads Producer") - So(hcMock.AddCheckCalls()[2].Name, ShouldResemble, "Mongo DB") - So(hcMock.AddCheckCalls()[3].Name, ShouldResemble, "Graph DB") + So(hcMock.AddCheckCalls()[2].Name, ShouldResemble, "Graph DB") + So(hcMock.AddCheckCalls()[3].Name, ShouldResemble, "Mongo DB") So(len(initMock.DoGetHTTPServerCalls()), ShouldEqual, 1) So(initMock.DoGetHTTPServerCalls()[0].BindAddr, ShouldEqual, ":22000") So(len(hcMock.StartCalls()), ShouldEqual, 1) @@ -282,14 +282,13 @@ func TestRun(t *testing.T) { So(err, ShouldBeNil) So(svcList.MongoDB, ShouldBeTrue) So(svcList.Graph, ShouldBeFalse) - So(svcList.GenerateDownloadsProducer, ShouldBeTrue) + So(svcList.GenerateDownloadsProducer, ShouldBeFalse) So(svcList.HealthCheck, ShouldBeTrue) }) - Convey("Only the checkers for Kafka and MongoDB are registered, and the healthcheck and http server started", func() { - So(len(hcMock.AddCheckCalls()), ShouldEqual, 2) - So(hcMock.AddCheckCalls()[0].Name, ShouldResemble, "Kafka Generate Downloads Producer") - So(hcMock.AddCheckCalls()[1].Name, ShouldResemble, "Mongo DB") + Convey("Only the checkers for MongoDB are registered, and the healthcheck and http server started", func() { + So(len(hcMock.AddCheckCalls()), ShouldEqual, 1) + So(hcMock.AddCheckCalls()[0].Name, ShouldResemble, "Mongo DB") So(len(initMock.DoGetHTTPServerCalls()), ShouldEqual, 1) So(initMock.DoGetHTTPServerCalls()[0].BindAddr, ShouldEqual, ":22000") So(len(hcMock.StartCalls()), ShouldEqual, 1) diff --git a/store/datastore.go b/store/datastore.go index 4e408188..cf0ee249 100644 --- a/store/datastore.go +++ b/store/datastore.go @@ -25,13 +25,13 @@ type dataMongoDB interface { CheckDatasetExists(ID, state string) error CheckEditionExists(ID, editionID, state string) error GetDataset(ID string) (*models.DatasetUpdate, error) - GetDatasets(ctx context.Context, offset, limit int, authorised bool) (*models.DatasetUpdateResults, error) + GetDatasets(ctx context.Context, offset, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) GetDimensionsFromInstance(ID string) (*models.DimensionNodeResults, error) GetDimensions(datasetID, versionID string) ([]bson.M, error) GetDimensionOptions(version *models.Version, dimension string, offset, limit int) (*models.DimensionOptionResults, error) GetDimensionOptionsFromIDs(version *models.Version, dimension string, ids []string) (*models.DimensionOptionResults, error) GetEdition(ID, editionID, state string) (*models.EditionUpdate, error) - GetEditions(ctx context.Context, ID, state string, offset, limit int, authorised bool) (*models.EditionUpdateResults, error) + GetEditions(ctx context.Context, ID, state string, offset, limit int, authorised bool) ([]*models.EditionUpdate, int, error) GetInstances(ctx context.Context, states []string, datasets []string, offset, limit int) (*models.InstanceResults, error) GetInstance(ID string) (*models.Instance, error) GetNextVersion(datasetID, editionID string) (int, error) @@ -40,7 +40,7 @@ type dataMongoDB interface { GetVersions(ctx context.Context, datasetID, editionID, state string, offset, limit int) (*models.VersionResults, error) UpdateDataset(ctx context.Context, ID string, dataset *models.Dataset, currentState string) error UpdateDatasetWithAssociation(ID, state string, version *models.Version) error - UpdateDimensionNodeID(dimension *models.DimensionOption) error + UpdateDimensionNodeIDAndOrder(dimension *models.DimensionOption) error UpdateInstance(ctx context.Context, ID string, instance *models.Instance) error UpdateObservationInserted(ID string, observationInserted int64) error UpdateImportObservationsTaskState(id, state string) error diff --git a/store/datastoretest/datastore.go b/store/datastoretest/datastore.go index 6f0aa9e3..26643952 100755 --- a/store/datastoretest/datastore.go +++ b/store/datastoretest/datastore.go @@ -7,7 +7,6 @@ import ( "context" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" - "github.com/ONSdigital/dp-graph/v2/observation" "github.com/globalsign/mgo/bson" "sync" ) @@ -18,127 +17,124 @@ var _ store.Storer = &StorerMock{} // StorerMock is a mock implementation of store.Storer. // -// func TestSomethingThatUsesStorer(t *testing.T) { +// func TestSomethingThatUsesStorer(t *testing.T) { // -// // make and configure a mocked store.Storer -// mockedStorer := &StorerMock{ -// AddDimensionToInstanceFunc: func(dimension *models.CachedDimensionOption) error { -// panic("mock out the AddDimensionToInstance method") -// }, -// AddEventToInstanceFunc: func(instanceID string, event *models.Event) error { -// panic("mock out the AddEventToInstance method") -// }, -// AddInstanceFunc: func(instance *models.Instance) (*models.Instance, error) { -// panic("mock out the AddInstance method") -// }, -// AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { -// panic("mock out the AddVersionDetailsToInstance method") -// }, -// CheckDatasetExistsFunc: func(ID string, state string) error { -// panic("mock out the CheckDatasetExists method") -// }, -// CheckEditionExistsFunc: func(ID string, editionID string, state string) error { -// panic("mock out the CheckEditionExists method") -// }, -// DeleteDatasetFunc: func(ID string) error { -// panic("mock out the DeleteDataset method") -// }, -// DeleteEditionFunc: func(ID string) error { -// panic("mock out the DeleteEdition method") -// }, -// GetDatasetFunc: func(ID string) (*models.DatasetUpdate, error) { -// panic("mock out the GetDataset method") -// }, -// GetDatasetsFunc: func(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) { -// panic("mock out the GetDatasets method") -// }, -// GetDimensionOptionsFunc: func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) { -// panic("mock out the GetDimensionOptions method") -// }, -// GetDimensionOptionsFromIDsFunc: func(version *models.Version, dimension string, ids []string) (*models.DimensionOptionResults, error) { -// panic("mock out the GetDimensionOptionsFromIDs method") -// }, -// GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { -// panic("mock out the GetDimensions method") -// }, -// GetDimensionsFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { -// panic("mock out the GetDimensionsFromInstance method") -// }, -// GetEditionFunc: func(ID string, editionID string, state string) (*models.EditionUpdate, error) { -// panic("mock out the GetEdition method") -// }, -// GetEditionsFunc: func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) { -// panic("mock out the GetEditions method") -// }, -// GetInstanceFunc: func(ID string) (*models.Instance, error) { -// panic("mock out the GetInstance method") -// }, -// GetInstancesFunc: func(ctx context.Context, states []string, datasets []string, offset int, limit int) (*models.InstanceResults, error) { -// panic("mock out the GetInstances method") -// }, -// GetNextVersionFunc: func(datasetID string, editionID string) (int, error) { -// panic("mock out the GetNextVersion method") -// }, -// GetUniqueDimensionAndOptionsFunc: func(ID string, dimension string) (*models.DimensionValues, error) { -// panic("mock out the GetUniqueDimensionAndOptions method") -// }, -// GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { -// panic("mock out the GetVersion method") -// }, -// GetVersionsFunc: func(ctx context.Context, datasetID string, editionID string, state string, offset int, limit int) (*models.VersionResults, error) { -// panic("mock out the GetVersions method") -// }, -// SetInstanceIsPublishedFunc: func(ctx context.Context, instanceID string) error { -// panic("mock out the SetInstanceIsPublished method") -// }, -// StreamCSVRowsFunc: func(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) { -// panic("mock out the StreamCSVRows method") -// }, -// UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { -// panic("mock out the UpdateBuildHierarchyTaskState method") -// }, -// UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { -// panic("mock out the UpdateBuildSearchTaskState method") -// }, -// UpdateDatasetFunc: func(ctx context.Context, ID string, dataset *models.Dataset, currentState string) error { -// panic("mock out the UpdateDataset method") -// }, -// UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { -// panic("mock out the UpdateDatasetWithAssociation method") -// }, -// UpdateDimensionNodeIDFunc: func(dimension *models.DimensionOption) error { -// panic("mock out the UpdateDimensionNodeID method") -// }, -// UpdateImportObservationsTaskStateFunc: func(id string, state string) error { -// panic("mock out the UpdateImportObservationsTaskState method") -// }, -// UpdateInstanceFunc: func(ctx context.Context, ID string, instance *models.Instance) error { -// panic("mock out the UpdateInstance method") -// }, -// UpdateObservationInsertedFunc: func(ID string, observationInserted int64) error { -// panic("mock out the UpdateObservationInserted method") -// }, -// UpdateVersionFunc: func(ID string, version *models.Version) error { -// panic("mock out the UpdateVersion method") -// }, -// UpsertContactFunc: func(ID string, update interface{}) error { -// panic("mock out the UpsertContact method") -// }, -// UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { -// panic("mock out the UpsertDataset method") -// }, -// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { -// panic("mock out the UpsertEdition method") -// }, -// UpsertVersionFunc: func(ID string, versionDoc *models.Version) error { -// panic("mock out the UpsertVersion method") -// }, -// } +// // make and configure a mocked store.Storer +// mockedStorer := &StorerMock{ +// AddDimensionToInstanceFunc: func(dimension *models.CachedDimensionOption) error { +// panic("mock out the AddDimensionToInstance method") +// }, +// AddEventToInstanceFunc: func(instanceID string, event *models.Event) error { +// panic("mock out the AddEventToInstance method") +// }, +// AddInstanceFunc: func(instance *models.Instance) (*models.Instance, error) { +// panic("mock out the AddInstance method") +// }, +// AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { +// panic("mock out the AddVersionDetailsToInstance method") +// }, +// CheckDatasetExistsFunc: func(ID string, state string) error { +// panic("mock out the CheckDatasetExists method") +// }, +// CheckEditionExistsFunc: func(ID string, editionID string, state string) error { +// panic("mock out the CheckEditionExists method") +// }, +// DeleteDatasetFunc: func(ID string) error { +// panic("mock out the DeleteDataset method") +// }, +// DeleteEditionFunc: func(ID string) error { +// panic("mock out the DeleteEdition method") +// }, +// GetDatasetFunc: func(ID string) (*models.DatasetUpdate, error) { +// panic("mock out the GetDataset method") +// }, +// GetDatasetsFunc: func(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { +// panic("mock out the GetDatasets method") +// }, +// GetDimensionOptionsFunc: func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) { +// panic("mock out the GetDimensionOptions method") +// }, +// GetDimensionOptionsFromIDsFunc: func(version *models.Version, dimension string, ids []string) (*models.DimensionOptionResults, error) { +// panic("mock out the GetDimensionOptionsFromIDs method") +// }, +// GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { +// panic("mock out the GetDimensions method") +// }, +// GetDimensionsFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { +// panic("mock out the GetDimensionsFromInstance method") +// }, +// GetEditionFunc: func(ID string, editionID string, state string) (*models.EditionUpdate, error) { +// panic("mock out the GetEdition method") +// }, +// GetEditionsFunc: func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { +// panic("mock out the GetEditions method") +// }, +// GetInstanceFunc: func(ID string) (*models.Instance, error) { +// panic("mock out the GetInstance method") +// }, +// GetInstancesFunc: func(ctx context.Context, states []string, datasets []string, offset int, limit int) (*models.InstanceResults, error) { +// panic("mock out the GetInstances method") +// }, +// GetNextVersionFunc: func(datasetID string, editionID string) (int, error) { +// panic("mock out the GetNextVersion method") +// }, +// GetUniqueDimensionAndOptionsFunc: func(ID string, dimension string) (*models.DimensionValues, error) { +// panic("mock out the GetUniqueDimensionAndOptions method") +// }, +// GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { +// panic("mock out the GetVersion method") +// }, +// GetVersionsFunc: func(ctx context.Context, datasetID string, editionID string, state string, offset int, limit int) (*models.VersionResults, error) { +// panic("mock out the GetVersions method") +// }, +// SetInstanceIsPublishedFunc: func(ctx context.Context, instanceID string) error { +// panic("mock out the SetInstanceIsPublished method") +// }, +// UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { +// panic("mock out the UpdateBuildHierarchyTaskState method") +// }, +// UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { +// panic("mock out the UpdateBuildSearchTaskState method") +// }, +// UpdateDatasetFunc: func(ctx context.Context, ID string, dataset *models.Dataset, currentState string) error { +// panic("mock out the UpdateDataset method") +// }, +// UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { +// panic("mock out the UpdateDatasetWithAssociation method") +// }, +// UpdateDimensionNodeIDAndOrderFunc: func(dimension *models.DimensionOption) error { +// panic("mock out the UpdateDimensionNodeIDAndOrder method") +// }, +// UpdateImportObservationsTaskStateFunc: func(id string, state string) error { +// panic("mock out the UpdateImportObservationsTaskState method") +// }, +// UpdateInstanceFunc: func(ctx context.Context, ID string, instance *models.Instance) error { +// panic("mock out the UpdateInstance method") +// }, +// UpdateObservationInsertedFunc: func(ID string, observationInserted int64) error { +// panic("mock out the UpdateObservationInserted method") +// }, +// UpdateVersionFunc: func(ID string, version *models.Version) error { +// panic("mock out the UpdateVersion method") +// }, +// UpsertContactFunc: func(ID string, update interface{}) error { +// panic("mock out the UpsertContact method") +// }, +// UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { +// panic("mock out the UpsertDataset method") +// }, +// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { +// panic("mock out the UpsertEdition method") +// }, +// UpsertVersionFunc: func(ID string, versionDoc *models.Version) error { +// panic("mock out the UpsertVersion method") +// }, +// } // -// // use mockedStorer in code that requires store.Storer -// // and then make assertions. +// // use mockedStorer in code that requires store.Storer +// // and then make assertions. // -// } +// } type StorerMock struct { // AddDimensionToInstanceFunc mocks the AddDimensionToInstance method. AddDimensionToInstanceFunc func(dimension *models.CachedDimensionOption) error @@ -168,7 +164,7 @@ type StorerMock struct { GetDatasetFunc func(ID string) (*models.DatasetUpdate, error) // GetDatasetsFunc mocks the GetDatasets method. - GetDatasetsFunc func(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) + GetDatasetsFunc func(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) // GetDimensionOptionsFunc mocks the GetDimensionOptions method. GetDimensionOptionsFunc func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) @@ -186,7 +182,7 @@ type StorerMock struct { GetEditionFunc func(ID string, editionID string, state string) (*models.EditionUpdate, error) // GetEditionsFunc mocks the GetEditions method. - GetEditionsFunc func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) + GetEditionsFunc func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) // GetInstanceFunc mocks the GetInstance method. GetInstanceFunc func(ID string) (*models.Instance, error) @@ -209,9 +205,6 @@ type StorerMock struct { // SetInstanceIsPublishedFunc mocks the SetInstanceIsPublished method. SetInstanceIsPublishedFunc func(ctx context.Context, instanceID string) error - // StreamCSVRowsFunc mocks the StreamCSVRows method. - StreamCSVRowsFunc func(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) - // UpdateBuildHierarchyTaskStateFunc mocks the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskStateFunc func(id string, dimension string, state string) error @@ -224,8 +217,8 @@ type StorerMock struct { // UpdateDatasetWithAssociationFunc mocks the UpdateDatasetWithAssociation method. UpdateDatasetWithAssociationFunc func(ID string, state string, version *models.Version) error - // UpdateDimensionNodeIDFunc mocks the UpdateDimensionNodeID method. - UpdateDimensionNodeIDFunc func(dimension *models.DimensionOption) error + // UpdateDimensionNodeIDAndOrderFunc mocks the UpdateDimensionNodeIDAndOrder method. + UpdateDimensionNodeIDAndOrderFunc func(dimension *models.DimensionOption) error // UpdateImportObservationsTaskStateFunc mocks the UpdateImportObservationsTaskState method. UpdateImportObservationsTaskStateFunc func(id string, state string) error @@ -446,19 +439,6 @@ type StorerMock struct { // InstanceID is the instanceID argument value. InstanceID string } - // StreamCSVRows holds details about calls to the StreamCSVRows method. - StreamCSVRows []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // InstanceID is the instanceID argument value. - InstanceID string - // FilterID is the filterID argument value. - FilterID string - // Filters is the filters argument value. - Filters *observation.DimensionFilters - // Limit is the limit argument value. - Limit *int - } // UpdateBuildHierarchyTaskState holds details about calls to the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskState []struct { // ID is the id argument value. @@ -497,8 +477,8 @@ type StorerMock struct { // Version is the version argument value. Version *models.Version } - // UpdateDimensionNodeID holds details about calls to the UpdateDimensionNodeID method. - UpdateDimensionNodeID []struct { + // UpdateDimensionNodeIDAndOrder holds details about calls to the UpdateDimensionNodeIDAndOrder method. + UpdateDimensionNodeIDAndOrder []struct { // Dimension is the dimension argument value. Dimension *models.DimensionOption } @@ -586,12 +566,11 @@ type StorerMock struct { lockGetVersion sync.RWMutex lockGetVersions sync.RWMutex lockSetInstanceIsPublished sync.RWMutex - lockStreamCSVRows sync.RWMutex lockUpdateBuildHierarchyTaskState sync.RWMutex lockUpdateBuildSearchTaskState sync.RWMutex lockUpdateDataset sync.RWMutex lockUpdateDatasetWithAssociation sync.RWMutex - lockUpdateDimensionNodeID sync.RWMutex + lockUpdateDimensionNodeIDAndOrder sync.RWMutex lockUpdateImportObservationsTaskState sync.RWMutex lockUpdateInstance sync.RWMutex lockUpdateObservationInserted sync.RWMutex @@ -914,7 +893,7 @@ func (mock *StorerMock) GetDatasetCalls() []struct { } // GetDatasets calls GetDatasetsFunc. -func (mock *StorerMock) GetDatasets(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) { +func (mock *StorerMock) GetDatasets(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { if mock.GetDatasetsFunc == nil { panic("StorerMock.GetDatasetsFunc: method is nil but Storer.GetDatasets was just called") } @@ -1144,7 +1123,7 @@ func (mock *StorerMock) GetEditionCalls() []struct { } // GetEditions calls GetEditionsFunc. -func (mock *StorerMock) GetEditions(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) { +func (mock *StorerMock) GetEditions(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { if mock.GetEditionsFunc == nil { panic("StorerMock.GetEditionsFunc: method is nil but Storer.GetEditions was just called") } @@ -1471,53 +1450,6 @@ func (mock *StorerMock) SetInstanceIsPublishedCalls() []struct { return calls } -// StreamCSVRows calls StreamCSVRowsFunc. -func (mock *StorerMock) StreamCSVRows(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) { - if mock.StreamCSVRowsFunc == nil { - panic("StorerMock.StreamCSVRowsFunc: method is nil but Storer.StreamCSVRows was just called") - } - callInfo := struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int - }{ - Ctx: ctx, - InstanceID: instanceID, - FilterID: filterID, - Filters: filters, - Limit: limit, - } - mock.lockStreamCSVRows.Lock() - mock.calls.StreamCSVRows = append(mock.calls.StreamCSVRows, callInfo) - mock.lockStreamCSVRows.Unlock() - return mock.StreamCSVRowsFunc(ctx, instanceID, filterID, filters, limit) -} - -// StreamCSVRowsCalls gets all the calls that were made to StreamCSVRows. -// Check the length with: -// len(mockedStorer.StreamCSVRowsCalls()) -func (mock *StorerMock) StreamCSVRowsCalls() []struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int -} { - var calls []struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int - } - mock.lockStreamCSVRows.RLock() - calls = mock.calls.StreamCSVRows - mock.lockStreamCSVRows.RUnlock() - return calls -} - // UpdateBuildHierarchyTaskState calls UpdateBuildHierarchyTaskStateFunc. func (mock *StorerMock) UpdateBuildHierarchyTaskState(id string, dimension string, state string) error { if mock.UpdateBuildHierarchyTaskStateFunc == nil { @@ -1678,34 +1610,34 @@ func (mock *StorerMock) UpdateDatasetWithAssociationCalls() []struct { return calls } -// UpdateDimensionNodeID calls UpdateDimensionNodeIDFunc. -func (mock *StorerMock) UpdateDimensionNodeID(dimension *models.DimensionOption) error { - if mock.UpdateDimensionNodeIDFunc == nil { - panic("StorerMock.UpdateDimensionNodeIDFunc: method is nil but Storer.UpdateDimensionNodeID was just called") +// UpdateDimensionNodeIDAndOrder calls UpdateDimensionNodeIDAndOrderFunc. +func (mock *StorerMock) UpdateDimensionNodeIDAndOrder(dimension *models.DimensionOption) error { + if mock.UpdateDimensionNodeIDAndOrderFunc == nil { + panic("StorerMock.UpdateDimensionNodeIDAndOrderFunc: method is nil but Storer.UpdateDimensionNodeIDAndOrder was just called") } callInfo := struct { Dimension *models.DimensionOption }{ Dimension: dimension, } - mock.lockUpdateDimensionNodeID.Lock() - mock.calls.UpdateDimensionNodeID = append(mock.calls.UpdateDimensionNodeID, callInfo) - mock.lockUpdateDimensionNodeID.Unlock() - return mock.UpdateDimensionNodeIDFunc(dimension) + mock.lockUpdateDimensionNodeIDAndOrder.Lock() + mock.calls.UpdateDimensionNodeIDAndOrder = append(mock.calls.UpdateDimensionNodeIDAndOrder, callInfo) + mock.lockUpdateDimensionNodeIDAndOrder.Unlock() + return mock.UpdateDimensionNodeIDAndOrderFunc(dimension) } -// UpdateDimensionNodeIDCalls gets all the calls that were made to UpdateDimensionNodeID. +// UpdateDimensionNodeIDAndOrderCalls gets all the calls that were made to UpdateDimensionNodeIDAndOrder. // Check the length with: -// len(mockedStorer.UpdateDimensionNodeIDCalls()) -func (mock *StorerMock) UpdateDimensionNodeIDCalls() []struct { +// len(mockedStorer.UpdateDimensionNodeIDAndOrderCalls()) +func (mock *StorerMock) UpdateDimensionNodeIDAndOrderCalls() []struct { Dimension *models.DimensionOption } { var calls []struct { Dimension *models.DimensionOption } - mock.lockUpdateDimensionNodeID.RLock() - calls = mock.calls.UpdateDimensionNodeID - mock.lockUpdateDimensionNodeID.RUnlock() + mock.lockUpdateDimensionNodeIDAndOrder.RLock() + calls = mock.calls.UpdateDimensionNodeIDAndOrder + mock.lockUpdateDimensionNodeIDAndOrder.RUnlock() return calls } diff --git a/store/datastoretest/graph.go b/store/datastoretest/graph.go index 2806f8a1..b7c6176c 100644 --- a/store/datastoretest/graph.go +++ b/store/datastoretest/graph.go @@ -6,7 +6,6 @@ package storetest import ( "context" "github.com/ONSdigital/dp-dataset-api/store" - "github.com/ONSdigital/dp-graph/v2/observation" "github.com/ONSdigital/dp-healthcheck/healthcheck" "sync" ) @@ -17,37 +16,34 @@ var _ store.GraphDB = &GraphDBMock{} // GraphDBMock is a mock implementation of store.GraphDB. // -// func TestSomethingThatUsesGraphDB(t *testing.T) { +// func TestSomethingThatUsesGraphDB(t *testing.T) { // -// // make and configure a mocked store.GraphDB -// mockedGraphDB := &GraphDBMock{ -// AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { -// panic("mock out the AddVersionDetailsToInstance method") -// }, -// CheckerFunc: func(in1 context.Context, in2 *healthcheck.CheckState) error { -// panic("mock out the Checker method") -// }, -// CloseFunc: func(ctx context.Context) error { -// panic("mock out the Close method") -// }, -// SetInstanceIsPublishedFunc: func(ctx context.Context, instanceID string) error { -// panic("mock out the SetInstanceIsPublished method") -// }, -// StreamCSVRowsFunc: func(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) { -// panic("mock out the StreamCSVRows method") -// }, -// } +// // make and configure a mocked store.GraphDB +// mockedGraphDB := &GraphDBMock{ +// AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { +// panic("mock out the AddVersionDetailsToInstance method") +// }, +// CheckerFunc: func(contextMoqParam context.Context, checkState *healthcheck.CheckState) error { +// panic("mock out the Checker method") +// }, +// CloseFunc: func(ctx context.Context) error { +// panic("mock out the Close method") +// }, +// SetInstanceIsPublishedFunc: func(ctx context.Context, instanceID string) error { +// panic("mock out the SetInstanceIsPublished method") +// }, +// } // -// // use mockedGraphDB in code that requires store.GraphDB -// // and then make assertions. +// // use mockedGraphDB in code that requires store.GraphDB +// // and then make assertions. // -// } +// } type GraphDBMock struct { // AddVersionDetailsToInstanceFunc mocks the AddVersionDetailsToInstance method. AddVersionDetailsToInstanceFunc func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error // CheckerFunc mocks the Checker method. - CheckerFunc func(in1 context.Context, in2 *healthcheck.CheckState) error + CheckerFunc func(contextMoqParam context.Context, checkState *healthcheck.CheckState) error // CloseFunc mocks the Close method. CloseFunc func(ctx context.Context) error @@ -55,9 +51,6 @@ type GraphDBMock struct { // SetInstanceIsPublishedFunc mocks the SetInstanceIsPublished method. SetInstanceIsPublishedFunc func(ctx context.Context, instanceID string) error - // StreamCSVRowsFunc mocks the StreamCSVRows method. - StreamCSVRowsFunc func(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) - // calls tracks calls to the methods. calls struct { // AddVersionDetailsToInstance holds details about calls to the AddVersionDetailsToInstance method. @@ -75,10 +68,10 @@ type GraphDBMock struct { } // Checker holds details about calls to the Checker method. Checker []struct { - // In1 is the in1 argument value. - In1 context.Context - // In2 is the in2 argument value. - In2 *healthcheck.CheckState + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context + // CheckState is the checkState argument value. + CheckState *healthcheck.CheckState } // Close holds details about calls to the Close method. Close []struct { @@ -92,25 +85,11 @@ type GraphDBMock struct { // InstanceID is the instanceID argument value. InstanceID string } - // StreamCSVRows holds details about calls to the StreamCSVRows method. - StreamCSVRows []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // InstanceID is the instanceID argument value. - InstanceID string - // FilterID is the filterID argument value. - FilterID string - // Filters is the filters argument value. - Filters *observation.DimensionFilters - // Limit is the limit argument value. - Limit *int - } } lockAddVersionDetailsToInstance sync.RWMutex lockChecker sync.RWMutex lockClose sync.RWMutex lockSetInstanceIsPublished sync.RWMutex - lockStreamCSVRows sync.RWMutex } // AddVersionDetailsToInstance calls AddVersionDetailsToInstanceFunc. @@ -161,33 +140,33 @@ func (mock *GraphDBMock) AddVersionDetailsToInstanceCalls() []struct { } // Checker calls CheckerFunc. -func (mock *GraphDBMock) Checker(in1 context.Context, in2 *healthcheck.CheckState) error { +func (mock *GraphDBMock) Checker(contextMoqParam context.Context, checkState *healthcheck.CheckState) error { if mock.CheckerFunc == nil { panic("GraphDBMock.CheckerFunc: method is nil but GraphDB.Checker was just called") } callInfo := struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState }{ - In1: in1, - In2: in2, + ContextMoqParam: contextMoqParam, + CheckState: checkState, } mock.lockChecker.Lock() mock.calls.Checker = append(mock.calls.Checker, callInfo) mock.lockChecker.Unlock() - return mock.CheckerFunc(in1, in2) + return mock.CheckerFunc(contextMoqParam, checkState) } // CheckerCalls gets all the calls that were made to Checker. // Check the length with: // len(mockedGraphDB.CheckerCalls()) func (mock *GraphDBMock) CheckerCalls() []struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState } { var calls []struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState } mock.lockChecker.RLock() calls = mock.calls.Checker @@ -260,50 +239,3 @@ func (mock *GraphDBMock) SetInstanceIsPublishedCalls() []struct { mock.lockSetInstanceIsPublished.RUnlock() return calls } - -// StreamCSVRows calls StreamCSVRowsFunc. -func (mock *GraphDBMock) StreamCSVRows(ctx context.Context, instanceID string, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) { - if mock.StreamCSVRowsFunc == nil { - panic("GraphDBMock.StreamCSVRowsFunc: method is nil but GraphDB.StreamCSVRows was just called") - } - callInfo := struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int - }{ - Ctx: ctx, - InstanceID: instanceID, - FilterID: filterID, - Filters: filters, - Limit: limit, - } - mock.lockStreamCSVRows.Lock() - mock.calls.StreamCSVRows = append(mock.calls.StreamCSVRows, callInfo) - mock.lockStreamCSVRows.Unlock() - return mock.StreamCSVRowsFunc(ctx, instanceID, filterID, filters, limit) -} - -// StreamCSVRowsCalls gets all the calls that were made to StreamCSVRows. -// Check the length with: -// len(mockedGraphDB.StreamCSVRowsCalls()) -func (mock *GraphDBMock) StreamCSVRowsCalls() []struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int -} { - var calls []struct { - Ctx context.Context - InstanceID string - FilterID string - Filters *observation.DimensionFilters - Limit *int - } - mock.lockStreamCSVRows.RLock() - calls = mock.calls.StreamCSVRows - mock.lockStreamCSVRows.RUnlock() - return calls -} diff --git a/store/datastoretest/mongo.go b/store/datastoretest/mongo.go index 5187d6c4..fd0e15e4 100644 --- a/store/datastoretest/mongo.go +++ b/store/datastoretest/mongo.go @@ -18,124 +18,124 @@ var _ store.MongoDB = &MongoDBMock{} // MongoDBMock is a mock implementation of store.MongoDB. // -// func TestSomethingThatUsesMongoDB(t *testing.T) { +// func TestSomethingThatUsesMongoDB(t *testing.T) { // -// // make and configure a mocked store.MongoDB -// mockedMongoDB := &MongoDBMock{ -// AddDimensionToInstanceFunc: func(dimension *models.CachedDimensionOption) error { -// panic("mock out the AddDimensionToInstance method") -// }, -// AddEventToInstanceFunc: func(instanceID string, event *models.Event) error { -// panic("mock out the AddEventToInstance method") -// }, -// AddInstanceFunc: func(instance *models.Instance) (*models.Instance, error) { -// panic("mock out the AddInstance method") -// }, -// CheckDatasetExistsFunc: func(ID string, state string) error { -// panic("mock out the CheckDatasetExists method") -// }, -// CheckEditionExistsFunc: func(ID string, editionID string, state string) error { -// panic("mock out the CheckEditionExists method") -// }, -// CheckerFunc: func(in1 context.Context, in2 *healthcheck.CheckState) error { -// panic("mock out the Checker method") -// }, -// CloseFunc: func(in1 context.Context) error { -// panic("mock out the Close method") -// }, -// DeleteDatasetFunc: func(ID string) error { -// panic("mock out the DeleteDataset method") -// }, -// DeleteEditionFunc: func(ID string) error { -// panic("mock out the DeleteEdition method") -// }, -// GetDatasetFunc: func(ID string) (*models.DatasetUpdate, error) { -// panic("mock out the GetDataset method") -// }, -// GetDatasetsFunc: func(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) { -// panic("mock out the GetDatasets method") -// }, -// GetDimensionOptionsFunc: func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) { -// panic("mock out the GetDimensionOptions method") -// }, -// GetDimensionOptionsFromIDsFunc: func(version *models.Version, dimension string, ids []string) (*models.DimensionOptionResults, error) { -// panic("mock out the GetDimensionOptionsFromIDs method") -// }, -// GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { -// panic("mock out the GetDimensions method") -// }, -// GetDimensionsFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { -// panic("mock out the GetDimensionsFromInstance method") -// }, -// GetEditionFunc: func(ID string, editionID string, state string) (*models.EditionUpdate, error) { -// panic("mock out the GetEdition method") -// }, -// GetEditionsFunc: func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) { -// panic("mock out the GetEditions method") -// }, -// GetInstanceFunc: func(ID string) (*models.Instance, error) { -// panic("mock out the GetInstance method") -// }, -// GetInstancesFunc: func(ctx context.Context, states []string, datasets []string, offset int, limit int) (*models.InstanceResults, error) { -// panic("mock out the GetInstances method") -// }, -// GetNextVersionFunc: func(datasetID string, editionID string) (int, error) { -// panic("mock out the GetNextVersion method") -// }, -// GetUniqueDimensionAndOptionsFunc: func(ID string, dimension string) (*models.DimensionValues, error) { -// panic("mock out the GetUniqueDimensionAndOptions method") -// }, -// GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { -// panic("mock out the GetVersion method") -// }, -// GetVersionsFunc: func(ctx context.Context, datasetID string, editionID string, state string, offset int, limit int) (*models.VersionResults, error) { -// panic("mock out the GetVersions method") -// }, -// UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { -// panic("mock out the UpdateBuildHierarchyTaskState method") -// }, -// UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { -// panic("mock out the UpdateBuildSearchTaskState method") -// }, -// UpdateDatasetFunc: func(ctx context.Context, ID string, dataset *models.Dataset, currentState string) error { -// panic("mock out the UpdateDataset method") -// }, -// UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { -// panic("mock out the UpdateDatasetWithAssociation method") -// }, -// UpdateDimensionNodeIDFunc: func(dimension *models.DimensionOption) error { -// panic("mock out the UpdateDimensionNodeID method") -// }, -// UpdateImportObservationsTaskStateFunc: func(id string, state string) error { -// panic("mock out the UpdateImportObservationsTaskState method") -// }, -// UpdateInstanceFunc: func(ctx context.Context, ID string, instance *models.Instance) error { -// panic("mock out the UpdateInstance method") -// }, -// UpdateObservationInsertedFunc: func(ID string, observationInserted int64) error { -// panic("mock out the UpdateObservationInserted method") -// }, -// UpdateVersionFunc: func(ID string, version *models.Version) error { -// panic("mock out the UpdateVersion method") -// }, -// UpsertContactFunc: func(ID string, update interface{}) error { -// panic("mock out the UpsertContact method") -// }, -// UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { -// panic("mock out the UpsertDataset method") -// }, -// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { -// panic("mock out the UpsertEdition method") -// }, -// UpsertVersionFunc: func(ID string, versionDoc *models.Version) error { -// panic("mock out the UpsertVersion method") -// }, -// } +// // make and configure a mocked store.MongoDB +// mockedMongoDB := &MongoDBMock{ +// AddDimensionToInstanceFunc: func(dimension *models.CachedDimensionOption) error { +// panic("mock out the AddDimensionToInstance method") +// }, +// AddEventToInstanceFunc: func(instanceID string, event *models.Event) error { +// panic("mock out the AddEventToInstance method") +// }, +// AddInstanceFunc: func(instance *models.Instance) (*models.Instance, error) { +// panic("mock out the AddInstance method") +// }, +// CheckDatasetExistsFunc: func(ID string, state string) error { +// panic("mock out the CheckDatasetExists method") +// }, +// CheckEditionExistsFunc: func(ID string, editionID string, state string) error { +// panic("mock out the CheckEditionExists method") +// }, +// CheckerFunc: func(contextMoqParam context.Context, checkState *healthcheck.CheckState) error { +// panic("mock out the Checker method") +// }, +// CloseFunc: func(contextMoqParam context.Context) error { +// panic("mock out the Close method") +// }, +// DeleteDatasetFunc: func(ID string) error { +// panic("mock out the DeleteDataset method") +// }, +// DeleteEditionFunc: func(ID string) error { +// panic("mock out the DeleteEdition method") +// }, +// GetDatasetFunc: func(ID string) (*models.DatasetUpdate, error) { +// panic("mock out the GetDataset method") +// }, +// GetDatasetsFunc: func(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { +// panic("mock out the GetDatasets method") +// }, +// GetDimensionOptionsFunc: func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) { +// panic("mock out the GetDimensionOptions method") +// }, +// GetDimensionOptionsFromIDsFunc: func(version *models.Version, dimension string, ids []string) (*models.DimensionOptionResults, error) { +// panic("mock out the GetDimensionOptionsFromIDs method") +// }, +// GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { +// panic("mock out the GetDimensions method") +// }, +// GetDimensionsFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { +// panic("mock out the GetDimensionsFromInstance method") +// }, +// GetEditionFunc: func(ID string, editionID string, state string) (*models.EditionUpdate, error) { +// panic("mock out the GetEdition method") +// }, +// GetEditionsFunc: func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { +// panic("mock out the GetEditions method") +// }, +// GetInstanceFunc: func(ID string) (*models.Instance, error) { +// panic("mock out the GetInstance method") +// }, +// GetInstancesFunc: func(ctx context.Context, states []string, datasets []string, offset int, limit int) (*models.InstanceResults, error) { +// panic("mock out the GetInstances method") +// }, +// GetNextVersionFunc: func(datasetID string, editionID string) (int, error) { +// panic("mock out the GetNextVersion method") +// }, +// GetUniqueDimensionAndOptionsFunc: func(ID string, dimension string) (*models.DimensionValues, error) { +// panic("mock out the GetUniqueDimensionAndOptions method") +// }, +// GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { +// panic("mock out the GetVersion method") +// }, +// GetVersionsFunc: func(ctx context.Context, datasetID string, editionID string, state string, offset int, limit int) (*models.VersionResults, error) { +// panic("mock out the GetVersions method") +// }, +// UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { +// panic("mock out the UpdateBuildHierarchyTaskState method") +// }, +// UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { +// panic("mock out the UpdateBuildSearchTaskState method") +// }, +// UpdateDatasetFunc: func(ctx context.Context, ID string, dataset *models.Dataset, currentState string) error { +// panic("mock out the UpdateDataset method") +// }, +// UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { +// panic("mock out the UpdateDatasetWithAssociation method") +// }, +// UpdateDimensionNodeIDAndOrderFunc: func(dimension *models.DimensionOption) error { +// panic("mock out the UpdateDimensionNodeIDAndOrder method") +// }, +// UpdateImportObservationsTaskStateFunc: func(id string, state string) error { +// panic("mock out the UpdateImportObservationsTaskState method") +// }, +// UpdateInstanceFunc: func(ctx context.Context, ID string, instance *models.Instance) error { +// panic("mock out the UpdateInstance method") +// }, +// UpdateObservationInsertedFunc: func(ID string, observationInserted int64) error { +// panic("mock out the UpdateObservationInserted method") +// }, +// UpdateVersionFunc: func(ID string, version *models.Version) error { +// panic("mock out the UpdateVersion method") +// }, +// UpsertContactFunc: func(ID string, update interface{}) error { +// panic("mock out the UpsertContact method") +// }, +// UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { +// panic("mock out the UpsertDataset method") +// }, +// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { +// panic("mock out the UpsertEdition method") +// }, +// UpsertVersionFunc: func(ID string, versionDoc *models.Version) error { +// panic("mock out the UpsertVersion method") +// }, +// } // -// // use mockedMongoDB in code that requires store.MongoDB -// // and then make assertions. +// // use mockedMongoDB in code that requires store.MongoDB +// // and then make assertions. // -// } +// } type MongoDBMock struct { // AddDimensionToInstanceFunc mocks the AddDimensionToInstance method. AddDimensionToInstanceFunc func(dimension *models.CachedDimensionOption) error @@ -153,10 +153,10 @@ type MongoDBMock struct { CheckEditionExistsFunc func(ID string, editionID string, state string) error // CheckerFunc mocks the Checker method. - CheckerFunc func(in1 context.Context, in2 *healthcheck.CheckState) error + CheckerFunc func(contextMoqParam context.Context, checkState *healthcheck.CheckState) error // CloseFunc mocks the Close method. - CloseFunc func(in1 context.Context) error + CloseFunc func(contextMoqParam context.Context) error // DeleteDatasetFunc mocks the DeleteDataset method. DeleteDatasetFunc func(ID string) error @@ -168,7 +168,7 @@ type MongoDBMock struct { GetDatasetFunc func(ID string) (*models.DatasetUpdate, error) // GetDatasetsFunc mocks the GetDatasets method. - GetDatasetsFunc func(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) + GetDatasetsFunc func(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) // GetDimensionOptionsFunc mocks the GetDimensionOptions method. GetDimensionOptionsFunc func(version *models.Version, dimension string, offset int, limit int) (*models.DimensionOptionResults, error) @@ -186,7 +186,7 @@ type MongoDBMock struct { GetEditionFunc func(ID string, editionID string, state string) (*models.EditionUpdate, error) // GetEditionsFunc mocks the GetEditions method. - GetEditionsFunc func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) + GetEditionsFunc func(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) // GetInstanceFunc mocks the GetInstance method. GetInstanceFunc func(ID string) (*models.Instance, error) @@ -218,8 +218,8 @@ type MongoDBMock struct { // UpdateDatasetWithAssociationFunc mocks the UpdateDatasetWithAssociation method. UpdateDatasetWithAssociationFunc func(ID string, state string, version *models.Version) error - // UpdateDimensionNodeIDFunc mocks the UpdateDimensionNodeID method. - UpdateDimensionNodeIDFunc func(dimension *models.DimensionOption) error + // UpdateDimensionNodeIDAndOrderFunc mocks the UpdateDimensionNodeIDAndOrder method. + UpdateDimensionNodeIDAndOrderFunc func(dimension *models.DimensionOption) error // UpdateImportObservationsTaskStateFunc mocks the UpdateImportObservationsTaskState method. UpdateImportObservationsTaskStateFunc func(id string, state string) error @@ -282,15 +282,15 @@ type MongoDBMock struct { } // Checker holds details about calls to the Checker method. Checker []struct { - // In1 is the in1 argument value. - In1 context.Context - // In2 is the in2 argument value. - In2 *healthcheck.CheckState + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context + // CheckState is the checkState argument value. + CheckState *healthcheck.CheckState } // Close holds details about calls to the Close method. Close []struct { - // In1 is the in1 argument value. - In1 context.Context + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context } // DeleteDataset holds details about calls to the DeleteDataset method. DeleteDataset []struct { @@ -470,8 +470,8 @@ type MongoDBMock struct { // Version is the version argument value. Version *models.Version } - // UpdateDimensionNodeID holds details about calls to the UpdateDimensionNodeID method. - UpdateDimensionNodeID []struct { + // UpdateDimensionNodeIDAndOrder holds details about calls to the UpdateDimensionNodeIDAndOrder method. + UpdateDimensionNodeIDAndOrder []struct { // Dimension is the dimension argument value. Dimension *models.DimensionOption } @@ -563,7 +563,7 @@ type MongoDBMock struct { lockUpdateBuildSearchTaskState sync.RWMutex lockUpdateDataset sync.RWMutex lockUpdateDatasetWithAssociation sync.RWMutex - lockUpdateDimensionNodeID sync.RWMutex + lockUpdateDimensionNodeIDAndOrder sync.RWMutex lockUpdateImportObservationsTaskState sync.RWMutex lockUpdateInstance sync.RWMutex lockUpdateObservationInserted sync.RWMutex @@ -746,33 +746,33 @@ func (mock *MongoDBMock) CheckEditionExistsCalls() []struct { } // Checker calls CheckerFunc. -func (mock *MongoDBMock) Checker(in1 context.Context, in2 *healthcheck.CheckState) error { +func (mock *MongoDBMock) Checker(contextMoqParam context.Context, checkState *healthcheck.CheckState) error { if mock.CheckerFunc == nil { panic("MongoDBMock.CheckerFunc: method is nil but MongoDB.Checker was just called") } callInfo := struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState }{ - In1: in1, - In2: in2, + ContextMoqParam: contextMoqParam, + CheckState: checkState, } mock.lockChecker.Lock() mock.calls.Checker = append(mock.calls.Checker, callInfo) mock.lockChecker.Unlock() - return mock.CheckerFunc(in1, in2) + return mock.CheckerFunc(contextMoqParam, checkState) } // CheckerCalls gets all the calls that were made to Checker. // Check the length with: // len(mockedMongoDB.CheckerCalls()) func (mock *MongoDBMock) CheckerCalls() []struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState } { var calls []struct { - In1 context.Context - In2 *healthcheck.CheckState + ContextMoqParam context.Context + CheckState *healthcheck.CheckState } mock.lockChecker.RLock() calls = mock.calls.Checker @@ -781,29 +781,29 @@ func (mock *MongoDBMock) CheckerCalls() []struct { } // Close calls CloseFunc. -func (mock *MongoDBMock) Close(in1 context.Context) error { +func (mock *MongoDBMock) Close(contextMoqParam context.Context) error { if mock.CloseFunc == nil { panic("MongoDBMock.CloseFunc: method is nil but MongoDB.Close was just called") } callInfo := struct { - In1 context.Context + ContextMoqParam context.Context }{ - In1: in1, + ContextMoqParam: contextMoqParam, } mock.lockClose.Lock() mock.calls.Close = append(mock.calls.Close, callInfo) mock.lockClose.Unlock() - return mock.CloseFunc(in1) + return mock.CloseFunc(contextMoqParam) } // CloseCalls gets all the calls that were made to Close. // Check the length with: // len(mockedMongoDB.CloseCalls()) func (mock *MongoDBMock) CloseCalls() []struct { - In1 context.Context + ContextMoqParam context.Context } { var calls []struct { - In1 context.Context + ContextMoqParam context.Context } mock.lockClose.RLock() calls = mock.calls.Close @@ -905,7 +905,7 @@ func (mock *MongoDBMock) GetDatasetCalls() []struct { } // GetDatasets calls GetDatasetsFunc. -func (mock *MongoDBMock) GetDatasets(ctx context.Context, offset int, limit int, authorised bool) (*models.DatasetUpdateResults, error) { +func (mock *MongoDBMock) GetDatasets(ctx context.Context, offset int, limit int, authorised bool) ([]*models.DatasetUpdate, int, error) { if mock.GetDatasetsFunc == nil { panic("MongoDBMock.GetDatasetsFunc: method is nil but MongoDB.GetDatasets was just called") } @@ -1135,7 +1135,7 @@ func (mock *MongoDBMock) GetEditionCalls() []struct { } // GetEditions calls GetEditionsFunc. -func (mock *MongoDBMock) GetEditions(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) (*models.EditionUpdateResults, error) { +func (mock *MongoDBMock) GetEditions(ctx context.Context, ID string, state string, offset int, limit int, authorised bool) ([]*models.EditionUpdate, int, error) { if mock.GetEditionsFunc == nil { panic("MongoDBMock.GetEditionsFunc: method is nil but MongoDB.GetEditions was just called") } @@ -1587,34 +1587,34 @@ func (mock *MongoDBMock) UpdateDatasetWithAssociationCalls() []struct { return calls } -// UpdateDimensionNodeID calls UpdateDimensionNodeIDFunc. -func (mock *MongoDBMock) UpdateDimensionNodeID(dimension *models.DimensionOption) error { - if mock.UpdateDimensionNodeIDFunc == nil { - panic("MongoDBMock.UpdateDimensionNodeIDFunc: method is nil but MongoDB.UpdateDimensionNodeID was just called") +// UpdateDimensionNodeIDAndOrder calls UpdateDimensionNodeIDAndOrderFunc. +func (mock *MongoDBMock) UpdateDimensionNodeIDAndOrder(dimension *models.DimensionOption) error { + if mock.UpdateDimensionNodeIDAndOrderFunc == nil { + panic("MongoDBMock.UpdateDimensionNodeIDAndOrderFunc: method is nil but MongoDB.UpdateDimensionNodeIDAndOrder was just called") } callInfo := struct { Dimension *models.DimensionOption }{ Dimension: dimension, } - mock.lockUpdateDimensionNodeID.Lock() - mock.calls.UpdateDimensionNodeID = append(mock.calls.UpdateDimensionNodeID, callInfo) - mock.lockUpdateDimensionNodeID.Unlock() - return mock.UpdateDimensionNodeIDFunc(dimension) + mock.lockUpdateDimensionNodeIDAndOrder.Lock() + mock.calls.UpdateDimensionNodeIDAndOrder = append(mock.calls.UpdateDimensionNodeIDAndOrder, callInfo) + mock.lockUpdateDimensionNodeIDAndOrder.Unlock() + return mock.UpdateDimensionNodeIDAndOrderFunc(dimension) } -// UpdateDimensionNodeIDCalls gets all the calls that were made to UpdateDimensionNodeID. +// UpdateDimensionNodeIDAndOrderCalls gets all the calls that were made to UpdateDimensionNodeIDAndOrder. // Check the length with: -// len(mockedMongoDB.UpdateDimensionNodeIDCalls()) -func (mock *MongoDBMock) UpdateDimensionNodeIDCalls() []struct { +// len(mockedMongoDB.UpdateDimensionNodeIDAndOrderCalls()) +func (mock *MongoDBMock) UpdateDimensionNodeIDAndOrderCalls() []struct { Dimension *models.DimensionOption } { var calls []struct { Dimension *models.DimensionOption } - mock.lockUpdateDimensionNodeID.RLock() - calls = mock.calls.UpdateDimensionNodeID - mock.lockUpdateDimensionNodeID.RUnlock() + mock.lockUpdateDimensionNodeIDAndOrder.RLock() + calls = mock.calls.UpdateDimensionNodeIDAndOrder + mock.lockUpdateDimensionNodeIDAndOrder.RUnlock() return calls } diff --git a/swagger.yaml b/swagger.yaml index 66a3489c..7b29bc34 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -34,6 +34,13 @@ parameters: in: query required: true type: string + patch_options: + required: true + name: patch + schema: + $ref: '#/definitions/PatchOptions' + description: "A list of patch operations for a dimension option" + in: body edition: name: edition description: "An edition of a dataset" @@ -834,23 +841,26 @@ paths: 500: $ref: '#/responses/InternalError' /instances/{instance_id}/dimensions/{dimension}/options/{option}: - put: + patch: tags: - "Private" - summary: "Add an option to an instance dimension" + summary: "Modify a dimension option for an instance" description: | - Add a option to a dimension which has been found in the V4 data + Modify a dimension option for an instance by setting values for node_id or order parameters: - $ref: '#/parameters/instance_id' - $ref: '#/parameters/dimension' - $ref: '#/parameters/option' + - $ref: '#/parameters/patch_options' produces: - - "application/json" + - "application/json-patch+json" security: - InternalAPIKey: [] responses: 200: - description: "The dimension was added" + description: "The dimension option was modified and the successfully applied patch operations are returned" + schema: + $ref: '#/definitions/PatchOptions' 400: $ref: '#/responses/InvalidRequestError' 401: @@ -863,6 +873,7 @@ paths: $ref: '#/responses/InternalError' /instances/{instance_id}/dimensions/{dimension}/options/{option}/node_id/{node_id}: put: + deprecated: true tags: - "Private" summary: "Update a dimension with the node_id" @@ -1235,6 +1246,27 @@ definitions: option: description: "An option for a dimension" type: string + PatchOptions: + description: "A list of operations to patch a dimension option. Can only handle adding values for /node_id and /order. Each element in the array is processed in sequential order." + type: array + items: + type: object + description: "Item containing all necessary information to make a single operation on the resource." + properties: + op: + description: | + The operation to be made on path. + * add - Sets the value for the provided path + type: string + enum: [add] + path: + description: "Path to value that needs to be operated on." + type: string + example: "/node_id" + enum: [/node_id, /order] + value: + description: "A value that will be set for the provided path. /node_id accepts string values, and /order accepts integer values." + example: "node_123" DownloadObject: description: "Object containing information of a downloadable file" type: object @@ -1904,6 +1936,9 @@ definitions: option: description: "The option of the dimension" type: string + order: + description: "The numerical order for the dimension option" + type: integer UpdateDownloadObject: description: "Object containing information of a downloadable file" type: object @@ -2039,6 +2074,9 @@ definitions: id: description: "The identifier for this version of an edition for a dataset" type: string + dataset_id: + description: "The identifier for the dataset." + type: string latest_changes: description: "A list of changes between version of an edition for a dataset and the previous version of the same dataset edition" type: array