Skip to content

Commit

Permalink
Merge branch 'develop' into release/1.28.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@andre-urbani
andre-urbani committed Mar 30, 2021
2 parents 6ba9a5f + c0b162a commit 1b69f15
Show file tree
Hide file tree
Showing 43 changed files with 2,755 additions and 1,245 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ debug
build
.idea
.vscode
def.sdmx.json
def.sdmx.json
vendor/
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ test:

.PHONY: test-component
test-component:
go test -race -cover -component
go test -race -cover -coverpkg=github.com/ONSdigital/dp-dataset-api/... -component


.PHONY: nomis
Expand Down
48 changes: 32 additions & 16 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/ONSdigital/dp-dataset-api/config"
"github.com/ONSdigital/dp-dataset-api/dimension"
"github.com/ONSdigital/dp-dataset-api/instance"
"github.com/ONSdigital/dp-dataset-api/pagination"
"github.com/ONSdigital/dp-dataset-api/store"
"github.com/ONSdigital/dp-dataset-api/url"
dphandlers "github.com/ONSdigital/dp-net/handlers"
Expand Down Expand Up @@ -91,6 +92,8 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d
maxLimit: cfg.DefaultMaxLimit,
}

paginator := pagination.NewPaginator(cfg.DefaultLimit, cfg.DefaultOffset, cfg.DefaultMaxLimit)

if api.enablePrivateEndpoints {
log.Event(ctx, "enabling private endpoints for dataset api", log.INFO)

Expand All @@ -114,21 +117,21 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d
Storer: api.dataStore.Backend,
}

api.enablePrivateDatasetEndpoints(ctx)
api.enablePrivateDatasetEndpoints(ctx, paginator)
api.enablePrivateInstancesEndpoints(instanceAPI)
api.enablePrivateDimensionsEndpoints(dimensionAPI)
} else {
log.Event(ctx, "enabling only public endpoints for dataset api", log.INFO)
api.enablePublicEndpoints(ctx)
api.enablePublicEndpoints(ctx, paginator)
}
return api
}

// enablePublicEndpoints register only the public GET endpoints.
func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) {
api.get("/datasets", api.getDatasets)
func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context, paginator *pagination.Paginator) {
api.get("/datasets", paginator.Paginate(api.getDatasets))
api.get("/datasets/{dataset_id}", api.getDataset)
api.get("/datasets/{dataset_id}/editions", api.getEditions)
api.get("/datasets/{dataset_id}/editions", paginator.Paginate(api.getEditions))
api.get("/datasets/{dataset_id}/editions/{edition}", api.getEdition)
api.get("/datasets/{dataset_id}/editions/{edition}/versions", api.getVersions)
api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}", api.getVersion)
Expand All @@ -140,10 +143,10 @@ func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) {

// enablePrivateDatasetEndpoints register the datasets endpoints with the appropriate authentication and authorisation
// checks required when running the dataset API in publishing (private) mode.
func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) {
func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context, paginator *pagination.Paginator) {
api.get(
"/datasets",
api.isAuthorised(readPermission, api.getDatasets),
api.isAuthorised(readPermission, paginator.Paginate(api.getDatasets)),
)

api.get(
Expand All @@ -154,7 +157,7 @@ func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) {

api.get(
"/datasets/{dataset_id}/editions",
api.isAuthorisedForDatasets(readPermission, api.getEditions),
api.isAuthorisedForDatasets(readPermission, paginator.Paginate(api.getEditions)),
)

api.get(
Expand Down Expand Up @@ -316,6 +319,14 @@ func (api *DatasetAPI) enablePrivateDimensionsEndpoints(dimensionAPI *dimension.
dimensionAPI.GetUniqueDimensionAndOptionsHandler)),
)

api.patch(
"/instances/{instance_id}/dimensions/{dimension}/options/{option}",
api.isAuthenticated(
api.isAuthorised(updatePermission,
api.isInstancePublished(dimensionAPI.PatchOptionHandler))),
)

// Deprecated
api.put(
"/instances/{instance_id}/dimensions/{dimension}/options/{option}/node_id/{node_id}",
api.isAuthenticated(
Expand Down Expand Up @@ -358,24 +369,29 @@ func (api *DatasetAPI) isVersionPublished(action string, handler http.HandlerFun
return api.versionPublishedChecker.Check(handler, action)
}

// get register a GET http.HandlerFunc.
// get registers a GET http.HandlerFunc.
func (api *DatasetAPI) get(path string, handler http.HandlerFunc) {
api.Router.HandleFunc(path, handler).Methods("GET")
api.Router.HandleFunc(path, handler).Methods(http.MethodGet)
}

// get register a PUT http.HandlerFunc.
// put registers a PUT http.HandlerFunc.
func (api *DatasetAPI) put(path string, handler http.HandlerFunc) {
api.Router.HandleFunc(path, handler).Methods("PUT")
api.Router.HandleFunc(path, handler).Methods(http.MethodPut)
}

// patch registers a PATCH http.HandlerFunc
func (api *DatasetAPI) patch(path string, handler http.HandlerFunc) {
api.Router.HandleFunc(path, handler).Methods(http.MethodPatch)
}

// get register a POST http.HandlerFunc.
// post registers a POST http.HandlerFunc.
func (api *DatasetAPI) post(path string, handler http.HandlerFunc) {
api.Router.HandleFunc(path, handler).Methods("POST")
api.Router.HandleFunc(path, handler).Methods(http.MethodPost)
}

// get register a DELETE http.HandlerFunc.
// delete registers a DELETE http.HandlerFunc.
func (api *DatasetAPI) delete(path string, handler http.HandlerFunc) {
api.Router.HandleFunc(path, handler).Methods("DELETE")
api.Router.HandleFunc(path, handler).Methods(http.MethodDelete)
}

func (api *DatasetAPI) authenticate(r *http.Request, logData log.Data) bool {
Expand Down
128 changes: 29 additions & 99 deletions api/dataset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import (
"encoding/json"
"fmt"
"net/http"
"strings"
"time"

errs "github.com/ONSdigital/dp-dataset-api/apierrors"
"github.com/ONSdigital/dp-dataset-api/models"
"github.com/ONSdigital/dp-dataset-api/utils"
dphttp "github.com/ONSdigital/dp-net/http"
"github.com/ONSdigital/log.go/log"
"github.com/gorilla/mux"
Expand Down Expand Up @@ -42,96 +42,20 @@ var (
}
)

func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request) {
func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request, limit int, offset int) (interface{}, int, error) {
ctx := r.Context()
logData := log.Data{}
offsetParameter := r.URL.Query().Get("offset")
limitParameter := r.URL.Query().Get("limit")

offset := api.defaultOffset
limit := api.defaultLimit

var err error

if offsetParameter != "" {
logData["offset"] = offsetParameter
offset, err = utils.ValidatePositiveInt(offsetParameter)
if err != nil {
log.Event(ctx, "invalid query parameter: offset", log.ERROR, log.Error(err), logData)
handleDatasetAPIErr(ctx, err, w, nil)
return
}
}

if limitParameter != "" {
logData["limit"] = limitParameter
limit, err = utils.ValidatePositiveInt(limitParameter)
if err != nil {
log.Event(ctx, "invalid query parameter: limit", log.ERROR, log.Error(err), logData)
handleDatasetAPIErr(ctx, err, w, nil)
return
}
}

if limit > api.maxLimit {
logData["max_limit"] = api.maxLimit
err = errs.ErrInvalidQueryParameter
log.Event(ctx, "limit is greater than the maximum allowed", log.ERROR, logData)
handleDatasetAPIErr(ctx, err, w, nil)
return
}

b, err := func() ([]byte, error) {

logData := log.Data{}

authorised := api.authenticate(r, logData)

datasets, err := api.dataStore.Backend.GetDatasets(ctx, offset, limit, authorised)
if err != nil {
log.Event(ctx, "api endpoint getDatasets datastore.GetDatasets returned an error", log.ERROR, log.Error(err))
return nil, err
}

var b []byte

var datasetsResponse interface{}

if authorised {
datasetsResponse = datasets
} else {
datasetsResponse = &models.DatasetResults{
Items: mapResults(datasets.Items),
Offset: offset,
Limit: limit,
Count: datasets.Count,
TotalCount: datasets.TotalCount,
}

}

b, err = json.Marshal(datasetsResponse)

if err != nil {
log.Event(ctx, "api endpoint getDatasets failed to marshal dataset resource into bytes", log.ERROR, log.Error(err), logData)
return nil, err
}

return b, nil
}()

authorised := api.authenticate(r, logData)
datasets, totalCount, err := api.dataStore.Backend.GetDatasets(ctx, offset, limit, authorised)
if err != nil {
handleDatasetAPIErr(ctx, err, w, nil)
return
log.Event(ctx, "api endpoint getDatasets datastore.GetDatasets returned an error", log.ERROR, log.Error(err))
handleDatasetAPIErr(ctx, err, w, logData)
return nil, 0, err
}

setJSONContentType(w)
if _, err = w.Write(b); err != nil {
log.Event(ctx, "api endpoint getDatasets error writing response body", log.ERROR, log.Error(err))
handleDatasetAPIErr(ctx, err, w, nil)
return
if authorised {
return datasets, totalCount, nil
}
log.Event(ctx, "api endpoint getDatasets request successful", log.INFO)
return mapResults(datasets), totalCount, nil
}

func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -236,6 +160,13 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) {
return nil, err
}

models.CleanDataset(dataset)

if err = models.ValidateDataset(dataset); err != nil {
log.Event(ctx, "addDataset endpoint: dataset failed validation checks", log.ERROR, log.Error(err))
return nil, err
}

dataset.Type = datasetType
dataset.State = models.CreatedState
dataset.ID = datasetID
Expand Down Expand Up @@ -321,20 +252,19 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) {
return err
}

models.CleanDataset(dataset)

if err = models.ValidateDataset(dataset); err != nil {
log.Event(ctx, "putDataset endpoint: failed validation check to update dataset", log.ERROR, log.Error(err), data)
return err
}

if dataset.State == models.PublishedState {
if err := api.publishDataset(ctx, currentDataset, nil); err != nil {
log.Event(ctx, "putDataset endpoint: failed to update dataset document to published", log.ERROR, log.Error(err), data)
return err
}
} else {
if err := models.CleanDataset(dataset); err != nil {
log.Event(ctx, "could not clean dataset", log.ERROR, log.Error(err))
return nil
}
if err := models.ValidateDataset(dataset); err != nil {
log.Event(ctx, "failed validation check to update dataset", log.ERROR, log.Error(err))
return nil
}
if err := api.dataStore.Backend.UpdateDataset(ctx, datasetID, dataset, currentDataset.Next.State); err != nil {
log.Event(ctx, "putDataset endpoint: failed to update dataset resource", log.ERROR, log.Error(err), data)
return err
Expand Down Expand Up @@ -410,15 +340,15 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) {
}

// Find any editions associated with this dataset
editionDocs, err := api.dataStore.Backend.GetEditions(ctx, currentDataset.ID, "", 0, 0, true)
editionDocs, _, err := api.dataStore.Backend.GetEditions(ctx, currentDataset.ID, "", 0, 0, true)
if err != nil {
log.Event(ctx, "unable to find the dataset editions", log.ERROR, log.Error(errs.ErrEditionsNotFound), logData)
return errs.ErrEditionsNotFound
}

// Then delete them
for i := range editionDocs.Items {
if err := api.dataStore.Backend.DeleteEdition(editionDocs.Items[i].ID); err != nil {
for i := range editionDocs {
if err := api.dataStore.Backend.DeleteEdition(editionDocs[i].ID); err != nil {
log.Event(ctx, "failed to delete edition", log.ERROR, log.Error(err), logData)
return err
}
Expand Down Expand Up @@ -455,7 +385,7 @@ func slice(full []string, offset, limit int) (sliced []string) {
return full[offset:end]
}

func mapResults(results []models.DatasetUpdate) []*models.Dataset {
func mapResults(results []*models.DatasetUpdate) []*models.Dataset {
items := []*models.Dataset{}
for _, item := range results {
if item.Current == nil {
Expand All @@ -478,7 +408,7 @@ func handleDatasetAPIErr(ctx context.Context, err error, w http.ResponseWriter,
status = http.StatusForbidden
case datasetsNoContent[err]:
status = http.StatusNoContent
case datasetsBadRequest[err]:
case datasetsBadRequest[err], strings.HasPrefix(err.Error(), "invalid fields:"):
status = http.StatusBadRequest
case resourcesNotFound[err]:
status = http.StatusNotFound
Expand Down
Loading

0 comments on commit 1b69f15

Please sign in to comment.