[API] list datasets

.github/integration/sda/rbac.json

@@ -5,6 +5,11 @@
           "path": "/c4gh-keys/*",
           "action": "(GET)|(POST)|(PUT)"
        },
+       {
+         "role": "submission",
+         "path": "/datasets/*",
+         "action": "GET"
+      },
        {
           "role": "submission",
           "path": "/file/ingest",
@@ -25,6 +30,11 @@
           "path": "/users/:username/files",
           "action": "GET"
        },
+       {
+          "role": "*",
+          "path": "/datasets",
+          "action": "GET"
+       },
        {
           "role": "*",
           "path": "/files",

.github/integration/tests/sda/40_mapper_test.sh

@@ -197,4 +197,14 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select count(id) from sda
     sleep 2
 done
 
+
+## Use API to list the datasets
+token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
+resp="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/datasets/list" | jq '. | length')"
+if [ "$resp" -ne 2 ]; then
+	echo "Error when listing key hash, expected 2 entries got: $resp"
+	exit 1
+fi
+
+
 echo "mapping test completed successfully"

sda/cmd/api/api.go

@@ -95,6 +95,7 @@ func setup(config *config.Config) *http.Server {
 	r := gin.Default()
 	r.GET("/ready", readinessResponse)
 	r.GET("/files", rbac(e), getFiles)
+	r.GET("/datasets", rbac(e), listDatasets)
 	// admin endpoints below here
 	r.POST("/c4gh-keys/add", rbac(e), addC4ghHash)                      // Adds a key hash to the database
 	r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes)                   // Lists key hashes in the database
@@ -104,6 +105,8 @@ func setup(config *config.Config) *http.Server {
 	r.POST("/file/accession", rbac(e), setAccession)             // assign accession ID to a file
 	r.POST("/dataset/create", rbac(e), createDataset)            // maps a set of files to a dataset
 	r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible
+	r.GET("/datasets/list", rbac(e), listAllDatasets)            // Lists all datasets with their status
+	r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user
 	r.GET("/users", rbac(e), listActiveUsers)                    // Lists all users
 	r.GET("/users/:username/files", rbac(e), listUserFiles)      // Lists all unmapped files for a user
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
@@ -599,3 +602,43 @@ func deprecateC4ghHash(c *gin.Context) {
 		return
 	}
 }
+
+func listAllDatasets(c *gin.Context) {
+	datasets, err := Conf.API.DB.ListDatasets()
+	if err != nil {
+		log.Errorf("ListAllDatasets failed, reason: %s", err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	c.JSON(http.StatusOK, datasets)
+}
+
+func listUserDatasets(c *gin.Context) {
+	username := strings.TrimPrefix(c.Param("username"), "/")
+	datasets, err := Conf.API.DB.ListUserDatasets(username)
+	if err != nil {
+		log.Errorf("ListUserDatasets failed, reason: %s", err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	c.JSON(http.StatusOK, datasets)
+}
+
+func listDatasets(c *gin.Context) {
+	token, err := auth.Authenticate(c.Request)
+	if err != nil {
+		c.JSON(401, err.Error())
+
+		return
+	}
+	datasets, err := Conf.API.DB.ListUserDatasets(token.Subject())
+	if err != nil {
+		log.Errorf("ListDatasets failed, reason: %s", err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	c.JSON(http.StatusOK, datasets)
+}

sda/cmd/api/api.md

@@ -21,6 +21,23 @@ Endpoints:
 
   If the `token` is invalid, 401 is returned.
 
+- `/datasets`
+  - accepts `GET` requests
+  - Returns all datasets, along with their status and last modified timestamp, for which the user has submitted data.
+
+  - Error codes
+    - `200` Query execute ok.
+    - `400` Error due to bad payload.
+    - `401` Token user is not in the list of admins.
+    - `500` Internal error due to DB failures.
+
+    Example:
+
+    ```bash
+    $curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/datasets
+    [{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"}]
+    ```
+
 ### Admin endpoints
 
 Admin endpoints are only available to a set of whitelisted users specified in the application config.
@@ -89,8 +106,42 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     curl -H "Authorization: Bearer $token" -X POST  https://HOSTNAME/dataset/release/my-dataset-01
     ```
 
+- `/datasets/list`
+  - accepts `GET` requests
+  - Returns all datasets together with their status and last modified timestamp.
+
+  - Error codes
+    - `200` Query execute ok.
+    - `400` Error due to bad payload.
+    - `401` Token user is not in the list of admins.
+    - `500` Internal error due to DB failures.
+
+    Example:
+
+    ```bash
+    $curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/datasets/list
+    [{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"},{"DatasetID":"SYNC-001-12345","Status":"registered","Timestamp":"2024-11-05T11:31:16.965226Z"}]
+    ```
+
+- `/datasets/list/:username`
+  - accepts `GET` requests with the username name as last part of the path`
+  - Returns all datasets, along with their status and last modified timestamp,for which the user has submitted data.
+
+  - Error codes
+    - `200` Query execute ok.
+    - `400` Error due to bad payload.
+    - `401` Token user is not in the list of admins.
+    - `500` Internal error due to DB failures.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/datasets/list/submission-user
+    [{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"}]
+    ```
+
 - `/users`
-  - accepts `GET` requests`
+  - accepts `GET` requests
   - Returns all users with active uploads as a JSON array
 
     Example:
@@ -105,7 +156,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     - `500` Internal error due to DB failure.
 
 - `/users/:username/files`
-  - accepts `GET` requests`
+  - accepts `GET` requests
   - Returns all files (that are not part of a dataset) for a user with active uploads as a JSON array
 
     Example:

sda/cmd/api/api_test.go

@@ -1769,3 +1769,175 @@ func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() {
 	assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode)
 	defer resp.Body.Close()
 }
+
+func (suite *TestSuite) TestListDatasets() {
+	for i := 0; i < 5; i++ {
+		fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy")
+		if err != nil {
+			suite.FailNow("failed to register file in database")
+		}
+
+		stableID := fmt.Sprintf("accession_%s_0%d", "dummy", i)
+		err = Conf.API.DB.SetAccessionID(stableID, fileID)
+		if err != nil {
+			suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_dummy_00", "accession_dummy_01", "accession_dummy_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_dummy_03", "accession_dummy_04"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/datasets/list", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/datasets/list", listAllDatasets)
+	router.GET("/dataset/list", listAllDatasets)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	datasets := []database.DatasetInfo{}
+	err = json.NewDecoder(okResponse.Body).Decode(&datasets)
+	assert.NoError(suite.T(), err, "failed to list datasets from DB")
+	assert.Equal(suite.T(), 2, len(datasets))
+	assert.Equal(suite.T(), "released", datasets[1].Status)
+	assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
+}
+
+func (suite *TestSuite) TestListUserDatasets() {
+	for i := 0; i < 5; i++ {
+		fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@"))
+		if err != nil {
+			suite.FailNow("failed to register file in database")
+		}
+
+		stableID := fmt.Sprintf("accession_%s_0%d", "user_example.org", i)
+		err = Conf.API.DB.SetAccessionID(stableID, fileID)
+		if err != nil {
+			suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/datasets/list/[email protected]", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/datasets/list/:username", listUserDatasets)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	datasets := []database.DatasetInfo{}
+	err = json.NewDecoder(okResponse.Body).Decode(&datasets)
+	assert.NoError(suite.T(), err, "failed to list datasets from DB")
+	assert.Equal(suite.T(), 2, len(datasets))
+	assert.Equal(suite.T(), "released", datasets[1].Status)
+	assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
+}
+
+func (suite *TestSuite) TestListDatasetsAsUser() {
+	for i := 0; i < 5; i++ {
+		fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), suite.User)
+		if err != nil {
+			suite.FailNow("failed to register file in database")
+		}
+
+		stableID := fmt.Sprintf("accession_user_example.org_0%d", i)
+		err = Conf.API.DB.SetAccessionID(stableID, fileID)
+		if err != nil {
+			suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+	if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
+		suite.FailNow("failed to update dataset event")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/datasets", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/datasets", listDatasets)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	datasets := []database.DatasetInfo{}
+	err = json.NewDecoder(okResponse.Body).Decode(&datasets)
+	assert.NoError(suite.T(), err, "failed to list datasets from DB")
+	assert.Equal(suite.T(), 2, len(datasets))
+	assert.Equal(suite.T(), "released", datasets[1].Status)
+	assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
+}

sda/internal/database/database.go

@@ -54,6 +54,12 @@ type SubmissionFileInfo struct {
 	CreateAt  string `json:"createAt"`
 }
 
+type DatasetInfo struct {
+	DatasetID string `json:"datasetID"`
+	Status    string `json:"status"`
+	Timestamp string `json:"timeStamp"`
+}
+
 // SchemaName is the name of the remote database schema to query
 var SchemaName = "sda"