Bug list size overload

.github/workflows/e2e-test.yml

@@ -30,11 +30,6 @@ jobs:
           echo AWS_S3_BUCKET='test-bucket' >> .env
           echo S3API_SERVICE_PORT='5005' >> .env
           echo AUTH_LEVEL=0 >> .env
-          echo POSTGRES_CONN_STRING='postgres://user:password@postgres:5432/db?sslmode=disable' >> .env
-          echo POSTGRES_PASSWORD='password' >> .env
-          echo POSTGRES_USER='user' >> .env
-          echo POSTGRES_DB='db' >> .env
-          echo PG_LOG_CHECKPOINTS='off' >> .env
 
       - name: Substitute secret variables in JSON
         env:

blobstore/blobhandler.go

@@ -21,6 +21,7 @@ type S3Controller struct {
 	Sess    *session.Session
 	S3Svc   *s3.S3
 	Buckets []string
+	S3Mock  bool
 }
 
 // Config holds the configuration settings for the REST API server.
@@ -92,13 +93,12 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 		}
 
 		// Configure the BlobHandler with MinIO session and bucket information
-		config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}}}
+		config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}, S3Mock: true}}
 		// Return the configured BlobHandler
 		return &config, nil
 	}
 
 	// Using AWS S3
-
 	// Load AWS credentials from the provided .env.json file
 	log.Debug("looking for .env.json")
 	awsConfig, err := newAWSConfig(envJson)
@@ -127,7 +127,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 			return nil, errMsg
 		}
 
-		S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC}
+		S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC, S3Mock: false}
 		// Retrieve the list of buckets for each account
 		result, err := S3Ctrl.ListBuckets()
 		if err != nil {
@@ -153,7 +153,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 		}
 
 		if len(bucketNames) > 0 {
-			config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames})
+			config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames, S3Mock: false})
 		}
 	}
 

blobstore/buckets.go

@@ -112,9 +112,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error {
 		}
 	}
 	bh.Mu.Unlock()
-
-	log.Info("HandleListBuckets: Successfully retrieved list of buckets")
-
+	log.Info("Successfully retrieved list of buckets")
 	return c.JSON(http.StatusOK, allBuckets)
 }
 

blobstore/delete.go

@@ -1,7 +1,6 @@
 package blobstore
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"strings"
@@ -12,39 +11,45 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error {
-	prefixPath := strings.Trim(prefix, "/") + "/"
-	query := &s3.ListObjectsV2Input{
-		Bucket: aws.String(bucket),
-		Prefix: aws.String(prefixPath),
+func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket string) error {
+	if len(page.Contents) == 0 {
+		return nil // No objects to delete in this page
 	}
-	resp, err := s3Ctrl.S3Svc.ListObjectsV2(query)
+
+	var objectsToDelete []*s3.ObjectIdentifier
+	for _, obj := range page.Contents {
+		objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{Key: obj.Key})
+	}
+
+	// Perform the delete operation for the current page
+	_, err := s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{
+		Bucket: aws.String(bucket),
+		Delete: &s3.Delete{
+			Objects: objectsToDelete,
+			Quiet:   aws.Bool(true),
+		},
+	})
 	if err != nil {
-		return fmt.Errorf("recursivelyDeleteObjects: error listing objects: %s", err)
+		return fmt.Errorf("error deleting objects: %v", err)
 	}
-	if len(resp.Contents) > 0 {
-		var objectsToDelete []*s3.ObjectIdentifier
 
-		for _, obj := range resp.Contents {
-			objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{
-				Key: obj.Key,
-			})
-		}
+	return nil
+}
 
-		if len(objectsToDelete) > 0 {
-			_, err = s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{
-				Bucket: aws.String(bucket),
-				Delete: &s3.Delete{
-					Objects: objectsToDelete,
-				},
-			})
-
-			if err != nil {
-				return fmt.Errorf("recursivelyDeleteObjects: error Deleting objects %v: %s", objectsToDelete, err)
-			}
+func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error {
+	var objectsFound bool
+	err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, func(page *s3.ListObjectsV2Output) error {
+		if len(page.Contents) > 0 {
+			objectsFound = true
 		}
-	} else {
-		return fmt.Errorf("recursivelyDeleteObjects: object %s not found and no objects were deleted", prefixPath)
+		return s3Ctrl.DeleteList(page, bucket)
+	})
+	if err != nil {
+		return fmt.Errorf("error processing objects for deletion: %v", err)
+	}
+
+	if !objectsFound {
+		return fmt.Errorf("prefix not found")
 	}
 	return nil
 }
@@ -56,28 +61,29 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error {
 	bucket := c.QueryParam("bucket")
 	s3Ctrl, err := bh.GetController(bucket)
 	if err != nil {
-		errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
+		errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error())
 		log.Error(errMsg.Error())
 		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
 
 	key := c.QueryParam("key")
 	if key == "" {
-		err := errors.New("parameter 'key' is required")
-		log.Errorf("HandleDeleteObjects: %s", err.Error())
-		return c.JSON(http.StatusUnprocessableEntity, err.Error())
+		errMsg := fmt.Errorf("parameter `key` is required")
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
 
 	// If the key is not a folder, proceed with deleting a single object
 	keyExist, err := s3Ctrl.KeyExists(bucket, key)
 	if err != nil {
-		log.Errorf("HandleDeleteObjects: Error checking if key exists: %s", err.Error())
-		return c.JSON(http.StatusInternalServerError, err)
+		errMsg := fmt.Errorf("error checking if key exists: %s", err.Error())
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusInternalServerError, errMsg.Error())
 	}
 	if !keyExist {
-		err := fmt.Errorf("object %s not found", key)
-		log.Errorf("HandleDeleteObjects: %s", err.Error())
-		return c.JSON(http.StatusNotFound, err.Error())
+		errMsg := fmt.Errorf("object %s not found", key)
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusNotFound, errMsg.Error())
 	}
 
 	deleteInput := &s3.DeleteObjectInput{
@@ -87,54 +93,45 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error {
 
 	_, err = s3Ctrl.S3Svc.DeleteObject(deleteInput)
 	if err != nil {
-		msg := fmt.Sprintf("error deleting object. %s", err.Error())
-		log.Errorf("HandleDeleteObjects: %s", err.Error())
-		return c.JSON(http.StatusInternalServerError, msg)
+		errMsg := fmt.Errorf("error deleting object. %s", err.Error())
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusInternalServerError, errMsg.Error())
 	}
 
-	log.Info("HandleDeleteObjects: Successfully deleted file with key:", key)
+	log.Infof("successfully deleted file with key: %s", key)
 	return c.JSON(http.StatusOK, fmt.Sprintf("Successfully deleted object: %s", key))
 }
 
 func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error {
 	bucket := c.QueryParam("bucket")
 	s3Ctrl, err := bh.GetController(bucket)
 	if err != nil {
-		errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
+		errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error())
 		log.Error(errMsg.Error())
 		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
-
 	prefix := c.QueryParam("prefix")
 	if prefix == "" {
-		err = errors.New("parameter 'prefix' is required")
-		log.Errorf("HandleDeleteObjects: %s", err.Error())
-		return c.JSON(http.StatusUnprocessableEntity, err.Error())
+		errMsg := fmt.Errorf("parameter `prefix` is required")
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
 	if !strings.HasSuffix(prefix, "/") {
 		prefix = prefix + "/"
 	}
-	response, err := s3Ctrl.GetList(bucket, prefix, false)
-	if err != nil {
-		log.Errorf("HandleDeleteObjects:  Error getting list: %s", err.Error())
-		return c.JSON(http.StatusInternalServerError, err)
-	}
-	if *response.KeyCount == 0 {
-		err := fmt.Errorf("the specified prefix %s does not exist in S3", prefix)
-		log.Errorf("HandleDeleteObjects: %s", err.Error())
-		return c.JSON(http.StatusNotFound, err.Error())
-	}
-	// This will recursively delete all objects with the specified prefix
 	err = s3Ctrl.RecursivelyDeleteObjects(bucket, prefix)
 	if err != nil {
-		msg := fmt.Sprintf("error deleting objects. %s", err.Error())
-		log.Errorf("HandleDeleteObjects: %s", msg)
-		return c.JSON(http.StatusInternalServerError, msg)
+		if strings.Contains(err.Error(), "prefix not found") {
+			errMsg := fmt.Errorf("no objects found with prefix: %s", prefix)
+			log.Error(errMsg.Error())
+			return c.JSON(http.StatusNotFound, errMsg.Error())
+		}
+		errMsg := fmt.Errorf("error deleting objects: %s", err.Error())
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusInternalServerError, errMsg.Error())
 	}
-
-	log.Info("HandleDeleteObjects: Successfully deleted prefix and its contents for prefix:", prefix)
+	log.Info("Successfully deleted prefix and its contents for prefix:", prefix)
 	return c.JSON(http.StatusOK, "Successfully deleted prefix and its contents")
-
 }
 
 func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error {
@@ -157,7 +154,7 @@ func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error {
 
 	_, err := s3Ctrl.S3Svc.DeleteObjects(input)
 	if err != nil {
-		return fmt.Errorf("deleteKeys: error Deleting objects: %s", err.Error())
+		return fmt.Errorf("error Deleting objects: %s", err.Error())
 	}
 	return nil
 }
@@ -169,21 +166,22 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
 	}
 	var deleteRequest DeleteRequest
 	if err := c.Bind(&deleteRequest); err != nil {
-		log.Errorf("HandleDeleteObjectsByList: Error parsing request body: %s" + err.Error())
-		return c.JSON(http.StatusBadRequest, "Invalid request body")
+		errMsg := fmt.Errorf("error parsing request body: %s" + err.Error())
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusBadRequest, errMsg.Error())
 	}
 
 	// Ensure there are keys to delete
 	if len(deleteRequest.Keys) == 0 {
-		errMsg := "No keys to delete. Please provide 'keys' in the request body."
-		log.Errorf("HandleDeleteObjectsByList: %s", errMsg)
-		return c.JSON(http.StatusUnprocessableEntity, errMsg)
+		errMsg := fmt.Errorf("no keys to delete. Please provide 'keys' in the request body")
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
 
 	bucket := c.QueryParam("bucket")
 	s3Ctrl, err := bh.GetController(bucket)
 	if err != nil {
-		errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
+		errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error())
 		log.Error(errMsg.Error())
 		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
 	}
@@ -197,14 +195,14 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
 		// Check if the key exists before appending it to the keys list
 		keyExists, err := s3Ctrl.KeyExists(bucket, s3Path)
 		if err != nil {
-			msg := fmt.Errorf("error checking if key exists. %s", err.Error())
-			log.Errorf("HandleDeleteObjectsByList: %s", msg)
-			return c.JSON(http.StatusInternalServerError, msg)
+			errMsg := fmt.Errorf("error checking if key exists. %s", err.Error())
+			log.Error(errMsg.Error())
+			return c.JSON(http.StatusInternalServerError, errMsg)
 		}
 		if !keyExists {
-			errMsg := fmt.Sprintf("object %s not found", s3Path)
-			log.Errorf("HandleDeleteObjectsByList: %s", errMsg)
-			return c.JSON(http.StatusNotFound, errMsg)
+			errMsg := fmt.Errorf("object %s not found", s3Path)
+			log.Error(errMsg.Error())
+			return c.JSON(http.StatusNotFound, errMsg.Error())
 		}
 
 		keys = append(keys, *key)
@@ -213,11 +211,11 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
 	// Delete the objects using the deleteKeys function
 	err = s3Ctrl.DeleteKeys(bucket, keys)
 	if err != nil {
-		msg := fmt.Sprintf("error deleting objects. %s", err.Error())
-		log.Errorf("HandleDeleteObjectsByList: %s", msg)
-		return c.JSON(http.StatusInternalServerError, msg)
+		errMsg := fmt.Errorf("error deleting objects. %s", err.Error())
+		log.Error(errMsg.Error())
+		return c.JSON(http.StatusInternalServerError, errMsg)
 	}
 
-	log.Info("HandleDeleteObjectsByList: Successfully deleted objects:", deleteRequest.Keys)
+	log.Info("Successfully deleted objects:", deleteRequest.Keys)
 	return c.JSON(http.StatusOK, "Successfully deleted objects")
 }