feat: add optional sidecar files for metadata

This adds the option to store metadata for objects and buckets within a hidden folder alongside the bucket or object: bucket: .vgw_meta.<bucket> object: .vgw_meta.<object> Example invocation: ./versitygw -a myaccess -s mysecret posix --sidecar /tmp/gw The attributes are stored by name within the hidden directory.
versity · Apr 23, 2024 · 1ca6ca3 · 1ca6ca3
1 parent 62209cf
commit 1ca6ca3
Show file tree

Hide file tree

Showing 6 changed files with 181 additions and 19 deletions.
diff --git a/backend/meta/sidecar.go b/backend/meta/sidecar.go
@@ -0,0 +1,108 @@
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct{}
+
+const sidecarname = ".vgw_meta."
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(bucket, sidecarname+object)
+	if object == "" {
+		metadir = sidecarname + bucket
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(bucket, sidecarname+object)
+	if object == "" {
+		metadir = sidecarname + bucket
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(bucket, sidecarname+object)
+	if object == "" {
+		metadir = sidecarname + bucket
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(bucket, sidecarname+object)
+	if object == "" {
+		metadir = sidecarname + bucket
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(bucket, sidecarname+object)
+	if object == "" {
+		metadir = sidecarname + bucket
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}
diff --git a/backend/posix/posix.go b/backend/posix/posix.go
@@ -59,6 +59,8 @@ type Posix struct {
 	// used to determine if chowning is needed
 	euid int
 	egid int
+
+	skipprefix []string // skip these prefixes when walking
 }
 
 var _ backend.Backend = &Posix{}
@@ -78,11 +80,13 @@ const (
 	bucketLockKey       = "bucket-lock"
 	objectRetentionKey  = "object-retention"
 	objectLegalHoldKey  = "object-legal-hold"
+	sidecarprefix       = ".vgw_meta."
 )
 
 type PosixOpts struct {
 	ChownUID bool
 	ChownGID bool
+	SideCar  bool
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -96,14 +100,20 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
+	var skipprefx []string
+	if opts.SideCar {
+		skipprefx = []string{sidecarprefix}
+	}
+
 	return &Posix{
-		meta:     meta,
-		rootfd:   f,
-		rootdir:  rootdir,
-		euid:     os.Geteuid(),
-		egid:     os.Getegid(),
-		chownuid: opts.ChownUID,
-		chowngid: opts.ChownGID,
+		meta:       meta,
+		rootfd:     f,
+		rootdir:    rootdir,
+		euid:       os.Geteuid(),
+		egid:       os.Getegid(),
+		chownuid:   opts.ChownUID,
+		chowngid:   opts.ChownGID,
+		skipprefix: skipprefx,
 	}, nil
 }
 
@@ -129,6 +139,11 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
 			continue
 		}
 
+		if containsprefix(entry.Name(), p.skipprefix) {
+			// skip directories that match the skip prefix
+			continue
+		}
+
 		fi, err := entry.Info()
 		if err != nil {
 			// skip entries returning errors
@@ -175,6 +190,15 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
 	}, nil
 }
 
+func containsprefix(a string, strs []string) bool {
+	for _, s := range strs {
+		if strings.HasPrefix(a, s) {
+			return true
+		}
+	}
+	return false
+}
+
 func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
@@ -1649,7 +1673,7 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
-		p.fileToObj(bucket), []string{metaTmpDir})
+		p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -1770,7 +1794,7 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
-		p.fileToObj(bucket), []string{metaTmpDir})
+		p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}

diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go
@@ -597,7 +597,7 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
-		s.fileToObj(bucket), []string{metaTmpDir})
+		s.fileToObj(bucket), []string{metaTmpDir}, []string{})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -647,7 +647,7 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
-		s.fileToObj(bucket), []string{metaTmpDir})
+		s.fileToObj(bucket), []string{metaTmpDir}, []string{})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}

diff --git a/backend/walk.go b/backend/walk.go
@@ -38,7 +38,7 @@ var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string, skipprefix []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
 	var objects []types.Object
 
@@ -62,6 +62,9 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if contains(d.Name(), skipdirs) {
 			return fs.SkipDir
 		}
+		if containsprefix(d.Name(), skipprefix) {
+			return fs.SkipDir
+		}
 
 		if pastMax {
 			if len(objects) != 0 {
@@ -220,3 +223,12 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
+
+func containsprefix(a string, strs []string) bool {
+	for _, s := range strs {
+		if strings.HasPrefix(a, s) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/backend/walk_test.go b/backend/walk_test.go
@@ -108,7 +108,7 @@ func TestWalk(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{}, []string{})
 		if err != nil {
 			t.Fatalf("walk: %v", err)
 		}

diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go
@@ -24,6 +24,7 @@ import (
 
 var (
 	chownuid, chowngid bool
+	sidecar            bool
 )
 
 func posixCommand() *cli.Command {
@@ -54,6 +55,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "sidecar",
+				Usage:       "use sidecar files for metadata storage",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
 		},
 	}
 }
@@ -64,15 +71,26 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+	opts := posix.PosixOpts{
 		ChownUID: chownuid,
 		ChownGID: chowngid,
-	})
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar:
+		ms = meta.SideCar{}
+		opts.SideCar = true
+	default: // default is to use xattr metadata
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %v", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}