Merge pull request #25 from ipfs/chore/refactor

Refactor datastore config functions

.gitignore

@@ -0,0 +1,6 @@
+*.out
+*.coverprofile
+*.test
+*~
+
+coverage.txt

repo/badgerds.go

@@ -0,0 +1,63 @@
+package repo
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	badgerds "github.com/ipfs/go-ds-badger"
+)
+
+type badgerdsDatastoreConfig struct {
+	path       string
+	syncWrites bool
+}
+
+// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
+// from the given parameters
+func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	var c badgerdsDatastoreConfig
+	var ok bool
+
+	c.path, ok = params["path"].(string)
+	if !ok {
+		return nil, errors.New("'path' field is missing or not string")
+	}
+
+	sw, ok := params["syncWrites"]
+	if !ok {
+		c.syncWrites = true
+	} else {
+		if swb, ok := sw.(bool); ok {
+			c.syncWrites = swb
+		} else {
+			return nil, errors.New("'syncWrites' field was not a boolean")
+		}
+	}
+
+	return &c, nil
+}
+
+func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec {
+	return map[string]interface{}{
+		"type": "badgerds",
+		"path": c.path,
+	}
+}
+
+func (c *badgerdsDatastoreConfig) Create(path string) (Datastore, error) {
+	p := c.path
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(path, p)
+	}
+
+	err := os.MkdirAll(p, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	defopts := badgerds.DefaultOptions
+	defopts.SyncWrites = c.syncWrites
+
+	return badgerds.NewDatastore(p, &defopts)
+}

repo/flatfsds.go

@@ -0,0 +1,58 @@
+package repo
+
+import (
+	"fmt"
+	"path/filepath"
+
+	flatfs "github.com/ipfs/go-ds-flatfs"
+)
+
+type flatfsDatastoreConfig struct {
+	path      string
+	shardFun  *flatfs.ShardIdV1
+	syncField bool
+}
+
+// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec
+func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	var c flatfsDatastoreConfig
+	var ok bool
+	var err error
+
+	c.path, ok = params["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("'path' field is missing or not boolean")
+	}
+
+	sshardFun, ok := params["shardFunc"].(string)
+	if !ok {
+		return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
+	}
+	c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
+	if err != nil {
+		return nil, err
+	}
+
+	c.syncField, ok = params["sync"].(bool)
+	if !ok {
+		return nil, fmt.Errorf("'sync' field is missing or not boolean")
+	}
+	return &c, nil
+}
+
+func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
+	return map[string]interface{}{
+		"type":      "flatfs",
+		"path":      c.path,
+		"shardFunc": c.shardFun.String(),
+	}
+}
+
+func (c *flatfsDatastoreConfig) Create(path string) (Datastore, error) {
+	p := c.path
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(path, p)
+	}
+
+	return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
+}

repo/levelds.go

@@ -0,0 +1,57 @@
+package repo
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	levelds "github.com/ipfs/go-ds-leveldb"
+	ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+type leveldsDatastoreConfig struct {
+	path        string
+	compression ldbopts.Compression
+}
+
+// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec
+func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	var c leveldsDatastoreConfig
+	var ok bool
+
+	c.path, ok = params["path"].(string)
+	if !ok {
+		return nil, errors.New("'path' field is missing or not string")
+	}
+
+	switch cm := params["compression"].(string); cm {
+	case "none":
+		c.compression = ldbopts.NoCompression
+	case "snappy":
+		c.compression = ldbopts.SnappyCompression
+	case "":
+		c.compression = ldbopts.DefaultCompression
+	default:
+		return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
+	}
+
+	return &c, nil
+}
+
+func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
+	return map[string]interface{}{
+		"type": "levelds",
+		"path": c.path,
+	}
+}
+
+func (c *leveldsDatastoreConfig) Create(path string) (Datastore, error) {
+	p := c.path
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(path, p)
+	}
+
+	return levelds.NewDatastore(p, &levelds.Options{
+		Compression: c.compression,
+	})
+}

repo/logds.go

@@ -0,0 +1,42 @@
+package repo
+
+import (
+	"errors"
+
+	ds "github.com/ipfs/go-datastore"
+)
+
+type logDatastoreConfig struct {
+	child DatastoreConfig
+	name  string
+}
+
+// LogDatastoreConfig returns a log DatastoreConfig from a spec
+func LogDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	childField, ok := params["child"].(map[string]interface{})
+	if !ok {
+		return nil, errors.New("'child' field is missing or not a map")
+	}
+	child, err := AnyDatastoreConfig(childField)
+	if err != nil {
+		return nil, err
+	}
+	name, ok := params["name"].(string)
+	if !ok {
+		return nil, errors.New("'name' field was missing or not a string")
+	}
+	return &logDatastoreConfig{child, name}, nil
+
+}
+
+func (c *logDatastoreConfig) Create(path string) (Datastore, error) {
+	child, err := c.child.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	return ds.NewLogDatastore(child, c.name), nil
+}
+
+func (c *logDatastoreConfig) DiskSpec() DiskSpec {
+	return c.child.DiskSpec()
+}

repo/measureds.go

@@ -0,0 +1,36 @@
+package repo
+
+import (
+	"errors"
+
+	measure "github.com/ipfs/go-ds-measure"
+)
+
+// MeasureDatastoreConfig returns a measure DatastoreConfig from a spec
+func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	childField, ok := params["child"].(map[string]interface{})
+	if !ok {
+		return nil, errors.New("'child' field is missing or not a map")
+	}
+	child, err := AnyDatastoreConfig(childField)
+	if err != nil {
+		return nil, err
+	}
+	prefix, ok := params["prefix"].(string)
+	if !ok {
+		return nil, errors.New("'prefix' field was missing or not a string")
+	}
+	return &measureDatastoreConfig{child, prefix}, nil
+}
+
+func (c *measureDatastoreConfig) DiskSpec() DiskSpec {
+	return c.child.DiskSpec()
+}
+
+func (c measureDatastoreConfig) Create(path string) (Datastore, error) {
+	child, err := c.child.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	return measure.New(c.prefix, child), nil
+}

repo/memds.go

@@ -0,0 +1,27 @@
+package repo
+
+import (
+	ds "github.com/ipfs/go-datastore"
+)
+
+type memDatastoreConfig struct {
+	cfg map[string]interface{}
+}
+
+// MemDatastoreConfig returns a memory DatastoreConfig from a spec
+func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	return &memDatastoreConfig{params}, nil
+}
+
+func (c *memDatastoreConfig) DiskSpec() DiskSpec {
+	return nil
+}
+
+func (c *memDatastoreConfig) Create(string) (Datastore, error) {
+	return ds.NewMapDatastore(), nil
+}
+
+type measureDatastoreConfig struct {
+	child  DatastoreConfig
+	prefix string
+}

repo/mountds.go

@@ -0,0 +1,82 @@
+package repo
+
+import (
+	"errors"
+	"sort"
+
+	ds "github.com/ipfs/go-datastore"
+	mount "github.com/ipfs/go-datastore/mount"
+)
+
+type mountDatastoreConfig struct {
+	mounts []premount
+}
+
+type premount struct {
+	ds     DatastoreConfig
+	prefix ds.Key
+}
+
+// MountDatastoreConfig returns a mount DatastoreConfig from a spec
+func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
+	var res mountDatastoreConfig
+	mounts, ok := params["mounts"].([]interface{})
+	if !ok {
+		return nil, errors.New("'mounts' field is missing or not an array")
+	}
+	for _, iface := range mounts {
+		cfg, ok := iface.(map[string]interface{})
+		if !ok {
+			return nil, errors.New("expected map for mountpoint")
+		}
+
+		child, err := AnyDatastoreConfig(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		prefix, found := cfg["mountpoint"]
+		if !found {
+			return nil, errors.New("no 'mountpoint' on mount")
+		}
+
+		res.mounts = append(res.mounts, premount{
+			ds:     child,
+			prefix: ds.NewKey(prefix.(string)),
+		})
+	}
+	sort.Slice(res.mounts,
+		func(i, j int) bool {
+			return res.mounts[i].prefix.String() > res.mounts[j].prefix.String()
+		})
+
+	return &res, nil
+}
+
+func (c *mountDatastoreConfig) DiskSpec() DiskSpec {
+	cfg := map[string]interface{}{"type": "mount"}
+	mounts := make([]interface{}, len(c.mounts))
+	for i, m := range c.mounts {
+		c := m.ds.DiskSpec()
+		if c == nil {
+			c = make(map[string]interface{})
+		}
+		c["mountpoint"] = m.prefix.String()
+		mounts[i] = c
+	}
+	cfg["mounts"] = mounts
+	return cfg
+}
+
+func (c *mountDatastoreConfig) Create(path string) (Datastore, error) {
+	mounts := make([]mount.Mount, len(c.mounts))
+	for i, m := range c.mounts {
+		ds, err := m.ds.Create(path)
+		if err != nil {
+			return nil, err
+		}
+		mounts[i].Datastore = ds
+		mounts[i].Prefix = m.prefix
+	}
+	return mount.New(mounts), nil
+}