This repository has been archived by the owner on Oct 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #25 from ipfs/chore/refactor
Refactor datastore config functions
- Loading branch information
Showing
9 changed files
with
371 additions
and
320 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
*.out | ||
*.coverprofile | ||
*.test | ||
*~ | ||
|
||
coverage.txt |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
package repo | ||
|
||
import ( | ||
"errors" | ||
"os" | ||
"path/filepath" | ||
|
||
badgerds "github.com/ipfs/go-ds-badger" | ||
) | ||
|
||
type badgerdsDatastoreConfig struct { | ||
path string | ||
syncWrites bool | ||
} | ||
|
||
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore | ||
// from the given parameters | ||
func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
var c badgerdsDatastoreConfig | ||
var ok bool | ||
|
||
c.path, ok = params["path"].(string) | ||
if !ok { | ||
return nil, errors.New("'path' field is missing or not string") | ||
} | ||
|
||
sw, ok := params["syncWrites"] | ||
if !ok { | ||
c.syncWrites = true | ||
} else { | ||
if swb, ok := sw.(bool); ok { | ||
c.syncWrites = swb | ||
} else { | ||
return nil, errors.New("'syncWrites' field was not a boolean") | ||
} | ||
} | ||
|
||
return &c, nil | ||
} | ||
|
||
func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec { | ||
return map[string]interface{}{ | ||
"type": "badgerds", | ||
"path": c.path, | ||
} | ||
} | ||
|
||
func (c *badgerdsDatastoreConfig) Create(path string) (Datastore, error) { | ||
p := c.path | ||
if !filepath.IsAbs(p) { | ||
p = filepath.Join(path, p) | ||
} | ||
|
||
err := os.MkdirAll(p, 0755) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
defopts := badgerds.DefaultOptions | ||
defopts.SyncWrites = c.syncWrites | ||
|
||
return badgerds.NewDatastore(p, &defopts) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package repo | ||
|
||
import ( | ||
"fmt" | ||
"path/filepath" | ||
|
||
flatfs "github.com/ipfs/go-ds-flatfs" | ||
) | ||
|
||
type flatfsDatastoreConfig struct { | ||
path string | ||
shardFun *flatfs.ShardIdV1 | ||
syncField bool | ||
} | ||
|
||
// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec | ||
func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
var c flatfsDatastoreConfig | ||
var ok bool | ||
var err error | ||
|
||
c.path, ok = params["path"].(string) | ||
if !ok { | ||
return nil, fmt.Errorf("'path' field is missing or not boolean") | ||
} | ||
|
||
sshardFun, ok := params["shardFunc"].(string) | ||
if !ok { | ||
return nil, fmt.Errorf("'shardFunc' field is missing or not a string") | ||
} | ||
c.shardFun, err = flatfs.ParseShardFunc(sshardFun) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
c.syncField, ok = params["sync"].(bool) | ||
if !ok { | ||
return nil, fmt.Errorf("'sync' field is missing or not boolean") | ||
} | ||
return &c, nil | ||
} | ||
|
||
func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec { | ||
return map[string]interface{}{ | ||
"type": "flatfs", | ||
"path": c.path, | ||
"shardFunc": c.shardFun.String(), | ||
} | ||
} | ||
|
||
func (c *flatfsDatastoreConfig) Create(path string) (Datastore, error) { | ||
p := c.path | ||
if !filepath.IsAbs(p) { | ||
p = filepath.Join(path, p) | ||
} | ||
|
||
return flatfs.CreateOrOpen(p, c.shardFun, c.syncField) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
package repo | ||
|
||
import ( | ||
"errors" | ||
"fmt" | ||
"path/filepath" | ||
|
||
levelds "github.com/ipfs/go-ds-leveldb" | ||
ldbopts "github.com/syndtr/goleveldb/leveldb/opt" | ||
) | ||
|
||
type leveldsDatastoreConfig struct { | ||
path string | ||
compression ldbopts.Compression | ||
} | ||
|
||
// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec | ||
func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
var c leveldsDatastoreConfig | ||
var ok bool | ||
|
||
c.path, ok = params["path"].(string) | ||
if !ok { | ||
return nil, errors.New("'path' field is missing or not string") | ||
} | ||
|
||
switch cm := params["compression"].(string); cm { | ||
case "none": | ||
c.compression = ldbopts.NoCompression | ||
case "snappy": | ||
c.compression = ldbopts.SnappyCompression | ||
case "": | ||
c.compression = ldbopts.DefaultCompression | ||
default: | ||
return nil, fmt.Errorf("unrecognized value for compression: %s", cm) | ||
} | ||
|
||
return &c, nil | ||
} | ||
|
||
func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec { | ||
return map[string]interface{}{ | ||
"type": "levelds", | ||
"path": c.path, | ||
} | ||
} | ||
|
||
func (c *leveldsDatastoreConfig) Create(path string) (Datastore, error) { | ||
p := c.path | ||
if !filepath.IsAbs(p) { | ||
p = filepath.Join(path, p) | ||
} | ||
|
||
return levelds.NewDatastore(p, &levelds.Options{ | ||
Compression: c.compression, | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package repo | ||
|
||
import ( | ||
"errors" | ||
|
||
ds "github.com/ipfs/go-datastore" | ||
) | ||
|
||
type logDatastoreConfig struct { | ||
child DatastoreConfig | ||
name string | ||
} | ||
|
||
// LogDatastoreConfig returns a log DatastoreConfig from a spec | ||
func LogDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
childField, ok := params["child"].(map[string]interface{}) | ||
if !ok { | ||
return nil, errors.New("'child' field is missing or not a map") | ||
} | ||
child, err := AnyDatastoreConfig(childField) | ||
if err != nil { | ||
return nil, err | ||
} | ||
name, ok := params["name"].(string) | ||
if !ok { | ||
return nil, errors.New("'name' field was missing or not a string") | ||
} | ||
return &logDatastoreConfig{child, name}, nil | ||
|
||
} | ||
|
||
func (c *logDatastoreConfig) Create(path string) (Datastore, error) { | ||
child, err := c.child.Create(path) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return ds.NewLogDatastore(child, c.name), nil | ||
} | ||
|
||
func (c *logDatastoreConfig) DiskSpec() DiskSpec { | ||
return c.child.DiskSpec() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
package repo | ||
|
||
import ( | ||
"errors" | ||
|
||
measure "github.com/ipfs/go-ds-measure" | ||
) | ||
|
||
// MeasureDatastoreConfig returns a measure DatastoreConfig from a spec | ||
func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
childField, ok := params["child"].(map[string]interface{}) | ||
if !ok { | ||
return nil, errors.New("'child' field is missing or not a map") | ||
} | ||
child, err := AnyDatastoreConfig(childField) | ||
if err != nil { | ||
return nil, err | ||
} | ||
prefix, ok := params["prefix"].(string) | ||
if !ok { | ||
return nil, errors.New("'prefix' field was missing or not a string") | ||
} | ||
return &measureDatastoreConfig{child, prefix}, nil | ||
} | ||
|
||
func (c *measureDatastoreConfig) DiskSpec() DiskSpec { | ||
return c.child.DiskSpec() | ||
} | ||
|
||
func (c measureDatastoreConfig) Create(path string) (Datastore, error) { | ||
child, err := c.child.Create(path) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return measure.New(c.prefix, child), nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package repo | ||
|
||
import ( | ||
ds "github.com/ipfs/go-datastore" | ||
) | ||
|
||
type memDatastoreConfig struct { | ||
cfg map[string]interface{} | ||
} | ||
|
||
// MemDatastoreConfig returns a memory DatastoreConfig from a spec | ||
func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
return &memDatastoreConfig{params}, nil | ||
} | ||
|
||
func (c *memDatastoreConfig) DiskSpec() DiskSpec { | ||
return nil | ||
} | ||
|
||
func (c *memDatastoreConfig) Create(string) (Datastore, error) { | ||
return ds.NewMapDatastore(), nil | ||
} | ||
|
||
type measureDatastoreConfig struct { | ||
child DatastoreConfig | ||
prefix string | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
package repo | ||
|
||
import ( | ||
"errors" | ||
"sort" | ||
|
||
ds "github.com/ipfs/go-datastore" | ||
mount "github.com/ipfs/go-datastore/mount" | ||
) | ||
|
||
type mountDatastoreConfig struct { | ||
mounts []premount | ||
} | ||
|
||
type premount struct { | ||
ds DatastoreConfig | ||
prefix ds.Key | ||
} | ||
|
||
// MountDatastoreConfig returns a mount DatastoreConfig from a spec | ||
func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) { | ||
var res mountDatastoreConfig | ||
mounts, ok := params["mounts"].([]interface{}) | ||
if !ok { | ||
return nil, errors.New("'mounts' field is missing or not an array") | ||
} | ||
for _, iface := range mounts { | ||
cfg, ok := iface.(map[string]interface{}) | ||
if !ok { | ||
return nil, errors.New("expected map for mountpoint") | ||
} | ||
|
||
child, err := AnyDatastoreConfig(cfg) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
prefix, found := cfg["mountpoint"] | ||
if !found { | ||
return nil, errors.New("no 'mountpoint' on mount") | ||
} | ||
|
||
res.mounts = append(res.mounts, premount{ | ||
ds: child, | ||
prefix: ds.NewKey(prefix.(string)), | ||
}) | ||
} | ||
sort.Slice(res.mounts, | ||
func(i, j int) bool { | ||
return res.mounts[i].prefix.String() > res.mounts[j].prefix.String() | ||
}) | ||
|
||
return &res, nil | ||
} | ||
|
||
func (c *mountDatastoreConfig) DiskSpec() DiskSpec { | ||
cfg := map[string]interface{}{"type": "mount"} | ||
mounts := make([]interface{}, len(c.mounts)) | ||
for i, m := range c.mounts { | ||
c := m.ds.DiskSpec() | ||
if c == nil { | ||
c = make(map[string]interface{}) | ||
} | ||
c["mountpoint"] = m.prefix.String() | ||
mounts[i] = c | ||
} | ||
cfg["mounts"] = mounts | ||
return cfg | ||
} | ||
|
||
func (c *mountDatastoreConfig) Create(path string) (Datastore, error) { | ||
mounts := make([]mount.Mount, len(c.mounts)) | ||
for i, m := range c.mounts { | ||
ds, err := m.ds.Create(path) | ||
if err != nil { | ||
return nil, err | ||
} | ||
mounts[i].Datastore = ds | ||
mounts[i].Prefix = m.prefix | ||
} | ||
return mount.New(mounts), nil | ||
} |
Oops, something went wrong.