Merge pull request peterbourgon#41 from epiclabs-io/master

support for slashes in key names and advanced key transformation

README.md

@@ -103,6 +103,50 @@ always flatly available on the disk. diskv will never do anything that would
 prevent you from accessing, copying, backing up, or otherwise interacting with
 your data via common UNIX commandline tools.
 
+## Advanced path transformation
+
+If you need more control over the file name written to disk or if you want to support
+slashes in your key name or special characters in the keys, you can use the
+AdvancedTransform property. You must supply a function that returns
+a special PathKey structure, which is a breakdown of a path and a file name. Strings
+returned must be clean of any slashes or special characters:
+
+```go
+func AdvancedTransformExample(key string) *diskv.PathKey {
+	path := strings.Split(key, "/")
+	last := len(path) - 1
+	return &diskv.PathKey{
+		Path:     path[:last],
+		FileName: path[last] + ".txt",
+	}
+}
+
+// If you provide an AdvancedTransform, you must also provide its
+// inverse:
+
+func InverseTransformExample(pathKey *diskv.PathKey) (key string) {
+	txt := pathKey.FileName[len(pathKey.FileName)-4:]
+	if txt != ".txt" {
+		panic("Invalid file found in storage folder!")
+	}
+	return strings.Join(pathKey.Path, "/") + pathKey.FileName[:len(pathKey.FileName)-4]
+}
+
+func main() {
+	d := diskv.New(diskv.Options{
+		BasePath:          "my-data-dir",
+		AdvancedTransform: AdvancedTransformExample,
+		InverseTransform:  InverseTransformExample,
+		CacheSizeMax:      1024 * 1024,
+	})
+	// Write some text to the key "alpha/beta/gamma".
+	key := "alpha/beta/gamma"
+	d.WriteString(key, "¡Hola!") // will be stored in "<basedir>/alpha/beta/gamma.txt"
+	fmt.Println(d.ReadString("alpha/beta/gamma"))
+}
+```
+
+
 ## Adding a cache
 
 An in-memory caching layer is provided by combining the BasicStore
@@ -139,3 +183,9 @@ data to be handled efficiently.
  * Needs plenty of robust testing: huge datasets, etc...
  * More thorough benchmarking
  * Your suggestions for use-cases I haven't thought of
+
+
+# Credits and contributions
+
+Original idea, design and implementation: [Peter Bourgon](https://github.com/peterbourgon)
+Other collaborations: [Javier Peletier](https://github.com/jpeletier) ([Epic Labs](https://www.epiclabs.io))

basic_test.go

@@ -3,6 +3,9 @@ package diskv
 import (
 	"bytes"
 	"errors"
+	"math/rand"
+	"regexp"
+	"strings"
 	"testing"
 	"time"
 )
@@ -334,3 +337,94 @@ func TestAtomicWrite(t *testing.T) {
 		t.Fatal("Store isn't empty")
 	}
 }
+
+const letterBytes = "abcdef0123456789"
+
+func randStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
+func TestHybridStore(t *testing.T) {
+	regex := regexp.MustCompile("[0-9a-fA-F]{64}")
+
+	transformFunc := func(s string) *PathKey {
+
+		if regex.MatchString(s) {
+			return &PathKey{Path: []string{"objects", s[0:2]},
+				FileName: s,
+			}
+		}
+
+		folders := strings.Split(s, "/")
+		lfolders := len(folders)
+		if lfolders > 1 {
+			return &PathKey{Path: folders[:lfolders-1],
+				FileName: folders[lfolders-1],
+			}
+		}
+
+		return &PathKey{Path: []string{},
+			FileName: s,
+		}
+	}
+
+	inverseTransformFunc := func(pathKey *PathKey) string {
+
+		if regex.MatchString(pathKey.FileName) {
+			return pathKey.FileName
+
+		}
+
+		if len(pathKey.Path) == 0 {
+			return pathKey.FileName
+		}
+
+		return strings.Join(pathKey.Path, "/") + "/" + pathKey.FileName
+
+	}
+	opts := Options{
+		BasePath:          "test-data",
+		CacheSizeMax:      1024,
+		AdvancedTransform: transformFunc,
+		InverseTransform:  inverseTransformFunc,
+	}
+	d := New(opts)
+	defer d.EraseAll()
+
+	testData := map[string]string{}
+
+	for i := 0; i < 100; i++ {
+		testData[randStringBytes(64)] = randStringBytes(100)
+	}
+
+	for i := 0; i < 100; i++ {
+		testData[randStringBytes(20)] = randStringBytes(100)
+	}
+
+	for i := 0; i < 100; i++ {
+		numsep := rand.Intn(10) + 1
+		key := ""
+		for j := 0; j < numsep; j++ {
+			key += randStringBytes(10) + "/"
+		}
+		key += randStringBytes(40)
+		testData[key] = randStringBytes(100)
+	}
+
+	for k, v := range testData {
+		d.WriteString(k, v)
+	}
+
+	for k, v := range testData {
+		readVal := d.ReadString(k)
+
+		if v != readVal {
+			t.Fatalf("read: expected %s, got %s", v, readVal)
+		}
+	}
+
+}