feat(selectors): negative values for slice matcher's From and To

testutil/simplebytes.go

@@ -0,0 +1,75 @@
+package testutil
+
+import (
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/ipld/go-ipld-prime/node/mixins"
+)
+
+var _ datamodel.Node = simpleBytes(nil)
+
+// simpleBytes is like basicnode's plainBytes but it doesn't implement
+// LargeBytesNode so we can exercise the non-LBN case.
+type simpleBytes []byte
+
+// NewSimpleBytes is identical to basicnode.NewBytes but the returned node
+// doesn't implement LargeBytesNode, which can be useful for testing cases
+// where we want to exercise non-LBN code paths.
+func NewSimpleBytes(value []byte) datamodel.Node {
+	v := simpleBytes(value)
+	return &v
+}
+
+// -- Node interface methods -->
+
+func (simpleBytes) Kind() datamodel.Kind {
+	return datamodel.Kind_Bytes
+}
+func (simpleBytes) LookupByString(string) (datamodel.Node, error) {
+	return mixins.Bytes{TypeName: "bytes"}.LookupByString("")
+}
+func (simpleBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) {
+	return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil)
+}
+func (simpleBytes) LookupByIndex(idx int64) (datamodel.Node, error) {
+	return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0)
+}
+func (simpleBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) {
+	return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg)
+}
+func (simpleBytes) MapIterator() datamodel.MapIterator {
+	return nil
+}
+func (simpleBytes) ListIterator() datamodel.ListIterator {
+	return nil
+}
+func (simpleBytes) Length() int64 {
+	return -1
+}
+func (simpleBytes) IsAbsent() bool {
+	return false
+}
+func (simpleBytes) IsNull() bool {
+	return false
+}
+func (simpleBytes) AsBool() (bool, error) {
+	return mixins.Bytes{TypeName: "bytes"}.AsBool()
+}
+func (simpleBytes) AsInt() (int64, error) {
+	return mixins.Bytes{TypeName: "bytes"}.AsInt()
+}
+func (simpleBytes) AsFloat() (float64, error) {
+	return mixins.Bytes{TypeName: "bytes"}.AsFloat()
+}
+func (simpleBytes) AsString() (string, error) {
+	return mixins.Bytes{TypeName: "bytes"}.AsString()
+}
+func (n simpleBytes) AsBytes() ([]byte, error) {
+	return []byte(n), nil
+}
+func (simpleBytes) AsLink() (datamodel.Link, error) {
+	return mixins.Bytes{TypeName: "bytes"}.AsLink()
+}
+func (simpleBytes) Prototype() datamodel.NodePrototype {
+	return basicnode.Prototype__Bytes{}
+}

traversal/selector/matcher.go

@@ -3,6 +3,7 @@
 import (
 	"fmt"
 	"io"
+	"math"
 
 	"github.com/ipld/go-ipld-prime/datamodel"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -26,11 +27,35 @@
 // The returned node will be limited based on slicing the specified range of the
 // node into a new node, or making use of the `AsLargeBytes` io.ReadSeeker to
 // restrict response with a SectionReader.
+//
+// Slice supports [From,To) ranges, where From is inclusive and To is exclusive.
+// Negative values for From and To are interpreted as offsets from the end of
+// the node. If To is greater than the node length, it will be truncated to the
+// node length. If From is greater than the node length or greater than To, the
+// result will be a non-match.
 type Slice struct {
 	From int64
 	To   int64
 }
 
+func sliceBounds(from, to, length int64) (bool, int64, int64) {
+	if to < 0 {
+		to = length + to
+	} else if length < to {
+		to = length
+	}
+	if from < 0 {
+		from = length + from
+		if from < 0 {
+			from = 0
+		}
+	}
+	if from > to || from >= length {
+		return false, 0, 0
+	}
+	return true, from, to
+}
+
 func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) {
 	var from, to int64
 	switch n.Kind() {
@@ -39,35 +64,52 @@
 		if err != nil {
 			return nil, err
 		}
-		to = s.To
-		if int64(len(str)) < to {
-			to = int64(len(str))
-		}
-		from = s.From
-		if int64(len(str)) < from {
-			from = int64(len(str))
+
+		var match bool
+		match, from, to = sliceBounds(s.From, s.To, int64(len(str)))
+		if !match {
+			return nil, nil
 		}
 		return basicnode.NewString(str[from:to]), nil
 	case datamodel.Kind_Bytes:
+		to = s.To
+		from = s.From
+		var length int64 = math.MaxInt64
+		var rdr io.ReadSeeker
+		var bytes []byte
+		var err error
+
 		if lbn, ok := n.(datamodel.LargeBytesNode); ok {
-			rdr, err := lbn.AsLargeBytes()
+			rdr, err = lbn.AsLargeBytes()
 			if err != nil {
 				return nil, err
 			}
-			sr := io.NewSectionReader(&readerat{rdr, 0}, s.From, s.To-s.From)
-			return basicnode.NewBytesFromReader(sr), nil
-		}
-		bytes, err := n.AsBytes()
-		if err != nil {
-			return nil, err
+			// calculate length from seeker
+			length, err = rdr.Seek(0, io.SeekEnd)
+			if err != nil {
+				return nil, err
+			}
+			// reset
+			_, err = rdr.Seek(0, io.SeekStart)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			bytes, err = n.AsBytes()
+			if err != nil {
+				return nil, err
+			}
+			length = int64(len(bytes))
 		}
-		to = s.To
-		if int64(len(bytes)) < to {
-			to = int64(len(bytes))
+
+		var match bool
+		match, from, to = sliceBounds(from, to, length)
+		if !match {
+			return nil, nil
 		}
-		from = s.From
-		if int64(len(bytes)) < from {
-			from = int64(len(bytes))
+		if rdr != nil {
+			sr := io.NewSectionReader(&readerat{rdr, 0}, from, to-from)
+			return basicnode.NewBytesFromReader(sr), nil
 		}
 		return basicnode.NewBytes(bytes[from:to]), nil
 	default:
@@ -129,12 +171,9 @@
 		if err != nil {
 			return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'to' key that is a number")
 		}
-		if fromN > toN {
+		if toN >= 0 && fromN > toN {
 			return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is less than or equal to the 'to' key")
 		}
-		if fromN < 0 || toN < 0 {
-			return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with keys not less than 0")
-		}
 		return Matcher{&Slice{
 			From: fromN,
 			To:   toN,

traversal/selector/matcher_test.go

@@ -3,6 +3,7 @@ package selector_test
 import (
 	"fmt"
 	"math"
+	"regexp"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -22,7 +23,7 @@ func TestSubsetMatch(t *testing.T) {
 		node datamodel.Node
 	}{
 		{"stringNode", basicnode.NewString(expectedString)},
-		{"bytesNode", basicnode.NewBytes([]byte(expectedString))},
+		{"bytesNode", testutil.NewSimpleBytes([]byte(expectedString))},
 		{"largeBytesNode", testutil.NewMultiByteNode(
 			[]byte("foo"),
 			[]byte("bar"),
@@ -50,26 +51,39 @@ func TestSubsetMatch(t *testing.T) {
 	}
 
 	for _, tc := range []struct {
-		from int64
-		to   int64
-		exp  string
+		from  int64
+		to    int64
+		exp   string
+		match bool
 	}{
-		{0, math.MaxInt64, expectedString},
-		{0, int64(len(expectedString)), expectedString},
-		{0, 0, ""},
-		{0, 1, "f"},
-		{0, 2, "fo"},
-		{0, 3, "foo"},
-		{0, 4, "foob"},
-		{1, 4, "oob"},
-		{2, 4, "ob"},
-		{3, 4, "b"},
-		{4, 4, ""},
-		{4, math.MaxInt64, "arbaz!"},
-		{4, int64(len(expectedString)), "arbaz!"},
-		{4, int64(len(expectedString) - 1), "arbaz"},
-		{0, int64(len(expectedString) - 1), expectedString[0 : len(expectedString)-1]},
-		{0, int64(len(expectedString) - 2), expectedString[0 : len(expectedString)-2]},
+		{0, math.MaxInt64, expectedString, true},
+		{0, int64(len(expectedString)), expectedString, true},
+		{0, 0, "", true},
+		{0, 1, "f", true},
+		{0, 2, "fo", true},
+		{0, 3, "foo", true},
+		{0, 4, "foob", true},
+		{1, 4, "oob", true},
+		{2, 4, "ob", true},
+		{3, 4, "b", true},
+		{4, 4, "", true},
+		{4, math.MaxInt64, "arbaz!", true},
+		{4, int64(len(expectedString)), "arbaz!", true},
+		{4, int64(len(expectedString) - 1), "arbaz", true},
+		{0, int64(len(expectedString) - 1), expectedString[0 : len(expectedString)-1], true},
+		{0, int64(len(expectedString) - 2), expectedString[0 : len(expectedString)-2], true},
+		{0, -1, expectedString[0 : len(expectedString)-1], true},
+		{0, -2, expectedString[0 : len(expectedString)-2], true},
+		{-2, -1, "z", true},
+		{-1, math.MaxInt64, "!", true},
+		{-int64(len(expectedString)), math.MaxInt64, expectedString, true},
+		{math.MaxInt64 - 1, math.MaxInt64, "", false},
+		{int64(len(expectedString)), math.MaxInt64, "", false},
+		{-1, -2, "", false},                 // To < From, no match
+		{-1, -1, "", true},                  // To==From, match zero bytes
+		{-1000, -100, "", false},            // From undeflow, adjusted to 0, To underflow, not adjusted, To < From, no match
+		{-100, -1000, "", false},            // From undeflow, adjusted to 0, To underflow, adjusted to 0, To < From, no match
+		{-1000, 1000, expectedString, true}, // From undeflow, adjusted to 0, To overflow, adjusted to len, match all
 	} {
 		for _, variant := range nodes {
 			t.Run(fmt.Sprintf("%s[%d:%d]", variant.name, tc.from, tc.to), func(t *testing.T) {
@@ -92,20 +106,35 @@ func TestSubsetMatch(t *testing.T) {
 				})
 				qt.Assert(t, err, qt.IsNil)
 
-				qt.Assert(t, got, qt.IsNotNil)
-				qt.Assert(t, got.Kind(), qt.Equals, variant.node.Kind())
-				var gotString string
-				switch got.Kind() {
-				case datamodel.Kind_String:
-					gotString, err = got.AsString()
-					qt.Assert(t, err, qt.IsNil)
-				case datamodel.Kind_Bytes:
-					byts, err := got.AsBytes()
-					qt.Assert(t, err, qt.IsNil)
-					gotString = string(byts)
+				if tc.match {
+					qt.Assert(t, got, qt.IsNotNil)
+					qt.Assert(t, got.Kind(), qt.Equals, variant.node.Kind())
+					var gotString string
+					switch got.Kind() {
+					case datamodel.Kind_String:
+						gotString, err = got.AsString()
+						qt.Assert(t, err, qt.IsNil)
+					case datamodel.Kind_Bytes:
+						byts, err := got.AsBytes()
+						qt.Assert(t, err, qt.IsNil)
+						gotString = string(byts)
+					}
+					qt.Assert(t, gotString, qt.DeepEquals, tc.exp)
+				} else {
+					qt.Assert(t, got, qt.IsNil)
 				}
-				qt.Assert(t, gotString, qt.DeepEquals, tc.exp)
 			})
 		}
 	}
+
+	// when both are positive, we can validate ranges up-front
+	t.Run("invalid range", func(t *testing.T) {
+		selNode, err := mkRangeSelector(1000, 100)
+		qt.Assert(t, err, qt.IsNil)
+		re, err := regexp.Compile("from.*less than or equal to.*to")
+		qt.Assert(t, err, qt.IsNil)
+		ss, err := selector.ParseSelector(selNode)
+		qt.Assert(t, ss, qt.IsNil)
+		qt.Assert(t, err, qt.ErrorMatches, re)
+	})
 }

traversal/selector/spec_test.go

@@ -10,6 +10,7 @@ import (
 	qt "github.com/frankban/quicktest"
 	"github.com/warpfork/go-testmark"
 
+	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/dagjson"
 	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/ipld/go-ipld-prime/datamodel"
@@ -34,9 +35,7 @@ func testOneSpecFixtureFile(t *testing.T, filename string) {
 	data = []byte(crre.ReplaceAllString(string(data), "\n")) // fix windows carriage-return
 
 	doc, err := testmark.Parse(data)
-	if err != nil {
-		t.Fatalf("spec file parse failed?!: %s", err)
-	}
+	qt.Assert(t, err, qt.IsNil)
 
 	// Data hunk in this spec file are in "directories" of a test scenario each.
 	doc.BuildDirIndex()
@@ -51,18 +50,13 @@ func testOneSpecFixtureFile(t *testing.T, filename string) {
 			fixtureExpect := dir.Children["expect-visit"].Hunk.Body
 
 			// Parse data into DMT form.
-			nb := basicnode.Prototype.Any.NewBuilder()
-			if err := dagjson.Decode(nb, bytes.NewReader(fixtureData)); err != nil {
-				t.Errorf("failed to parse fixture data: %s", err)
-			}
-			dataDmt := nb.Build()
+			dataDmt, err := ipld.Decode(fixtureData, dagjson.Decode)
+			qt.Assert(t, err, qt.IsNil)
 
 			// Parse and compile Selector.
 			// (This is already arguably a test event on its own.
 			selector, err := selectorparse.ParseAndCompileJSONSelector(string(fixtureSelector))
-			if err != nil {
-				t.Errorf("failed to parse+compile selector: %s", err)
-			}
+			qt.Assert(t, err, qt.IsNil)
 
 			// Go!
 			//  We'll store the logs of our visit events as... ipld Nodes, actually.
@@ -113,18 +107,16 @@ func testOneSpecFixtureFile(t *testing.T, filename string) {
 				if len(line) == 0 {
 					continue
 				}
-				nb := basicnode.Prototype.Any.NewBuilder()
-				if err := json.Decode(nb, bytes.NewReader(line)); err != nil {
-					t.Errorf("failed to parse fixture visit descriptions: %s", err)
-				}
-				json.Encode(nb.Build(), &fixtureExpectNormBuf)
+				exp, err := ipld.Decode(line, json.Decode)
+				qt.Assert(t, err, qt.IsNil)
+				qt.Assert(t, ipld.EncodeStreaming(&fixtureExpectNormBuf, exp, json.Encode), qt.IsNil)
 				fixtureExpectNormBuf.WriteByte('\n')
 			}
 
 			// Serialize our own visit logs now too.
 			var visitLogString bytes.Buffer
 			for _, logEnt := range visitLogs {
-				json.Encode(logEnt, &visitLogString)
+				qt.Assert(t, ipld.EncodeStreaming(&visitLogString, logEnt, json.Encode), qt.IsNil)
 				visitLogString.WriteByte('\n')
 			}