working

contentprovider.go

@@ -219,7 +219,9 @@ func (p *contentProvider) fillContentMatches(ms []*candidateMatch, numContextLin
 	var result []LineMatch
 	for len(ms) > 0 {
 		m := ms[0]
-		num, lineStart, lineEnd := p.newlines().atOffset(m.byteOffset)
+		num := p.newlines().atOffset(m.byteOffset)
+		lineStart := int(p.newlines().lineStart(num))
+		lineEnd := int(p.newlines().lineEnd(num, true))
 
 		var lineCands []*candidateMatch
 
@@ -461,12 +463,6 @@ func (nls newlines) atOffset(offset uint32) (lineNumber int) {
 	return idx + 1
 }
 
-// lineBounds returns the byte offsets of the start and end of the 1-based
-// lineNumber.
-func (nls newlines) lineBounds(lineNumber int) (start, end uint32) {
-	return nls.lineStart(lineNumber), nls.lineEnd(lineNumber)
-}
-
 // lineStart returns the byte offset of the beginning of the given line.
 // lineNumber is 1-based. If lineNumber is out of range of the lines in the
 // file, the return value will be clamped to [0,fileSize].
@@ -484,18 +480,23 @@ func (nls newlines) lineStart(lineNumber int) uint32 {
 }
 
 // lineEnd returns the (exclusive) byte offset pointing immediately after the
-// last byte of the line. This includes the terminating newline if it exists (a
-// line may be terminated by EOF). lineNumber is 1-based. If lineNumber is out
+// last byte of the line. lineNumber is 1-based. If lineNumber is out
 // of range of the lines in the file, the return value will be clamped to
 // [0,fileSize].
-func (nls newlines) lineEnd(lineNumber int) uint32 {
+//
+// If trimNewline is set, the offset exclude any line-terminating newline.
+// Note: this does _not_ trim any carriage returns and should probably not be
+// used by default. It only exists for backwards compatibility.
+func (nls newlines) lineEnd(lineNumber int, trimNewline bool) uint32 {
 	// nls.locs[0] + 1 is the start of the 2nd line of data.
 	endIdx := lineNumber - 1
 
 	if endIdx < 0 {
 		return 0
 	} else if endIdx >= len(nls.locs) {
 		return nls.fileSize
+	} else if trimNewline {
+		return nls.locs[endIdx]
 	} else {
 		return nls.locs[endIdx] + 1
 	}
@@ -509,7 +510,7 @@ func (nls newlines) getLines(data []byte, low, high int) []byte {
 	}
 
 	lowStart := nls.lineStart(low)
-	highEnd := nls.lineEnd(high - 1)
+	highEnd := nls.lineEnd(high-1, false)
 
 	return data[lowStart:highEnd]
 }

contentprovider_test.go

@@ -137,7 +137,7 @@ func TestAtOffset(t *testing.T) {
 			if gotLineStart := nls.lineStart(gotLineNumber); gotLineStart != tt.lineStart {
 				t.Fatalf("expected line start %d, got %d", tt.lineStart, gotLineStart)
 			}
-			if gotLineEnd := nls.lineEnd(gotLineNumber); gotLineEnd != tt.lineEnd {
+			if gotLineEnd := nls.lineEnd(gotLineNumber, false); gotLineEnd != tt.lineEnd {
 				t.Fatalf("expected line end %d, got %d", tt.lineEnd, gotLineEnd)
 			}
 		})
@@ -187,10 +187,11 @@ func TestLineBounds(t *testing.T) {
 	for _, tt := range cases {
 		t.Run("", func(t *testing.T) {
 			nls := getNewlines(tt.data)
-			gotStart, gotEnd := nls.lineBounds(tt.lineNumber)
+			gotStart := nls.lineStart(tt.lineNumber)
 			if gotStart != tt.start {
 				t.Fatalf("expected line start %d, got %d", tt.start, gotStart)
 			}
+			gotEnd := nls.lineEnd(tt.lineNumber, false)
 			if gotEnd != tt.end {
 				t.Fatalf("expected line end %d, got %d", tt.end, gotEnd)
 			}

index_test.go

@@ -2452,7 +2452,7 @@ func TestIOStats(t *testing.T) {
 		res := searchForTest(t, b, q)
 
 		// 4096 (content) + 2 (overhead: newlines or doc sections)
-		if got, want := res.Stats.ContentBytesLoaded, int64(4098); got != want {
+		if got, want := res.Stats.ContentBytesLoaded, int64(4100); got != want {
 			t.Errorf("got content I/O %d, want %d", got, want)
 		}
 
@@ -3243,7 +3243,7 @@ func TestLineAnd(t *testing.T) {
 	}
 	t.Run("LineMatches", func(t *testing.T) {
 		res := searchForTest(t, b, &q)
-		wantRegexpCount := 1
+		wantRegexpCount := 2
 		if gotRegexpCount := res.RegexpsConsidered; gotRegexpCount != wantRegexpCount {
 			t.Errorf("got %d, wanted %d", gotRegexpCount, wantRegexpCount)
 		}
@@ -3254,7 +3254,7 @@ func TestLineAnd(t *testing.T) {
 
 	t.Run("ChunkMatches", func(t *testing.T) {
 		res := searchForTest(t, b, &q, chunkOpts)
-		wantRegexpCount := 1
+		wantRegexpCount := 2 // TODO: justify this change
 		if gotRegexpCount := res.RegexpsConsidered; gotRegexpCount != wantRegexpCount {
 			t.Errorf("got %d, wanted %d", gotRegexpCount, wantRegexpCount)
 		}

matchtree.go

@@ -699,7 +699,7 @@ func (t *andLineMatchTree) matches(cp *contentProvider, cost int, known map[matc
 		}
 		prev = line
 		byteStart := int(cp.newlines().lineStart(line))
-		byteEnd := int(cp.newlines().lineEnd(line))
+		byteEnd := int(cp.newlines().lineEnd(line, false))
 		lines = append(lines, lineRange{byteStart, byteEnd})
 	}