e2e: recall and visual indication of ranks

This adds a new field to the golden files "targetRank" which records the
rank of the document we are looking for. Additionally the document is
marked with "**" in the golden files.

Additionally we add a new golden file which contains recall@1, recall@5
and the MRR.

I set the target documents by looking at the existing results and
guessing which was the one we wanted based on memory. In some cases we
no longer had the top document, for example for generate unit test.

Test Plan: go test

internal/e2e/e2e_rank_test.go

@@ -41,21 +41,24 @@ func TestRanking(t *testing.T) {
 		"https://github.com/golang/go/tree/go1.21.4",
 		"https://github.com/sourcegraph/cody/tree/vscode-v0.14.5",
 	}
-	queries := []string{
+	q := func(query, target string) rankingQuery {
+		return rankingQuery{Query: query, Target: target}
+	}
+	queries := []rankingQuery{
 		// golang/go
-		"test server",
-		"bytes buffer",
-		"bufio buffer",
+		q("test server", "github.com/golang/go/src/net/http/httptest/server.go"),
+		q("bytes buffer", "github.com/golang/go/src/bytes/buffer.go"),
+		q("bufio buffer", "github.com/golang/go/src/bufio/scan.go"),
 
 		// sourcegraph/sourcegraph
-		"graphql type User",
-		"Get database/user",
-		"InternalDoer",
-		"Repository metadata Write rbac",
+		q("graphql type User", "github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql"),
+		q("Get database/user", "github.com/sourcegraph/sourcegraph/internal/database/users.go"),
+		q("InternalDoer", "github.com/sourcegraph/sourcegraph/internal/httpcli/client.go"),
+		q("Repository metadata Write rbac", "github.com/sourcegraph/sourcegraph/internal/rbac/constants.go"), // unsure if this is the best doc?
 
 		// cody
-		"generate unit test",
-		"r:cody sourcegraph url",
+		q("generate unit test", "github.com/sourcegraph/cody/lib/shared/src/chat/recipes/generate-test.ts"),
+		q("r:cody sourcegraph url", "github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts"),
 	}
 
 	indexDir := t.TempDir()
@@ -72,7 +75,8 @@ func TestRanking(t *testing.T) {
 	}
 	defer ss.Close()
 
-	for _, queryStr := range queries {
+	var ranks []int
+	for _, rq := range queries {
 		// normalise queryStr for writing to fs
 		name := strings.Map(func(r rune) rune {
 			if strings.ContainsRune(" :", r) {
@@ -84,10 +88,10 @@ func TestRanking(t *testing.T) {
 				return r
 			}
 			return -1
-		}, queryStr)
+		}, rq.Query)
 
 		t.Run(name, func(t *testing.T) {
-			q, err := query.Parse(queryStr)
+			q, err := query.Parse(rq.Query)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -107,28 +111,75 @@ func TestRanking(t *testing.T) {
 				t.Fatal(err)
 			}
 
+			ranks = append(ranks, targetRank(rq, result.Files))
+
 			var gotBuf bytes.Buffer
-			marshalMatches(&gotBuf, queryStr, q, result.Files)
-			got := gotBuf.Bytes()
+			marshalMatches(&gotBuf, rq, q, result.Files)
+			assertGolden(t, name, gotBuf.Bytes())
+		})
+	}
+
+	t.Run("rank_stats", func(t *testing.T) {
+		if len(ranks) != len(queries) {
+			t.Skip("not computing rank stats since not all query cases ran")
+		}
+
+		var gotBuf bytes.Buffer
+		printf := func(format string, a ...any) {
+			_, _ = fmt.Fprintf(&gotBuf, format, a...)
+		}
+
+		printf("queries: %d\n", len(ranks))
 
-			wantPath := filepath.Join("testdata", name+".txt")
-			if *update {
-				if err := os.WriteFile(wantPath, got, 0600); err != nil {
-					t.Fatal(err)
+		for _, recallThreshold := range []int{1, 5} {
+			count := 0
+			for _, rank := range ranks {
+				if rank <= recallThreshold && rank > 0 {
+					count++
 				}
 			}
-			want, err := os.ReadFile(wantPath)
-			if err != nil {
-				t.Fatal(err)
-			}
+			countp := float64(count) * 100 / float64(len(ranks))
+			printf("recall@%d: %d (%.0f%%)\n", recallThreshold, count, countp)
+		}
 
-			if d := cmp.Diff(string(want), string(got)); d != "" {
-				t.Fatalf("unexpected (-want, +got):\n%s", d)
+		// Mean reciprocal rank
+		mrr := float64(0)
+		for _, rank := range ranks {
+			if rank > 0 {
+				mrr += 1 / float64(rank)
 			}
-		})
+		}
+		mrr /= float64(len(ranks))
+		printf("mrr: %f\n", mrr)
+
+		assertGolden(t, "rank_stats", gotBuf.Bytes())
+	})
+}
+
+func assertGolden(t *testing.T, name string, got []byte) {
+	t.Helper()
+
+	wantPath := filepath.Join("testdata", name+".txt")
+	if *update {
+		if err := os.WriteFile(wantPath, got, 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	want, err := os.ReadFile(wantPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if d := cmp.Diff(string(want), string(got)); d != "" {
+		t.Fatalf("unexpected (-want, +got):\n%s", d)
 	}
 }
 
+type rankingQuery struct {
+	Query  string
+	Target string
+}
+
 var tarballCache = "/tmp/zoekt-test-ranking-tarballs-" + os.Getenv("USER")
 
 func indexURL(indexDir, u string) error {
@@ -156,7 +207,7 @@ func indexURL(indexDir, u string) error {
 	// TODO scip
 	// languageMap := make(ctags.LanguageMap)
 	// for _, lang := range []string{"kotlin", "rust", "ruby", "go", "python", "javascript", "c_sharp", "scala", "typescript", "zig"} {
-	// 	languageMap[lang] = ctags.ScipCTags
+	//	languageMap[lang] = ctags.ScipCTags
 	// }
 
 	err := archive.Index(opts, build.Options{
@@ -203,13 +254,22 @@ const (
 	fileMatchesPerSearch = 6
 )
 
-func marshalMatches(w io.Writer, queryStr string, q query.Q, files []zoekt.FileMatch) {
-	_, _ = fmt.Fprintf(w, "queryString: %s\n", queryStr)
-	_, _ = fmt.Fprintf(w, "query: %s\n\n", q)
+func docName(f zoekt.FileMatch) string {
+	return f.Repository + "/" + f.FileName
+}
+
+func marshalMatches(w io.Writer, rq rankingQuery, q query.Q, files []zoekt.FileMatch) {
+	_, _ = fmt.Fprintf(w, "queryString: %s\n", rq.Query)
+	_, _ = fmt.Fprintf(w, "query: %s\n", q)
+	_, _ = fmt.Fprintf(w, "targetRank: %d\n\n", targetRank(rq, files))
 
 	files, hiddenFiles := splitAtIndex(files, fileMatchesPerSearch)
 	for _, f := range files {
-		_, _ = fmt.Fprintf(w, "%s/%s%s\n", f.Repository, f.FileName, addTabIfNonEmpty(f.Debug))
+		doc := docName(f)
+		if doc == rq.Target {
+			doc = "**" + doc + "**"
+		}
+		_, _ = fmt.Fprintf(w, "%s%s\n", doc, addTabIfNonEmpty(f.Debug))
 
 		chunks, hidden := splitAtIndex(f.ChunkMatches, chunkMatchesPerFile)
 
@@ -228,6 +288,15 @@ func marshalMatches(w io.Writer, queryStr string, q query.Q, files []zoekt.FileM
 	}
 }
 
+func targetRank(rq rankingQuery, files []zoekt.FileMatch) int {
+	for i, f := range files {
+		if docName(f) == rq.Target {
+			return i + 1
+		}
+	}
+	return -1
+}
+
 func splitAtIndex[E any](s []E, idx int) ([]E, []E) {
 	if idx < len(s) {
 		return s[:idx], s[idx:]

internal/e2e/testdata/Get_databaseuser.txt

@@ -1,5 +1,6 @@
 queryString: Get database/user
 query: (and case_substr:"Get" substr:"database/user")
+targetRank: 3
 
 github.com/sourcegraph/sourcegraph/internal/database/user_emails.go
 161:func (s *userEmailsStore) Get(ctx context.Context, userID int32, email string) (emailCanonicalCase string, verified bool, err error) {
@@ -13,7 +14,7 @@ github.com/sourcegraph/sourcegraph/internal/database/user_roles.go
 365:func (r *userRoleStore) GetByRoleID(ctx context.Context, opts GetUserRoleOpts) ([]*types.UserRole, error) {
 hidden 8 more line matches
 
-github.com/sourcegraph/sourcegraph/internal/database/users.go
+**github.com/sourcegraph/sourcegraph/internal/database/users.go**
 940:func (u *userStore) GetByID(ctx context.Context, id int32) (*types.User, error) {
 947:func (u *userStore) GetByVerifiedEmail(ctx context.Context, email string) (*types.User, error) {
 951:func (u *userStore) GetByUsername(ctx context.Context, username string) (*types.User, error) {

internal/e2e/testdata/InternalDoer.txt

@@ -1,7 +1,8 @@
 queryString: InternalDoer
 query: case_substr:"InternalDoer"
+targetRank: 1
 
-github.com/sourcegraph/sourcegraph/internal/httpcli/client.go
+**github.com/sourcegraph/sourcegraph/internal/httpcli/client.go**
 217:var InternalDoer, _ = InternalClientFactory.Doer()
 215:// InternalDoer is a shared client for internal communication. This is a
 

internal/e2e/testdata/Repository_metadata_Write_rbac.txt

@@ -1,5 +1,6 @@
 queryString: Repository metadata Write rbac
 query: (and case_substr:"Repository" substr:"metadata" case_substr:"Write" substr:"rbac")
+targetRank: -1
 
 github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/repository_metadata.go
 54:func (r *schemaResolver) AddRepoMetadata(ctx context.Context, args struct {

internal/e2e/testdata/bufio_buffer.txt

@@ -1,13 +1,14 @@
 queryString: bufio buffer
 query: (and substr:"bufio" substr:"buffer")
+targetRank: 2
 
 github.com/golang/go/src/bytes/buffer.go
 20:type Buffer struct {
 60:func (b *Buffer) AvailableBuffer() []byte { return b.buf[len(b.buf):] }
 472:func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }
 hidden 108 more line matches
 
-github.com/golang/go/src/bufio/scan.go
+**github.com/golang/go/src/bufio/scan.go**
 267:func (s *Scanner) Buffer(buf []byte, max int) {
 5:package bufio
 25:// large to fit in the buffer. When a scan stops, the reader may have

internal/e2e/testdata/bytes_buffer.txt

@@ -1,7 +1,8 @@
 queryString: bytes buffer
 query: (and substr:"bytes" substr:"buffer")
+targetRank: 1
 
-github.com/golang/go/src/bytes/buffer.go
+**github.com/golang/go/src/bytes/buffer.go**
 20:type Buffer struct {
 54:func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
 5:package bytes

internal/e2e/testdata/generate_unit_test.txt

@@ -1,5 +1,6 @@
 queryString: generate unit test
 query: (and substr:"generate" substr:"unit" substr:"test")
+targetRank: 11
 
 github.com/sourcegraph/sourcegraph/cmd/frontend/internal/insights/resolvers/insight_series_resolver.go
 300:func (j *seriesResolverGenerator) Generate(ctx context.Context, series types.InsightViewSeries, baseResolver baseInsightResolver, filters types.InsightViewFilters, options types.SeriesDisplayOptions) ([]graphqlbackend.InsightSeriesResolver, error) {

internal/e2e/testdata/graphql_type_User.txt

@@ -1,7 +1,8 @@
 queryString: graphql type User
 query: (and substr:"graphql" substr:"type" case_substr:"User")
+targetRank: 1
 
-github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql
+**github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql**
 6376:type User implements Node & SettingsSubject & Namespace {
 3862:        type: GitRefType
 5037:    type: GitRefType!

internal/e2e/testdata/r_cody_sourcegraph_url.txt

@@ -1,7 +1,8 @@
 queryString: r:cody sourcegraph url
 query: (and repo:cody substr:"sourcegraph" substr:"url")
+targetRank: 1
 
-github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts
+**github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts**
 611:        const url = buildGraphQLUrl({ request: query, baseUrl: this.config.serverEndpoint })
 626:        const url = buildGraphQLUrl({ request: query, baseUrl: this.dotcomUrl.href })
 641:        const url = 'http://localhost:49300/.api/testLogging'

internal/e2e/testdata/rank_stats.txt

@@ -0,0 +1,4 @@
+queries: 9
+recall@1: 5 (56%)
+recall@5: 7 (78%)
+mrr: 0.658249

internal/e2e/testdata/test_server.txt

@@ -1,7 +1,8 @@
 queryString: test server
 query: (and substr:"test" substr:"server")
+targetRank: 1
 
-github.com/golang/go/src/net/http/httptest/server.go
+**github.com/golang/go/src/net/http/httptest/server.go**
 26:type Server struct {
 105:func NewServer(handler http.Handler) *Server {
 117:func NewUnstartedServer(handler http.Handler) *Server {