From 9dad685ca2112be542a0514b506f8dcb415a037f Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Tue, 14 Nov 2023 22:59:13 +0200 Subject: [PATCH] archive: e2e test for ranking against sourcegraph repo This is an initial framework for having golden file results for search results against a real repository. At first we have only added one query and one repository, but it should be straightforward to grow this list further. The golden files we write to disk are a summary of results with debug information. This matches how we have been using the zoekt CLI tool on the keyword branch during our ranking work. Test Plan: go test --- cmd/zoekt-archive-index/e2e_rank_test.go | 221 ++++++++++++++++++ .../testdata/graphql_type_User.txt | 40 ++++ 2 files changed, 261 insertions(+) create mode 100644 cmd/zoekt-archive-index/e2e_rank_test.go create mode 100644 cmd/zoekt-archive-index/testdata/graphql_type_User.txt diff --git a/cmd/zoekt-archive-index/e2e_rank_test.go b/cmd/zoekt-archive-index/e2e_rank_test.go new file mode 100644 index 000000000..252066bb9 --- /dev/null +++ b/cmd/zoekt-archive-index/e2e_rank_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/sourcegraph/zoekt" + "github.com/sourcegraph/zoekt/build" + "github.com/sourcegraph/zoekt/query" + "github.com/sourcegraph/zoekt/shards" +) + +var update = flag.Bool("update", false, "update golden file") + +func TestRanking(t *testing.T) { + requireCTags(t) + + archiveURLs := []string{ + "https://github.com/sourcegraph/sourcegraph/tree/v5.2.2", + } + queries := []string{ + "graphql type User", + } + + indexDir := t.TempDir() + + for _, u := range archiveURLs { + if err := indexURL(indexDir, u); err != nil { + t.Fatal(err) + } + } + + ss, err := shards.NewDirectorySearcher(indexDir) + if err != nil { + t.Fatalf("NewDirectorySearcher(%s): %v", indexDir, err) + } + defer ss.Close() + + for _, queryStr := range queries { + // normalise queryStr for writing to fs + name := strings.Map(func(r rune) rune { + if strings.ContainsRune(" :", r) { + return '_' + } + if '0' <= r && r <= '9' || + 'a' <= r && r <= 'z' || + 'A' <= r && r <= 'Z' { + return r + } + return -1 + }, queryStr) + + t.Run(name, func(t *testing.T) { + q, err := query.Parse(queryStr) + if err != nil { + t.Fatal(err) + } + + sOpts := zoekt.SearchOptions{ + DebugScore: true, + } + result, err := ss.Search(context.Background(), q, &sOpts) + if err != nil { + t.Fatal(err) + } + + var gotBuf bytes.Buffer + marshalMatches(&gotBuf, queryStr, q, result.Files) + got := gotBuf.Bytes() + + wantPath := filepath.Join("testdata", name+".txt") + if *update { + if err := os.WriteFile(wantPath, got, 0600); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(wantPath) + if err != nil { + t.Fatal(err) + } + + if d := cmp.Diff(string(want), string(got)); d != "" { + t.Fatalf("unexpected (-want, +got):\n%s", d) + } + }) + } +} + +var tarballCache = "/tmp/zoekt-test-ranking-tarballs-" + os.Getenv("USER") + +func indexURL(indexDir, u string) error { + if err := os.MkdirAll(tarballCache, 0700); err != nil { + return err + } + + opts := Options{ + Archive: u, + } + opts.SetDefaults() // sets metadata like Name and the codeload URL + u = opts.Archive + + // update Archive location to cached location + cacheBase := fmt.Sprintf("%s-%s%s.tar.gz", url.QueryEscape(opts.Name), opts.Branch, opts.Commit) // assume .tar.gz + path := filepath.Join(tarballCache, cacheBase) + opts.Archive = path + + if _, err := os.Stat(path); os.IsNotExist(err) { + if err := download(u, path); err != nil { + return err + } + } + + // TODO scip + // languageMap := make(ctags.LanguageMap) + // for _, lang := range []string{"kotlin", "rust", "ruby", "go", "python", "javascript", "c_sharp", "scala", "typescript", "zig"} { + // languageMap[lang] = ctags.ScipCTags + // } + + err := do(opts, build.Options{ + IndexDir: indexDir, + CTagsMustSucceed: true, + }) + if err != nil { + return fmt.Errorf("failed to index %s: %w", opts.Archive, err) + } + + return nil +} + +func download(url, dst string) error { + tmpPath := dst + ".part" + + rc, err := openReader(url) + if err != nil { + return err + } + defer rc.Close() + + f, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(f, rc) + if err != nil { + return err + } + + err = f.Close() + if err != nil { + return err + } + + return os.Rename(tmpPath, dst) +} + +const ( + lineMatchesPerFile = 3 + fileMatchesPerSearch = 6 +) + +func marshalMatches(w io.Writer, queryStr string, q query.Q, files []zoekt.FileMatch) { + _, _ = fmt.Fprintf(w, "queryString: %s\n", queryStr) + _, _ = fmt.Fprintf(w, "query: %s\n\n", q) + + files, hiddenFiles := splitAtIndex(files, fileMatchesPerSearch) + for _, f := range files { + _, _ = fmt.Fprintf(w, "%s/%s\t%s\n", f.Repository, f.FileName, f.Debug) + + lines, hidden := splitAtIndex(f.LineMatches, lineMatchesPerFile) + + for _, m := range lines { + _, _ = fmt.Fprintf(w, "%d:%s\t%s\n", m.LineNumber, m.Line, m.DebugScore) + } + + if len(hidden) > 0 { + _, _ = fmt.Fprintf(w, "hidden %d more line matches\n", len(hidden)) + } + _, _ = fmt.Fprintln(w) + } + + if len(hiddenFiles) > 0 { + fmt.Fprintf(w, "hidden %d more file matches\n", len(hiddenFiles)) + } +} + +func splitAtIndex[E any](s []E, idx int) ([]E, []E) { + if idx < len(s) { + return s[:idx], s[idx:] + } + return s, nil +} + +func requireCTags(tb testing.TB) { + tb.Helper() + + if os.Getenv("CTAGS_COMMAND") != "" { + return + } + if _, err := exec.LookPath("universal-ctags"); err == nil { + return + } + + // On CI we require ctags to be available. Otherwise we skip + if os.Getenv("CI") != "" { + tb.Fatal("universal-ctags is missing") + } else { + tb.Skip("universal-ctags is missing") + } +} diff --git a/cmd/zoekt-archive-index/testdata/graphql_type_User.txt b/cmd/zoekt-archive-index/testdata/graphql_type_User.txt new file mode 100644 index 000000000..4c3ca025d --- /dev/null +++ b/cmd/zoekt-archive-index/testdata/graphql_type_User.txt @@ -0,0 +1,40 @@ +queryString: graphql type User +query: (and substr:"graphql" substr:"type" case_substr:"User") + +github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql score:8807.21 <- atom(4):300.00, fragment:8500.00, doc-order:7.21 +6376:type User implements Node & SettingsSubject & Namespace { score:8500.00 <- WordMatch:500.00, Symbol:7000.00, kind:GraphQL:type:1000.00 +3862: type: GitRefType score:8050.00 <- WordMatch:500.00, Symbol:7000.00, kind:GraphQL:field:550.00 +5037: type: GitRefType! score:8050.00 <- WordMatch:500.00, Symbol:7000.00, kind:GraphQL:field:550.00 +hidden 460 more line matches + +github.com/sourcegraph/sourcegraph/internal/types/types.go score:8759.73 <- atom(4):300.00, fragment:8450.00, doc-order:9.73 +850:type User struct { score:8450.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:struct:950.00 +1372: Type *SearchCountStatistics score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +1766: Type string score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +hidden 234 more line matches + +github.com/sourcegraph/sourcegraph/client/web/src/enterprise/insights/core/backend/gql-backend/methods/get-dashboard-owners.ts score:8269.38 <- atom(3):266.67, fragment:8000.00, doc-order:2.71 +22: type: InsightsDashboardOwnerType.Global, score:8000.00 <- WordMatch:500.00, Symbol:7000.00, kind:TypeScript:constant:500.00 +32: type: InsightsDashboardOwnerType.Personal, score:8000.00 <- WordMatch:500.00, Symbol:7000.00, kind:TypeScript:constant:500.00 +18: const { currentUser, site } = data score:6500.00 <- WordMatch:500.00, EdgeSymbol:5500.00, kind:TypeScript:constant:500.00 +hidden 8 more line matches + +github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/apitest/types.go score:8751.64 <- atom(4):300.00, fragment:8450.00, doc-order:1.64 +47:type User struct { score:8450.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:struct:950.00 +9: Typename string `json:"__typename"` score:6300.00 <- PartialWordMatch:50.00, EdgeSymbol:5500.00, kind:Go:member:750.00 +32: Typename string `json:"__typename"` score:6300.00 <- PartialWordMatch:50.00, EdgeSymbol:5500.00, kind:Go:member:750.00 +hidden 11 more line matches + +github.com/sourcegraph/sourcegraph/cmd/frontend/internal/batches/resolvers/apitest/types.go score:8751.15 <- atom(4):300.00, fragment:8450.00, doc-order:1.15 +52:type User struct { score:8450.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:struct:950.00 +364: User *User score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +393: Type string score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +hidden 68 more line matches + +github.com/sourcegraph/sourcegraph/internal/extsvc/github/common.go score:8725.50 <- atom(3):266.67, fragment:8450.00, doc-order:8.84 +2030:type User struct { score:8450.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:struct:950.00 +66: User *Actor `json:"User,omitempty"` score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +527: Type string score:8250.00 <- WordMatch:500.00, Symbol:7000.00, kind:Go:member:750.00 +hidden 136 more line matches + +hidden 743 more file matches