Symbol search: support content-based lang detection

cmd/symbols/internal/api/handler_test.go

@@ -43,14 +43,24 @@ func TestHandler(t *testing.T) {
 		pathToEntries := map[string][]*ctags.Entry{
 			"a.js": {
 				{
-					Name: "x",
-					Path: "a.js",
-					Line: 1, // ctags line numbers are 1-based
+					Name:     "x",
+					Path:     "a.js",
+					Language: "JavaScript",
+					Line:     1, // ctags line numbers are 1-based
 				},
 				{
-					Name: "y",
-					Path: "a.js",
-					Line: 2,
+					Name:     "y",
+					Path:     "a.js",
+					Language: "JavaScript",
+					Line:     2,
+				},
+			},
+			".zshrc": {
+				{
+					Name:     "z",
+					Path:     ".zshrc",
+					Language: "Zsh",
+					Line:     1,
 				},
 			},
 		}
@@ -62,7 +72,8 @@ func TestHandler(t *testing.T) {
 	}
 
 	files := map[string]string{
-		"a.js": "var x = 1\nvar y = 2",
+		"a.js":   "var x = 1\nvar y = 2",
+		".zshrc": "z=42",
 	}
 	gitserverClient := NewMockGitserverClient()
 	gitserverClient.FetchTarFunc.SetDefaultHook(gitserver.CreateTestFetchTarFunc(files))
@@ -83,15 +94,16 @@ func TestHandler(t *testing.T) {
 		GRPCConnectionCache: connectionCache,
 	}
 
-	x := result.Symbol{Name: "x", Path: "a.js", Line: 0, Character: 4}
-	y := result.Symbol{Name: "y", Path: "a.js", Line: 1, Character: 4}
+	x := result.Symbol{Name: "x", Path: "a.js", Language: "JavaScript", Line: 0, Character: 4}
+	y := result.Symbol{Name: "y", Path: "a.js", Language: "JavaScript", Line: 1, Character: 4}
+	z := result.Symbol{Name: "z", Path: ".zshrc", Language: "Zsh", Line: 0, Character: 0}
 
 	testCases := map[string]struct {
 		args     search.SymbolsParameters
 		expected result.Symbols
 	}{
 		"simple": {
-			args:     search.SymbolsParameters{First: 10},
+			args:     search.SymbolsParameters{IncludePatterns: []string{"^a.js$"}, First: 10},
 			expected: []result.Symbol{x, y},
 		},
 		"onematch": {
@@ -128,7 +140,19 @@ func TestHandler(t *testing.T) {
 		},
 		"exclude": {
 			args:     search.SymbolsParameters{ExcludePattern: "a.js", IsCaseSensitive: true, First: 10},
-			expected: nil,
+			expected: []result.Symbol{z},
+		},
+		"include lang filters": {
+			args:     search.SymbolsParameters{Query: ".*", IncludeLangs: []string{"Javascript"}, IsCaseSensitive: true, First: 10},
+			expected: []result.Symbol{x, y},
+		},
+		"include lang filters with ctags conversion": {
+			args:     search.SymbolsParameters{Query: ".*", IncludeLangs: []string{"Shell"}, IsCaseSensitive: true, First: 10},
+			expected: []result.Symbol{z},
+		},
+		"exclude lang filters": {
+			args:     search.SymbolsParameters{Query: ".*", ExcludeLangs: []string{"Javascript"}, IsCaseSensitive: true, First: 10},
+			expected: []result.Symbol{z},
 		},
 	}
 

cmd/symbols/internal/api/search_sqlite.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"context"
-	"strings"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -34,8 +33,10 @@ func MakeSqliteSearchFunc(observationCtx *observation.Context, cachedDatabaseWri
 			attribute.Bool("isRegExp", args.IsRegExp),
 			attribute.Bool("isCaseSensitive", args.IsCaseSensitive),
 			attribute.Int("numIncludePatterns", len(args.IncludePatterns)),
-			attribute.String("includePatterns", strings.Join(args.IncludePatterns, ":")),
+			attribute.StringSlice("includePatterns", args.IncludePatterns),
 			attribute.String("excludePattern", args.ExcludePattern),
+			attribute.StringSlice("includeLangs", args.IncludeLangs),
+			attribute.StringSlice("excludeLangs", args.ExcludeLangs),
 			attribute.Int("first", args.First),
 			attribute.Float64("timeoutSeconds", args.Timeout.Seconds()),
 		}})

cmd/symbols/internal/database/store/search.go

@@ -75,6 +75,13 @@ func makeSearchConditions(args search.SymbolsParameters) []*sqlf.Query {
 		conditions = append(conditions, makeSearchCondition("path", includePattern, args.IsCaseSensitive))
 	}
 
+	for _, includeLang := range args.IncludeLangs {
+		conditions = append(conditions, makeLangCondition(includeLang))
+	}
+	for _, excludeLang := range args.ExcludeLangs {
+		conditions = append(conditions, negate(makeLangCondition(excludeLang)))
+	}
+
 	filtered := conditions[:0]
 	for _, condition := range conditions {
 		if condition != nil {
@@ -120,6 +127,36 @@ func makeSearchCondition(column string, regex string, isCaseSensitive bool) *sql
 	return sqlf.Sprintf(column+" REGEXP %s", regex)
 }
 
+func makeLangCondition(lang string) *sqlf.Query {
+	// We need to convert the lang since language filters use normalized enry names, but
+	// the database stores ctags languages.
+	ctagsLangs := convertEnryToCTagsLangs(lang)
+	var queries []*sqlf.Query
+	for _, ctagsLang := range ctagsLangs {
+		queries = append(queries, sqlf.Sprintf("%s", ctagsLang))
+	}
+	return sqlf.Sprintf("lower(language) IN (%s) ", sqlf.Join(queries, ", "))
+}
+
+// convertEnryToCTagsLangs performs a best-effort mapping from normalized go-enry
+// language names to ctags names. This list is currently not exhaustive.
+func convertEnryToCTagsLangs(enryLang string) []string {
+	lower := strings.ToLower(enryLang)
+	switch lower {
+	case "protocol buffer":
+		return []string{"protobuf"}
+	case "objective-c":
+		return []string{"objectivec"}
+	case "assembly":
+		return []string{"asm"}
+	case "raku":
+		return []string{"perl6"}
+	case "shell":
+		return []string{"sh", "zsh"}
+	}
+	return []string{lower}
+}
+
 // isLiteralEquality returns true if the given regex matches literal strings exactly.
 // If so, this function returns true along with the literal search query. If not, this
 // function returns false.

internal/rockskip/BUILD.bazel

@@ -23,6 +23,7 @@ go_library(
         "//internal/database/dbutil",
         "//internal/gitserver/gitdomain",
         "//internal/search",
+        "//internal/search/query",
         "//internal/search/result",
         "//lib/errors",
         "@com_github_amit7itz_goset//:goset",

internal/rockskip/search.go

@@ -19,6 +19,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/database"
 	"github.com/sourcegraph/sourcegraph/internal/search"
+	"github.com/sourcegraph/sourcegraph/internal/search/query"
 	"github.com/sourcegraph/sourcegraph/internal/search/result"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
@@ -437,6 +438,16 @@ func convertSearchArgsToSqlQuery(args search.SymbolsParameters) *sqlf.Query {
 	// ExcludePaths
 	conjunctOrNils = append(conjunctOrNils, negate(regexMatch(pathConditions, args.ExcludePattern, args.IsCaseSensitive)))
 
+	// Rockskip doesn't store the file's language, so we convert the language filters into path filters as a
+	// best effort approximation. We ignore the search's case-sensitivity, since it doesn't apply to these filters.
+	for _, includeLang := range args.IncludeLangs {
+		conjunctOrNils = append(conjunctOrNils, regexMatch(pathConditions, query.LangToFileRegexp(includeLang), false))
+	}
+
+	for _, excludeLang := range args.ExcludeLangs {
+		conjunctOrNils = append(conjunctOrNils, negate(regexMatch(pathConditions, query.LangToFileRegexp(excludeLang), false)))
+	}
+
 	// Drop nils
 	conjuncts := []*sqlf.Query{}
 	for _, condition := range conjunctOrNils {

internal/rockskip/server_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/sourcegraph/go-ctags"
+
 	"github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher"
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
@@ -118,7 +119,11 @@ func TestIndex(t *testing.T) {
 	verifyBlobs := func() {
 		repo := "somerepo"
 		commit := getHead()
-		args := search.SymbolsParameters{Repo: api.RepoName(repo), CommitID: api.CommitID(commit), Query: ""}
+		args := search.SymbolsParameters{
+			Repo:         api.RepoName(repo),
+			CommitID:     api.CommitID(commit),
+			Query:        "",
+			IncludeLangs: []string{"Text"}}
 		symbols, err := service.Search(context.Background(), args)
 		fatalIfError(err, "Search")
 
@@ -133,7 +138,10 @@ func TestIndex(t *testing.T) {
 		}
 		wantPaths := []string{}
 		for wantPath := range state {
-			wantPaths = append(wantPaths, wantPath)
+			// We only want .txt files since we're filtering by lang: text
+			if strings.Contains(wantPath, ".txt") {
+				wantPaths = append(wantPaths, wantPath)
+			}
 		}
 		sort.Strings(gotPaths)
 		sort.Strings(wantPaths)
@@ -179,6 +187,9 @@ func TestIndex(t *testing.T) {
 	add("c.txt", "sym1\nsym2")
 	commit("add another file with 2 symbols")
 
+	add("a.java", "sym1\nsym2")
+	commit("System.out.println(\"hello, world!\"")
+
 	add("a.txt", "sym1\nsym2")
 	commit("add a symbol to a.txt")
 

internal/search/job/jobutil/job.go

@@ -373,7 +373,7 @@ func NewFlatJob(searchInputs *search.Inputs, f query.Flat) (job.Job, error) {
 		if resultTypes.Has(result.TypeSymbol) {
 			// Create Symbol Search jobs over repo set.
 			if !skipRepoSubsetSearch {
-				request, err := toSymbolSearchRequest(f)
+				request, err := toSymbolSearchRequest(f, searchInputs.Features)
 				if err != nil {
 					return nil, err
 				}
@@ -629,7 +629,7 @@ func mapSlice(values []string, f func(string) string) []string {
 	return res
 }
 
-func toSymbolSearchRequest(f query.Flat) (*searcher.SymbolSearchRequest, error) {
+func toSymbolSearchRequest(f query.Flat, feat *search.Features) (*searcher.SymbolSearchRequest, error) {
 	if f.Pattern != nil && f.Pattern.Negated {
 		return nil, &query.UnsupportedError{
 			Msg: "symbol search does not support negation.",
@@ -640,17 +640,29 @@ func toSymbolSearchRequest(f query.Flat) (*searcher.SymbolSearchRequest, error)
 	// assumes that a literal pattern is an escaped regular expression.
 	regexpPattern := f.ToBasic().PatternString()
 
+	// Handle file: and -file: filters.
 	filesInclude, filesExclude := f.IncludeExcludeValues(query.FieldFile)
-	langInclude, langExclude := f.IncludeExcludeValues(query.FieldLang)
 
-	filesInclude = append(filesInclude, mapSlice(langInclude, query.LangToFileRegexp)...)
-	filesExclude = append(filesExclude, mapSlice(langExclude, query.LangToFileRegexp)...)
+	// Handle lang: and -lang: filters.
+	langAliasInclude, langAliasExclude := f.IncludeExcludeValues(query.FieldLang)
+	var langInclude, langExclude []string
+	if feat.ContentBasedLangFilters {
+		langInclude = toLangFilters(langAliasInclude)
+		langExclude = toLangFilters(langAliasExclude)
+	} else {
+		// If the 'search-content-based-lang-detection' feature is disabled, then we convert the filters
+		// to file path regexes and do not pass any explicit language filters to the backend.
+		filesInclude = append(filesInclude, mapSlice(langAliasInclude, query.LangToFileRegexp)...)
+		filesExclude = append(filesExclude, mapSlice(langAliasExclude, query.LangToFileRegexp)...)
+	}
 
 	return &searcher.SymbolSearchRequest{
 		RegexpPattern:   regexpPattern,
 		IsCaseSensitive: f.IsCaseSensitive(),
 		IncludePatterns: filesInclude,
 		ExcludePattern:  query.UnionRegExps(filesExclude),
+		IncludeLangs:    langInclude,
+		ExcludeLangs:    langExclude,
 	}, nil
 }
 

internal/search/job/jobutil/job_test.go

@@ -1059,41 +1059,46 @@ func TestToSymbolSearchRequest(t *testing.T) {
 	cases := []struct {
 		input   string
 		output  autogold.Value
+		feat    search.Features
 		wantErr bool
 	}{{
 		input:  `repo:go-diff patterntype:literal HunkNoChunksize select:symbol  file:^README\.md `,
-		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":["^README\\.md"],"ExcludePattern":""}`),
+		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":["^README\\.md"],"ExcludePattern":"","IncludeLangs":null,"ExcludeLangs":null}`),
 	}, {
 		input:  `repo:go-diff patterntype:literal type:symbol HunkNoChunksize select:symbol -file:^README\.md `,
-		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":null,"ExcludePattern":"^README\\.md"}`),
+		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":null,"ExcludePattern":"^README\\.md","IncludeLangs":null,"ExcludeLangs":null}`),
 	}, {
 		input:  `repo:go-diff type:symbol`,
-		output: autogold.Expect(`{"RegexpPattern":"","IsCaseSensitive":false,"IncludePatterns":null,"ExcludePattern":""}`),
+		output: autogold.Expect(`{"RegexpPattern":"","IsCaseSensitive":false,"IncludePatterns":null,"ExcludePattern":"","IncludeLangs":null,"ExcludeLangs":null}`),
 	}, {
 		input:   `type:symbol NOT option`,
 		output:  autogold.Expect("null"),
 		wantErr: true,
+	}, {
+		input:  `repo:go-diff type:symbol HunkNoChunksize lang:Julia -lang:R`,
+		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":["\\.jl$"],"ExcludePattern":"(?:\\.r$)|(?:\\.rd$)|(?:\\.rsx$)|(?:(^|/)\\.Rprofile$)|(?:(^|/)expr-dist$)","IncludeLangs":null,"ExcludeLangs":null}`),
+	}, {
+		input:  `repo:go-diff type:symbol HunkNoChunksize lang:Julia -lang:R`,
+		feat:   search.Features{ContentBasedLangFilters: true},
+		output: autogold.Expect(`{"RegexpPattern":"HunkNoChunksize","IsCaseSensitive":false,"IncludePatterns":null,"ExcludePattern":"","IncludeLangs":["Julia"],"ExcludeLangs":["R"]}`),
 	}}
 
-	createRequest := func(input string) (*searcher.SymbolSearchRequest, error) {
-		plan, err := query.Pipeline(query.Init(input, query.SearchTypeLiteral))
-		if err != nil {
-			t.Fatal(err)
-		}
+	for _, tc := range cases {
+		t.Run(tc.input, func(t *testing.T) {
+			plan, err := query.Pipeline(query.Init(tc.input, query.SearchTypeLiteral))
+			if err != nil {
+				t.Fatal(err)
+			}
 
-		b := plan[0]
-		var pattern *query.Pattern
-		if p, ok := b.Pattern.(query.Pattern); ok {
-			pattern = &p
-		}
+			b := plan[0]
+			var pattern *query.Pattern
+			if p, ok := b.Pattern.(query.Pattern); ok {
+				pattern = &p
+			}
 
-		f := query.Flat{Parameters: b.Parameters, Pattern: pattern}
-		return toSymbolSearchRequest(f)
-	}
+			f := query.Flat{Parameters: b.Parameters, Pattern: pattern}
+			r, err := toSymbolSearchRequest(f, &tc.feat)
 
-	for _, tc := range cases {
-		t.Run(tc.input, func(t *testing.T) {
-			r, err := createRequest(tc.input)
 			if (err != nil) != tc.wantErr {
 				t.Fatalf("mismatch error = %v, wantErr %v", err, tc.wantErr)
 			}

internal/search/searcher/symbol_search_job.go

@@ -111,6 +111,8 @@ func searchInRepo(ctx context.Context, gitserverClient gitserver.Client, repoRev
 		IsRegExp:        true,
 		IncludePatterns: request.IncludePatterns,
 		ExcludePattern:  request.ExcludePattern,
+		IncludeLangs:    request.IncludeLangs,
+		ExcludeLangs:    request.ExcludeLangs,
 		// Ask for limit + 1 so we can detect whether there are more results than the limit.
 		First: limit + 1,
 	})
@@ -175,6 +177,8 @@ type SymbolSearchRequest struct {
 	IsCaseSensitive bool
 	IncludePatterns []string
 	ExcludePattern  string
+	IncludeLangs    []string
+	ExcludeLangs    []string
 }
 
 func (r *SymbolSearchRequest) Fields() []attribute.KeyValue {
@@ -194,5 +198,11 @@ func (r *SymbolSearchRequest) Fields() []attribute.KeyValue {
 	if r.ExcludePattern != "" {
 		add(attribute.String("excludePattern", r.ExcludePattern))
 	}
+	if len(r.IncludeLangs) > 0 {
+		add(attribute.StringSlice("includeLangs", r.IncludeLangs))
+	}
+	if len(r.ExcludeLangs) > 0 {
+		add(attribute.StringSlice("excludeLangs", r.ExcludeLangs))
+	}
 	return res
 }

internal/search/types.go

@@ -128,6 +128,10 @@ type SymbolsParameters struct {
 	// need to match to get included in the result
 	ExcludePattern string
 
+	// IncludeLangs and ExcludeLangs hold the language filters to apply.
+	IncludeLangs []string
+	ExcludeLangs []string
+
 	// First indicates that only the first n symbols should be returned.
 	First int