From 8619902b5d0d7ff40e6c5558aee085d0869f9050 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Sun, 21 Apr 2024 10:47:32 +0200 Subject: [PATCH 01/31] gitindex: include environ for git tests (#760) gitindex: include environ for git tests A recent change introduced GIT_CONFIG_GLOBAL and GIT_CONFIG_SYSTEM, but ran into the footgun of unsetting the rest of the environ when cmd.Env is non-empty. This for example broke tests for me since we didn't have my custom $PATH set. Instead we revert the change and make a much smaller change to just always set the relevant environment variables. This reverts commit ab1b8f09199ea7a731fd80876fc5b0f376ff6882. Test Plan: go test --- gitindex/index_test.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/gitindex/index_test.go b/gitindex/index_test.go index 1125a2039..a14f32fe3 100644 --- a/gitindex/index_test.go +++ b/gitindex/index_test.go @@ -526,16 +526,13 @@ func TestIndexDeltaBasic(t *testing.T) { repositoryDir := t.TempDir() // setup: initialize the repository and all of its branches - runGitScript := func(t *testing.T, dir, script string) { - runScript(t, dir, script, "GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM=") - } - runGitScript(t, repositoryDir, "git init -b master") - runGitScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com")) - runGitScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name")) + runScript(t, repositoryDir, "git init -b master") + runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com")) + runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name")) for _, b := range test.branches { - runGitScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b)) - runGitScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit")) + runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b)) + runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit")) } for _, step := range test.steps { @@ -545,7 +542,7 @@ func TestIndexDeltaBasic(t *testing.T) { hadChange := false - runGitScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b)) + runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b)) for _, d := range step.deletedDocuments[b] { hadChange = true @@ -557,7 +554,7 @@ func TestIndexDeltaBasic(t *testing.T) { t.Fatalf("deleting file %q: %s", d.Name, err) } - runGitScript(t, repositoryDir, fmt.Sprintf("git add %q", file)) + runScript(t, repositoryDir, fmt.Sprintf("git add %q", file)) } for _, d := range step.addedDocuments[b] { @@ -575,14 +572,14 @@ func TestIndexDeltaBasic(t *testing.T) { t.Fatalf("writing file %q: %s", d.Name, err) } - runGitScript(t, repositoryDir, fmt.Sprintf("git add %q", file)) + runScript(t, repositoryDir, fmt.Sprintf("git add %q", file)) } if !hadChange { continue } - runGitScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name)) + runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name)) } // setup: prepare indexOptions with given overrides @@ -755,7 +752,9 @@ func TestRepoPathRanks(t *testing.T) { } } -func runScript(t *testing.T, cwd string, script string, env ...string) { +func runScript(t *testing.T, cwd string, script string) { + t.Helper() + err := os.MkdirAll(cwd, 0o755) if err != nil { t.Fatalf("ensuring path %q exists: %s", cwd, err) @@ -763,7 +762,7 @@ func runScript(t *testing.T, cwd string, script string, env ...string) { cmd := exec.Command("sh", "-euxc", script) cmd.Dir = cwd - cmd.Env = env + cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("execution error: %v, output %s", err, out) From 55b7aeee2dbb24970a295d378a1cd297da7194e3 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Mon, 22 Apr 2024 16:54:27 +0200 Subject: [PATCH 02/31] docker: simplify apk add lines (#759) The images produced are no longer consumed directly by sourcegraph so we don't have security scanners running on them. Additionally we upgraded to the latest alpine now. Both combined means we can simplify the apk add lines. Test Plan: built locally docker build -t zoekt . docker build -t zoekt-indexserver . -f Dockerfile.indexserver docker build -t zoekt-webserver . -f Dockerfile.webserver --- Dockerfile | 6 ++---- Dockerfile.indexserver | 5 ++--- Dockerfile.webserver | 3 +-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91ecd39e7..b0a07a5ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,7 @@ RUN go install -ldflags "-X github.com/sourcegraph/zoekt.Version=$VERSION" ./cmd FROM rust:alpine3.19 AS rust-builder -RUN apk update --no-cache && apk upgrade --no-cache && \ - apk add --no-cache git wget musl-dev>=1.1.24-r10 build-base +RUN apk add --no-cache git wget musl-dev build-base RUN wget -qO- https://github.com/sourcegraph/sourcegraph/archive/0c8aa18eece45922a2b56dc0f94e21b1bb533e7d.tar.gz | tar xz && mv sourcegraph-* sourcegraph @@ -29,8 +28,7 @@ RUN cargo install --path sourcegraph/docker-images/syntax-highlighter --root /sy FROM alpine:3.19 AS zoekt -RUN apk update --no-cache && apk upgrade --no-cache && \ - apk add --no-cache git ca-certificates bind-tools tini jansson wget +RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget COPY install-ctags-alpine.sh . RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh diff --git a/Dockerfile.indexserver b/Dockerfile.indexserver index b4e9f08d3..38d81c65c 100644 --- a/Dockerfile.indexserver +++ b/Dockerfile.indexserver @@ -1,8 +1,7 @@ FROM alpine:3.19 -RUN apk update --no-cache && apk upgrade --no-cache && \ - apk add --no-cache ca-certificates bind-tools tini 'git>=2.38.5-r0' jansson && \ - apk add --upgrade --no-cache 'libcrypto1.1>=1.1.1n-r0' 'libssl1.1>=1.1.1n-r0' 'pcre2>=10.40-r0' 'e2fsprogs>=1.46.6-r0' +RUN apk add --no-cache ca-certificates bind-tools tini git jansson + # Run as non-root user sourcegraph. External volumes should be mounted under /data (which will be owned by sourcegraph). RUN mkdir -p /home/sourcegraph RUN addgroup -S sourcegraph && adduser -S -G sourcegraph -h /home/sourcegraph sourcegraph && mkdir -p /data && chown -R sourcegraph:sourcegraph /data diff --git a/Dockerfile.webserver b/Dockerfile.webserver index 0a477fc48..cccdfa83d 100644 --- a/Dockerfile.webserver +++ b/Dockerfile.webserver @@ -1,7 +1,6 @@ FROM alpine:3.19 -RUN apk update --no-cache && apk upgrade --no-cache && \ - apk add --no-cache ca-certificates bind-tools tini +RUN apk add --no-cache ca-certificates bind-tools tini # Run as non-root user sourcegraph. External volumes should be mounted under /data (which will be owned by sourcegraph). RUN mkdir -p /home/sourcegraph From 734f5b64f8ff1d5270aca195da54417f89c08232 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Tue, 23 Apr 2024 12:44:05 +0200 Subject: [PATCH 03/31] gomod: update deps for CVE-2023-45288 and CVE-2024-24786 (#766) After this update "trivy fs go.mod" has a clean report. go get golang.org/x/net@v0.23.0 google.golang.org/protobuf@v1.33.0 Test Plan: go test --- go.mod | 9 ++++----- go.sum | 22 ++++++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index cd2baa8db..116179420 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 - github.com/keegancsmith/rpc v1.3.0 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f github.com/opentracing/opentracing-go v1.2.0 github.com/peterbourgon/ff/v3 v3.3.2 @@ -48,12 +47,12 @@ require ( go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.5.2 - golang.org/x/net v0.20.0 + golang.org/x/net v0.23.0 golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.18.0 google.golang.org/grpc v1.61.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 ) require ( @@ -124,7 +123,7 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.18.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index 812d6bf44..cfa9b329c 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1: github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk= -github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -377,8 +375,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -418,8 +416,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= @@ -465,16 +463,16 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -545,8 +543,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 6df055493b1600051e467b79d296fa292f658d3f Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Tue, 23 Apr 2024 12:52:11 +0200 Subject: [PATCH 04/31] gitindex: interpret SSH URLs (#764) This allows indexing repos that are used for development and therefore have an SSH 'origin' URL. --- gitindex/index.go | 16 ++++++++++++++++ gitindex/index_test.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/gitindex/index.go b/gitindex/index.go index d3a5e3c64..eb2588b3c 100644 --- a/gitindex/index.go +++ b/gitindex/index.go @@ -27,6 +27,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -111,6 +112,11 @@ func FindGitRepos(dir string) ([]string, error) { // setTemplates fills in URL templates for known git hosting // sites. func setTemplates(repo *zoekt.Repository, u *url.URL, typ string) error { + if u.Scheme == "ssh+git" { + u.Scheme = "https" + u.User = nil + } + repo.URL = u.String() switch typ { case "gitiles": @@ -192,6 +198,8 @@ func configLookupRemoteURL(cfg *config.Config, key string) string { return rc.URLs[0] } +var sshRelativeURLRegexp = regexp.MustCompile(`^([^@]+)@([^:]+):(.*)$`) + func setTemplatesFromConfig(desc *zoekt.Repository, repoDir string) error { repo, err := git.PlainOpen(repoDir) if err != nil { @@ -228,6 +236,14 @@ func setTemplatesFromConfig(desc *zoekt.Repository, repoDir string) error { if remoteURL == "" { return nil } + if sm := sshRelativeURLRegexp.FindStringSubmatch(remoteURL); sm != nil { + user := sm[1] + host := sm[2] + path := sm[3] + + remoteURL = fmt.Sprintf("ssh+git://%s@%s/%s", user, host, path) + } + u, err := url.Parse(remoteURL) if err != nil { return err diff --git a/gitindex/index_test.go b/gitindex/index_test.go index a14f32fe3..bf5993717 100644 --- a/gitindex/index_test.go +++ b/gitindex/index_test.go @@ -768,3 +768,19 @@ func runScript(t *testing.T, cwd string, script string) { t.Fatalf("execution error: %v, output %s", err, out) } } + +func TestSetTemplate(t *testing.T) { + repositoryDir := t.TempDir() + + // setup: initialize the repository and all of its branches + runScript(t, repositoryDir, "git init -b master") + runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git") + desc := zoekt.Repository{} + if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil { + t.Fatalf("setTemplatesFromConfig: %v", err) + } + + if got, want := desc.FileURLTemplate, "https://github.com/sourcegraph/zoekt/blob/{{.Version}}/{{.Path}}"; got != want { + t.Errorf("got %q, want %q", got, want) + } +} From 5ecbc14cac1cb641855e64dcf6e1ca6892b41ac5 Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Mon, 29 Apr 2024 11:23:39 +0200 Subject: [PATCH 05/31] zoekt-indexserver: prune branches on fetch (#769) issue: #768 --- cmd/zoekt-indexserver/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zoekt-indexserver/main.go b/cmd/zoekt-indexserver/main.go index 05140ea75..2107503a6 100644 --- a/cmd/zoekt-indexserver/main.go +++ b/cmd/zoekt-indexserver/main.go @@ -129,7 +129,7 @@ func periodicFetch(repoDir, indexDir string, opts *Options, pendingRepos chan<- // fetchGitRepo runs git-fetch, and returns true if there was an // update. func fetchGitRepo(dir string) bool { - cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin") + cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin", "--prune") output, err := cmd.CombinedOutput() if err != nil { From 5411e9b32eb5a822492cb167e905462af97d0469 Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Mon, 29 Apr 2024 13:12:42 +0200 Subject: [PATCH 06/31] zoekt-mirror-gerrit: allow to use reponame without host in the index (#771) The repo is indexed only with the reponame which is more user-friendly in the search when there is onely one host Filter string will be "r:my-repo" instead of "r:my-gerrit-sever.com/my-repo" --- cmd/zoekt-mirror-gerrit/main.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go index 853da1767..e593b62d9 100644 --- a/cmd/zoekt-mirror-gerrit/main.go +++ b/cmd/zoekt-mirror-gerrit/main.go @@ -26,6 +26,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strconv" "strings" @@ -73,9 +74,22 @@ func newLoggingClient() *http.Client { } } +const qualifiedRepoNameFormat = "qualified" +const projectRepoNameFormat = "project" + +var validRepoNameFormat = []string{qualifiedRepoNameFormat, projectRepoNameFormat} + +func validateRepoNameFormat(s string) { + if !slices.Contains(validRepoNameFormat, s) { + log.Fatal(fmt.Sprintf("repo-name-format must be one of %s", strings.Join(validRepoNameFormat, ", "))) + } +} + func main() { + dest := flag.String("dest", "", "destination directory") namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.") + repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", "))) excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") deleteRepos := flag.Bool("delete", false, "delete missing repos") httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.") @@ -85,6 +99,7 @@ func main() { if len(flag.Args()) < 1 { log.Fatal("must provide URL argument.") } + validateRepoNameFormat(*repoNameFormat) rootURL, err := url.Parse(flag.Arg(0)) if err != nil { @@ -162,8 +177,15 @@ func main() { } name := filepath.Join(cloneURL.Host, cloneURL.Path) + var zoektName string + switch *repoNameFormat { + case qualifiedRepoNameFormat: + zoektName = name + case projectRepoNameFormat: + zoektName = k + } config := map[string]string{ - "zoekt.name": name, + "zoekt.name": zoektName, "zoekt.gerrit-project": k, "zoekt.gerrit-host": rootURL.String(), "zoekt.archived": marshalBool(v.State == "READ_ONLY"), From 570757e20b467174defc40c67ac097cf81a72082 Mon Sep 17 00:00:00 2001 From: Matthew Manela Date: Mon, 29 Apr 2024 14:23:08 -0400 Subject: [PATCH 07/31] Honor regex flags for case-sensitivity (#767) * Honor regex flags for case-sensitivity * change * add test * pr comment --- .vscode/launch.json | 14 +++++++++++++ eval.go | 3 ++- eval_test.go | 49 +++++++++++++++++++++++++++++---------------- 3 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..dffa3c19e --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Process (from list)", + "type": "go", + "request": "attach", + "mode": "local" + }, + ] +} \ No newline at end of file diff --git a/eval.go b/eval.go index 7cbd3658f..cc0e61f27 100644 --- a/eval.go +++ b/eval.go @@ -641,7 +641,8 @@ func (d *indexData) regexpToMatchTreeRecursive(r *syntax.Regexp, minTextSize int case syntax.OpLiteral: s := string(r.Rune) if len(s) >= minTextSize { - mt, err := d.newSubstringMatchTree(&query.Substring{Pattern: s, FileName: fileName, CaseSensitive: caseSensitive}) + ignoreCase := syntax.FoldCase == (r.Flags & syntax.FoldCase) + mt, err := d.newSubstringMatchTree(&query.Substring{Pattern: s, FileName: fileName, CaseSensitive: !ignoreCase && caseSensitive}) return mt, true, !strings.Contains(s, "\n"), err } case syntax.OpCapture: diff --git a/eval_test.go b/eval_test.go index aeff6c27e..32cefdea0 100644 --- a/eval_test.go +++ b/eval_test.go @@ -59,24 +59,35 @@ func printRegexp(t *testing.T, r *syntax.Regexp, lvl int) { } } +func caseSensitiveSubstrMT(pattern string) matchTree { + d := &indexData{} + mt, _ := d.newSubstringMatchTree(&query.Substring{ + Pattern: pattern, + CaseSensitive: true, + }) + return mt +} + func substrMT(pattern string) matchTree { d := &indexData{} mt, _ := d.newSubstringMatchTree(&query.Substring{ - Pattern: pattern, + Pattern: pattern, + CaseSensitive: false, }) return mt } func TestRegexpParse(t *testing.T) { type testcase struct { - in string - query matchTree - isEquivalent bool + in string + query matchTree + isEquivalent bool + caseSensitive bool } cases := []testcase{ - {"(foo|)bar", substrMT("bar"), false}, - {"(foo|)", &bruteForceMatchTree{}, false}, + {"(foo|)bar", substrMT("bar"), false, false}, + {"(foo|)", &bruteForceMatchTree{}, false, false}, {"(foo|bar)baz.*bla", &andMatchTree{[]matchTree{ &orMatchTree{[]matchTree{ substrMT("foo"), @@ -84,32 +95,35 @@ func TestRegexpParse(t *testing.T) { }}, substrMT("baz"), substrMT("bla"), - }}, false}, + }}, false, false}, { "^[a-z](People)+barrabas$", &andMatchTree{[]matchTree{ substrMT("People"), substrMT("barrabas"), - }}, false, + }}, false, false, }, - {"foo", substrMT("foo"), true}, - {"^foo", substrMT("foo"), false}, - {"(foo) (bar)", &andMatchTree{[]matchTree{substrMT("foo"), substrMT("bar")}}, false}, + {"foo", substrMT("foo"), true, false}, + {"foo", caseSensitiveSubstrMT("foo"), true, true}, + {"(?i)foo", substrMT("FOO"), true, false}, + {"(?i)foo", substrMT("FOO"), true, true}, + {"^foo", substrMT("foo"), false, false}, + {"(foo) (bar)", &andMatchTree{[]matchTree{substrMT("foo"), substrMT("bar")}}, false, false}, {"(thread|needle|haystack)", &orMatchTree{[]matchTree{ substrMT("thread"), substrMT("needle"), substrMT("haystack"), - }}, true}, + }}, true, false}, {"(foo)(?-s:.)*?(bar)", &andLineMatchTree{andMatchTree{[]matchTree{ substrMT("foo"), substrMT("bar"), - }}}, false}, + }}}, false, false}, {"(foo)(?-s:.)*?[[:space:]](?-s:.)*?(bar)", &andMatchTree{[]matchTree{ substrMT("foo"), substrMT("bar"), - }}, false}, - {"(foo){2,}", substrMT("foo"), false}, - {"(...)(...)", &bruteForceMatchTree{}, false}, + }}, false, false}, + {"(foo){2,}", substrMT("foo"), false, false}, + {"(...)(...)", &bruteForceMatchTree{}, false, false}, } for _, c := range cases { @@ -120,7 +134,8 @@ func TestRegexpParse(t *testing.T) { } d := indexData{} q := query.Regexp{ - Regexp: r, + Regexp: r, + CaseSensitive: c.caseSensitive, } gotQuery, isEq, _, _ := d.regexpToMatchTreeRecursive(q.Regexp, 3, q.FileName, q.CaseSensitive) if !reflect.DeepEqual(c.query, gotQuery) { From 68d04651cc8e4989e64ad72470fe2bc27efb91c2 Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Tue, 30 Apr 2024 10:50:51 +0200 Subject: [PATCH 08/31] zoekt-mirror-gerrit: handle http authentication (#770) --- cmd/zoekt-mirror-gerrit/main.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go index e593b62d9..5763bb040 100644 --- a/cmd/zoekt-mirror-gerrit/main.go +++ b/cmd/zoekt-mirror-gerrit/main.go @@ -139,6 +139,11 @@ func main() { for _, s := range []string{"http", "anonymous http"} { if schemeInfo, ok := info.Download.Schemes[s]; ok { projectURL = schemeInfo.URL + if s == "http" && schemeInfo.IsAuthRequired { + projectURL = addPassword(projectURL, rootURL.User) + // remove "/a/" prefix needed for API call with basic auth but not with git command → cleaner repo name + projectURL = strings.Replace(projectURL, "/a/${project}", "/${project}", 1) + } break } } @@ -187,7 +192,7 @@ func main() { config := map[string]string{ "zoekt.name": zoektName, "zoekt.gerrit-project": k, - "zoekt.gerrit-host": rootURL.String(), + "zoekt.gerrit-host": anonymousURL(rootURL), "zoekt.archived": marshalBool(v.State == "READ_ONLY"), "zoekt.public": marshalBool(v.State != "HIDDEN"), } @@ -246,3 +251,15 @@ func marshalBool(b bool) string { } return "0" } + +func anonymousURL(u *url.URL) string { + anon := *u + anon.User = nil + return anon.String() +} + +func addPassword(u string, user *url.Userinfo) string { + password, _ := user.Password() + username := user.Username() + return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1) +} From 72f95004e6d6136fed7bf973616e89e6d37e4eaa Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Wed, 1 May 2024 09:21:56 +0200 Subject: [PATCH 09/31] fix: don't modify finalCands (#773) While working on ranking, I noticed that sum-tf is wrong if we have filename and content matches. We use `finalCands` in our BM25 scoring, however, `finalCands` is modified in `fillChunkMatches` and `fillMatches` which can lead to surprising scores. Test plan: updated unit test --- build/scoring_test.go | 4 ++-- contentprovider.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/scoring_test.go b/build/scoring_test.go index e3941fecd..9bf243fc3 100644 --- a/build/scoring_test.go +++ b/build/scoring_test.go @@ -77,8 +77,8 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "example"}, content: exampleJava, language: "Java", - // keyword-score:1.63 (sum-tf: 6.00, length-ratio: 2.00) - wantScore: 1.63, + // keyword-score:1.69 (sum-tf: 7.00, length-ratio: 2.00) + wantScore: 1.69, }, { // Matches only on content fileName: "example.java", diff --git a/contentprovider.go b/contentprovider.go index c66b1f471..e4a1ce1db 100644 --- a/contentprovider.go +++ b/contentprovider.go @@ -147,7 +147,7 @@ func (p *contentProvider) findOffset(filename bool, r uint32) uint32 { // returned by the API it needs to be copied. func (p *contentProvider) fillMatches(ms []*candidateMatch, numContextLines int, language string, debug bool) []LineMatch { var filenameMatches []*candidateMatch - contentMatches := ms[:0] + contentMatches := make([]*candidateMatch, 0, len(ms)) for _, m := range ms { if m.fileName { @@ -194,7 +194,7 @@ func (p *contentProvider) fillMatches(ms []*candidateMatch, numContextLines int, // returned by the API it needs to be copied. func (p *contentProvider) fillChunkMatches(ms []*candidateMatch, numContextLines int, language string, debug bool) []ChunkMatch { var filenameMatches []*candidateMatch - contentMatches := ms[:0] + contentMatches := make([]*candidateMatch, 0, len(ms)) for _, m := range ms { if m.fileName { From 69068bff5b05ec9b8af89c51ee7f014f21c258ae Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Fri, 3 May 2024 09:55:07 +0200 Subject: [PATCH 10/31] zoekt-mirror-gerrit: add an option to fetch meta/config branch (#774) --- cmd/zoekt-indexserver/config.go | 4 ++++ cmd/zoekt-mirror-gerrit/main.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go index 0a8bc3662..ff66c2f4e 100644 --- a/cmd/zoekt-indexserver/config.go +++ b/cmd/zoekt-indexserver/config.go @@ -50,6 +50,7 @@ type ConfigEntry struct { ExcludeTopics []string Active bool NoArchived bool + GerritFetchMetaConfig bool } func randomize(entries []ConfigEntry) []ConfigEntry { @@ -259,6 +260,9 @@ func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string if c.Active { cmd.Args = append(cmd.Args, "-active") } + if c.GerritFetchMetaConfig { + cmd.Args = append(cmd.Args, "-fetch-meta-config") + } cmd.Args = append(cmd.Args, c.GerritApiURL) } else { log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c) diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go index 5763bb040..98d49ff67 100644 --- a/cmd/zoekt-mirror-gerrit/main.go +++ b/cmd/zoekt-mirror-gerrit/main.go @@ -31,6 +31,8 @@ import ( "strings" gerrit "github.com/andygrunwald/go-gerrit" + git "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/sourcegraph/zoekt/gitindex" ) @@ -92,6 +94,7 @@ func main() { repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", "))) excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") deleteRepos := flag.Bool("delete", false, "delete missing repos") + fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch") httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.") active := flag.Bool("active", false, "mirror only active projects") flag.Parse() @@ -216,6 +219,11 @@ func main() { } else { fmt.Println(dest) } + if *fetchMetaConfig { + if err := addMetaConfigFetch(filepath.Join(*dest, name+".git")); err != nil { + log.Fatalf("addMetaConfigFetch: %v", err) + } + } } if *deleteRepos { if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil { @@ -263,3 +271,28 @@ func addPassword(u string, user *url.Userinfo) string { username := user.Username() return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1) } + +func addMetaConfigFetch(repoDir string) error { + repo, err := git.PlainOpen(repoDir) + if err != nil { + return err + } + + cfg, err := repo.Config() + if err != nil { + return err + } + + rm := cfg.Remotes["origin"] + if rm != nil { + configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta/config") + if !slices.Contains(rm.Fetch, configRefSpec) { + rm.Fetch = append(rm.Fetch, configRefSpec) + } + } + if err := repo.Storer.SetConfig(cfg); err != nil { + return err + } + + return nil +} From 647e86c96a227411ae6823879f6bc43e607a6449 Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Fri, 3 May 2024 09:59:29 +0200 Subject: [PATCH 11/31] zoekt-indexserver: handle gerrit-mirror RepoNameFormat config (#772) --- cmd/zoekt-indexserver/config.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go index ff66c2f4e..a1fb3923d 100644 --- a/cmd/zoekt-indexserver/config.go +++ b/cmd/zoekt-indexserver/config.go @@ -51,6 +51,7 @@ type ConfigEntry struct { Active bool NoArchived bool GerritFetchMetaConfig bool + GerritRepoNameFormat string } func randomize(entries []ConfigEntry) []ConfigEntry { @@ -260,8 +261,11 @@ func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string if c.Active { cmd.Args = append(cmd.Args, "-active") } - if c.GerritFetchMetaConfig { + if c.GerritFetchMetaConfig { cmd.Args = append(cmd.Args, "-fetch-meta-config") + } + if c.GerritRepoNameFormat != "" { + cmd.Args = append(cmd.Args, "-repo-name-format", c.GerritRepoNameFormat) } cmd.Args = append(cmd.Args, c.GerritApiURL) } else { From 7c5b77843b4e44ebf640fa5afb08ef9e6dc778c3 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Fri, 3 May 2024 21:12:31 +0200 Subject: [PATCH 12/31] gitindex: update remote.origin.url if clone already exists (#776) This is so that when authentication details change we correctly update the CloneURL for repositories managed by our mirror commands. --- gitindex/clone.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitindex/clone.go b/gitindex/clone.go index 2399e29fa..8bee54de0 100644 --- a/gitindex/clone.go +++ b/gitindex/clone.go @@ -18,6 +18,7 @@ import ( "bytes" "fmt" "log" + "maps" "os" "os/exec" "path/filepath" @@ -60,7 +61,9 @@ func CloneRepo(destDir, name, cloneURL string, settings map[string]string) (stri repoDest := filepath.Join(parent, filepath.Base(name)+".git") if _, err := os.Lstat(repoDest); err == nil { - // Repository exists, ensure settings are in sync + // Repository exists, ensure settings are in sync including the clone URL + settings := maps.Clone(settings) + settings["remote.origin.url"] = cloneURL if err := updateZoektGitConfig(repoDest, settings); err != nil { return "", fmt.Errorf("failed to update repository settings: %w", err) } From 9f35cb185def23cb544e37fbde044c1a436c909e Mon Sep 17 00:00:00 2001 From: Xavier Calland Date: Mon, 6 May 2024 14:56:06 +0200 Subject: [PATCH 13/31] zoekt-mirror-gerrit: fix fetch for meta/config (#777) The current configuration creates a conflict with default fetch config `+refs/heads/*:refs/heads/*`. The first fetch is ok but once a local "refs/heads/meta/config" exists fetch fails because 2 fetch rules match for local "refs/heads/meta/config" : "refs/meta/config" and "refs/heads/meta/config" Changing local ref name fixes the problem --- cmd/zoekt-mirror-gerrit/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go index 98d49ff67..bf0d8ac04 100644 --- a/cmd/zoekt-mirror-gerrit/main.go +++ b/cmd/zoekt-mirror-gerrit/main.go @@ -285,7 +285,7 @@ func addMetaConfigFetch(repoDir string) error { rm := cfg.Remotes["origin"] if rm != nil { - configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta/config") + configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta-config") if !slices.Contains(rm.Fetch, configRefSpec) { rm.Fetch = append(rm.Fetch, configRefSpec) } From 4e674a49795c87b2a8199a42ae2d1b3e5ac2eb9e Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 7 May 2024 10:57:42 -0700 Subject: [PATCH 14/31] Rename UseKeywordScoring to mention BM25 (#778) It's confusing to call this `UseKeywordScoring`, since we do not use it for the `keyword` patterntype in Sourcegraph. This commit clarifies the name to mention BM25. --- api.go | 10 +- api_proto.go | 4 +- build/scoring_test.go | 16 +- eval.go | 2 +- .../protos/zoekt/webserver/v1/webserver.pb.go | 765 +++++++++--------- .../protos/zoekt/webserver/v1/webserver.proto | 4 +- score.go | 8 +- shards/shards_test.go | 4 +- 8 files changed, 406 insertions(+), 407 deletions(-) diff --git a/api.go b/api.go index 6d2497d10..192b6a834 100644 --- a/api.go +++ b/api.go @@ -946,10 +946,10 @@ type SearchOptions struct { // will be used. This option is temporary and is only exposed for testing/ tuning purposes. DocumentRanksWeight float64 - // EXPERIMENTAL. If true, use keyword-style scoring instead of the default scoring formula. - // Currently, this treats each match in a file as a term and computes an approximation to BM25. - // When enabled, all other scoring signals are ignored, including document ranks. - UseKeywordScoring bool + // EXPERIMENTAL. If true, use text-search style scoring instead of the default scoring formula. + // The scoring algorithm treats each match in a file as a term and computes an approximation to + // BM25. When enabled, all other scoring signals are ignored, including document ranks. + UseBM25Scoring bool // Trace turns on opentracing for this request if true and if the Jaeger address was provided as // a command-line flag @@ -1015,7 +1015,7 @@ func (s *SearchOptions) String() string { addBool("Whole", s.Whole) addBool("ChunkMatches", s.ChunkMatches) addBool("UseDocumentRanks", s.UseDocumentRanks) - addBool("UseKeywordScoring", s.UseKeywordScoring) + addBool("UseBM25Scoring", s.UseBM25Scoring) addBool("Trace", s.Trace) addBool("DebugScore", s.DebugScore) diff --git a/api_proto.go b/api_proto.go index 7e17b8ca0..368b689d3 100644 --- a/api_proto.go +++ b/api_proto.go @@ -700,7 +700,7 @@ func SearchOptionsFromProto(p *proto.SearchOptions) *SearchOptions { DocumentRanksWeight: p.GetDocumentRanksWeight(), Trace: p.GetTrace(), DebugScore: p.GetDebugScore(), - UseKeywordScoring: p.GetUseKeywordScoring(), + UseBM25Scoring: p.GetUseBm25Scoring(), } } @@ -725,6 +725,6 @@ func (s *SearchOptions) ToProto() *proto.SearchOptions { DocumentRanksWeight: s.DocumentRanksWeight, Trace: s.Trace, DebugScore: s.DebugScore, - UseKeywordScoring: s.UseKeywordScoring, + UseBm25Scoring: s.UseBM25Scoring, } } diff --git a/build/scoring_test.go b/build/scoring_test.go index 9bf243fc3..03a139281 100644 --- a/build/scoring_test.go +++ b/build/scoring_test.go @@ -77,7 +77,7 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "example"}, content: exampleJava, language: "Java", - // keyword-score:1.69 (sum-tf: 7.00, length-ratio: 2.00) + // bm25-score:1.69 (sum-tf: 7.00, length-ratio: 2.00) wantScore: 1.69, }, { // Matches only on content @@ -89,7 +89,7 @@ func TestBM25(t *testing.T) { }}, content: exampleJava, language: "Java", - // keyword-score:5.75 (sum-tf: 56.00, length-ratio: 2.00) + // bm25-score:5.75 (sum-tf: 56.00, length-ratio: 2.00) wantScore: 5.75, }, { @@ -98,7 +98,7 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "java"}, content: exampleJava, language: "Java", - // keyword-score:1.07 (sum-tf: 2.00, length-ratio: 2.00) + // bm25-score:1.07 (sum-tf: 2.00, length-ratio: 2.00) wantScore: 1.07, }, { @@ -106,7 +106,7 @@ func TestBM25(t *testing.T) { fileName: "a/b/c/config.go", query: &query.Substring{Pattern: "config.go"}, language: "Go", - // keyword-score:1.91 (sum-tf: 2.00, length-ratio: 0.00) + // bm25-score:1.91 (sum-tf: 2.00, length-ratio: 0.00) wantScore: 1.91, }, } @@ -584,7 +584,7 @@ func skipIfCTagsUnavailable(t *testing.T, parserType ctags.CTagsParserType) { } } -func checkScoring(t *testing.T, c scoreCase, keywordScoring bool, parserType ctags.CTagsParserType) { +func checkScoring(t *testing.T, c scoreCase, useBM25 bool, parserType ctags.CTagsParserType) { skipIfCTagsUnavailable(t, parserType) name := c.language @@ -625,9 +625,9 @@ func checkScoring(t *testing.T, c scoreCase, keywordScoring bool, parserType cta defer ss.Close() srs, err := ss.Search(context.Background(), c.query, &zoekt.SearchOptions{ - UseKeywordScoring: keywordScoring, - ChunkMatches: true, - DebugScore: true}) + UseBM25Scoring: useBM25, + ChunkMatches: true, + DebugScore: true}) if err != nil { t.Fatal(err) } diff --git a/eval.go b/eval.go index cc0e61f27..0d8ec91bd 100644 --- a/eval.go +++ b/eval.go @@ -317,7 +317,7 @@ nextFileMatch: fileMatch.LineMatches = cp.fillMatches(finalCands, opts.NumContextLines, fileMatch.Language, opts.DebugScore) } - if opts.UseKeywordScoring { + if opts.UseBM25Scoring { d.scoreFileUsingBM25(&fileMatch, nextDoc, finalCands, opts) } else { // Use the standard, non-experimental scoring method by default diff --git a/grpc/protos/zoekt/webserver/v1/webserver.pb.go b/grpc/protos/zoekt/webserver/v1/webserver.pb.go index f7bfb4427..d080bf379 100644 --- a/grpc/protos/zoekt/webserver/v1/webserver.pb.go +++ b/grpc/protos/zoekt/webserver/v1/webserver.pb.go @@ -388,10 +388,10 @@ type SearchOptions struct { Trace bool `protobuf:"varint,13,opt,name=trace,proto3" json:"trace,omitempty"` // If set, the search results will contain debug information for scoring. DebugScore bool `protobuf:"varint,14,opt,name=debug_score,json=debugScore,proto3" json:"debug_score,omitempty"` - // EXPERIMENTAL. If true, use keyword-style scoring instead of the default scoring formula. + // EXPERIMENTAL. If true, use text search scoring instead of the default scoring formula. // Currently, this treats each match in a file as a term and computes an approximation to BM25. // When enabled, all other scoring signals are ignored, including document ranks. - UseKeywordScoring bool `protobuf:"varint,15,opt,name=use_keyword_scoring,json=useKeywordScoring,proto3" json:"use_keyword_scoring,omitempty"` + UseBm25Scoring bool `protobuf:"varint,15,opt,name=use_bm25_scoring,json=useBm25Scoring,proto3" json:"use_bm25_scoring,omitempty"` } func (x *SearchOptions) Reset() { @@ -531,9 +531,9 @@ func (x *SearchOptions) GetDebugScore() bool { return false } -func (x *SearchOptions) GetUseKeywordScoring() bool { +func (x *SearchOptions) GetUseBm25Scoring() bool { if x != nil { - return x.UseKeywordScoring + return x.UseBm25Scoring } return false } @@ -2337,7 +2337,7 @@ var file_zoekt_webserver_v1_webserver_proto_rawDesc = []byte{ 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x06, 0x22, 0xfb, 0x05, 0x0a, 0x0d, + 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x06, 0x22, 0xf5, 0x05, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x6f, 0x63, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x65, 0x73, 0x74, 0x69, 0x6d, @@ -2382,389 +2382,388 @@ var file_zoekt_webserver_v1_webserver_proto_rawDesc = []byte{ 0x68, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x64, - 0x65, 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, - 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x77, 0x6f, - 0x72, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x6f, 0x0a, 0x0b, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, - 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x52, 0x05, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x33, 0x0a, 0x04, 0x6f, 0x70, 0x74, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, + 0x65, 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x75, 0x73, 0x65, + 0x5f, 0x62, 0x6d, 0x32, 0x35, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x42, 0x6d, 0x32, 0x35, 0x53, 0x63, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x22, 0x6f, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, + 0x33, 0x0a, 0x04, 0x6f, 0x70, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, + 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x04, + 0x6f, 0x70, 0x74, 0x73, 0x22, 0xd2, 0x01, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x43, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x04, 0x6f, 0x70, 0x74, 0x73, 0x22, 0xd2, 0x01, 0x0a, 0x0b, 0x4c, - 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x43, 0x0a, 0x05, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, - 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4c, - 0x69, 0x73, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x22, - 0x78, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x12, 0x27, 0x0a, 0x23, 0x52, 0x45, 0x50, 0x4f, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x5f, 0x46, 0x49, - 0x45, 0x4c, 0x44, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x50, - 0x4f, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x52, 0x45, 0x50, - 0x4f, 0x53, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x52, 0x45, 0x50, 0x4f, 0x5f, 0x4c, 0x49, 0x53, - 0x54, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x5f, 0x4d, 0x41, - 0x50, 0x10, 0x03, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x10, 0x10, 0x11, 0x22, - 0xd0, 0x02, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x37, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x4b, 0x0a, 0x09, 0x72, 0x65, 0x70, - 0x6f, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x7a, - 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, - 0x65, 0x70, 0x6f, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x72, 0x65, - 0x70, 0x6f, 0x73, 0x4d, 0x61, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, - 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x73, 0x1a, 0x65, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x4d, 0x61, - 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x3e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, - 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x6e, - 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x05, - 0x10, 0x06, 0x22, 0xce, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x3e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, - 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x79, 0x12, 0x48, 0x0a, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x7a, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x22, 0x78, 0x0a, 0x0d, 0x52, 0x65, 0x70, + 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x27, 0x0a, 0x23, 0x52, 0x45, + 0x50, 0x4f, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x50, 0x4f, 0x5f, 0x4c, 0x49, 0x53, 0x54, + 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x01, 0x12, 0x1d, + 0x0a, 0x19, 0x52, 0x45, 0x50, 0x4f, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x5f, 0x46, 0x49, 0x45, 0x4c, + 0x44, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x5f, 0x4d, 0x41, 0x50, 0x10, 0x03, 0x22, 0x04, 0x08, + 0x02, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x10, 0x10, 0x11, 0x22, 0xd0, 0x02, 0x0a, 0x0c, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x05, 0x72, 0x65, + 0x70, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, + 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x65, + 0x70, 0x6f, 0x73, 0x12, 0x4b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x5f, 0x6d, 0x61, 0x70, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, + 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x4d, 0x61, + 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x4d, 0x61, 0x70, + 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, + 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, + 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x1a, + 0x65, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x3e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x65, + 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xce, 0x01, 0x0a, + 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x3e, + 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x79, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x48, + 0x0a, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, + 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, + 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xf2, 0x06, + 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x08, 0x62, 0x72, + 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, - 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x22, 0xf2, 0x06, 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x12, 0x40, 0x0a, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, + 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x72, 0x61, 0x6e, + 0x63, 0x68, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x0c, + 0x73, 0x75, 0x62, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x79, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x65, 0x73, 0x12, 0x50, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x6d, - 0x61, 0x70, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, - 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x2e, 0x53, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, - 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x52, 0x65, 0x70, - 0x6f, 0x4d, 0x61, 0x70, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x75, - 0x72, 0x6c, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x55, 0x72, 0x6c, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, - 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x12, 0x34, 0x0a, 0x16, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x66, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, - 0x74, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x14, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, - 0x74, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, - 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0a, 0x72, 0x61, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, - 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, - 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x2e, 0x52, 0x61, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x72, 0x61, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, - 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x61, 0x73, - 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x68, 0x61, 0x73, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x6f, - 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, - 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x61, 0x74, 0x65, - 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x10, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x10, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x61, - 0x74, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x6f, 0x6d, 0x62, 0x73, - 0x74, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x6c, - 0x65, 0x54, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x73, 0x1a, 0x5d, 0x0a, 0x0f, 0x53, - 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3c, 0x0a, 0x0e, 0x52, 0x61, - 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd6, 0x03, 0x0a, 0x0d, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x46, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x13, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x15, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x69, 0x6e, 0x52, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x73, - 0x63, 0x69, 0x69, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x70, 0x6c, 0x61, 0x69, 0x6e, - 0x41, 0x73, 0x63, 0x69, 0x69, 0x12, 0x55, 0x0a, 0x0c, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, - 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x7a, 0x6f, - 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, - 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0b, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x12, 0x23, 0x0a, 0x0d, - 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x1a, 0x3e, 0x0a, 0x10, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xa1, 0x01, 0x0a, 0x14, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x65, 0x70, - 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x61, - 0x73, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0a, 0x68, 0x61, 0x73, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x40, 0x0a, 0x08, 0x62, - 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x72, 0x61, - 0x6e, 0x63, 0x68, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, - 0x0f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x54, 0x69, 0x6d, - 0x65, 0x55, 0x6e, 0x69, 0x78, 0x22, 0x40, 0x0a, 0x10, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, - 0x6f, 0x72, 0x79, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xcd, 0x02, 0x0a, 0x09, 0x52, 0x65, 0x70, 0x6f, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x79, 0x74, - 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x62, 0x79, - 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x77, 0x5f, 0x6c, - 0x69, 0x6e, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x0d, 0x6e, 0x65, 0x77, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, - 0x42, 0x0a, 0x1e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x62, 0x72, 0x61, 0x6e, 0x63, - 0x68, 0x5f, 0x6e, 0x65, 0x77, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1a, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x4e, 0x65, 0x77, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x42, 0x0a, 0x1e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x62, 0x72, 0x61, - 0x6e, 0x63, 0x68, 0x65, 0x73, 0x5f, 0x6e, 0x65, 0x77, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x5f, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1a, 0x6f, 0x74, 0x68, - 0x65, 0x72, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x4e, 0x65, 0x77, 0x4c, 0x69, 0x6e, - 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa9, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x12, 0x30, 0x0a, 0x14, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x61, - 0x64, 0x65, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x10, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x65, - 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x14, 0x73, 0x68, 0x61, 0x72, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x43, 0x6f, 0x6e, - 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, - 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, - 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x4c, - 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x73, - 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x66, 0x69, - 0x6c, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x5f, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x53, 0x63, 0x61, 0x6e, 0x6e, 0x65, - 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, - 0x70, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x15, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x03, 0x52, 0x13, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x53, - 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, - 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, - 0x0d, 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x0e, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x04, 0x77, 0x61, 0x69, 0x74, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x77, 0x61, 0x69, - 0x74, 0x12, 0x51, 0x0a, 0x17, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x13, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x15, 0x6d, - 0x61, 0x74, 0x63, 0x68, 0x54, 0x72, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x45, 0x0a, 0x11, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x74, 0x72, - 0x65, 0x65, 0x5f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x6d, 0x61, 0x74, 0x63, - 0x68, 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2d, 0x0a, 0x12, 0x72, - 0x65, 0x67, 0x65, 0x78, 0x70, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, - 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x72, 0x65, 0x67, 0x65, 0x78, 0x70, 0x73, - 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x12, 0x42, 0x0a, 0x0c, 0x66, 0x6c, - 0x75, 0x73, 0x68, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x1f, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x52, 0x0b, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x23, - 0x0a, 0x0d, 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x73, 0x18, - 0x12, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x4c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x73, 0x22, 0x58, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x01, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x30, 0x0a, 0x14, 0x6d, - 0x61, 0x78, 0x5f, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, - 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, - 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x22, 0xb9, 0x04, - 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x73, - 0x63, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, - 0x12, 0x40, 0x0a, 0x0c, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, - 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, - 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x6e, 0x65, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0d, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6d, 0x61, 0x74, 0x63, - 0x68, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, - 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x0c, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x70, 0x6f, 0x73, - 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x49, 0x64, 0x12, 0x2f, 0x0a, 0x13, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, - 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x01, 0x52, 0x12, 0x72, 0x65, 0x70, 0x6f, 0x73, - 0x69, 0x74, 0x6f, 0x72, 0x79, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, - 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x73, 0x75, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x73, 0x75, 0x6d, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, - 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x12, - 0x2e, 0x0a, 0x13, 0x73, 0x75, 0x62, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, - 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x73, 0x75, - 0x62, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x2e, 0x0a, 0x13, 0x73, 0x75, 0x62, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, - 0x79, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x73, 0x75, - 0x62, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xca, 0x02, 0x0a, 0x09, 0x4c, 0x69, - 0x6e, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, - 0x69, 0x6e, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6c, 0x69, - 0x6e, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6c, 0x69, - 0x6e, 0x65, 0x45, 0x6e, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6c, 0x69, 0x6e, 0x65, - 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x61, - 0x66, 0x74, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, - 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, - 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, - 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x4c, 0x0a, 0x0e, 0x6c, 0x69, 0x6e, 0x65, - 0x5f, 0x66, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x25, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, - 0x6e, 0x74, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x0d, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, - 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xc5, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x6e, 0x65, 0x46, - 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1f, 0x0a, 0x0b, - 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0a, 0x6c, 0x69, 0x6e, 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6f, - 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x6d, 0x61, 0x74, - 0x63, 0x68, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x44, 0x0a, 0x0b, 0x73, 0x79, 0x6d, 0x62, - 0x6f, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x72, 0x79, 0x2e, 0x53, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x61, 0x70, 0x12, 0x2e, + 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x5f, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x55, 0x72, 0x6c, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x2a, + 0x0a, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x55, + 0x72, 0x6c, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x6c, 0x69, + 0x6e, 0x65, 0x5f, 0x66, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x6c, 0x69, 0x6e, 0x65, + 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x01, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0a, + 0x72, 0x61, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x2d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, + 0x2e, 0x52, 0x61, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x09, 0x72, 0x61, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, + 0x6e, 0x6b, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x23, + 0x0a, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x61, 0x73, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, + 0x6c, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x68, 0x61, 0x73, 0x53, 0x79, 0x6d, + 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, + 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, + 0x6e, 0x65, 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x0f, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x73, 0x18, + 0x11, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x6f, 0x6d, 0x62, 0x73, + 0x74, 0x6f, 0x6e, 0x65, 0x73, 0x1a, 0x5d, 0x0a, 0x0f, 0x53, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, + 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, + 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3c, 0x0a, 0x0e, 0x52, 0x61, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xd6, 0x03, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x66, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x12, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x13, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x4d, 0x69, 0x6e, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1f, + 0x0a, 0x0b, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x73, 0x63, 0x69, 0x69, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x41, 0x73, 0x63, 0x69, 0x69, 0x12, + 0x55, 0x0a, 0x0c, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x18, + 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, + 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, + 0x65, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x6c, 0x61, 0x6e, 0x67, 0x75, + 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x12, 0x23, 0x0a, 0x0d, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x7a, + 0x6f, 0x65, 0x6b, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x1a, 0x3e, 0x0a, 0x10, 0x4c, + 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa1, 0x01, 0x0a, 0x14, + 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x61, 0x73, 0x5f, 0x73, 0x79, 0x6d, 0x62, + 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x68, 0x61, 0x73, 0x53, 0x79, + 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x40, 0x0a, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, + 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x08, 0x62, + 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x22, + 0x40, 0x0a, 0x10, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x72, 0x61, + 0x6e, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0xcd, 0x02, 0x0a, 0x09, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, + 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x0a, + 0x09, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x77, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x5f, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x6e, 0x65, 0x77, 0x4c, + 0x69, 0x6e, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x42, 0x0a, 0x1e, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x6e, 0x65, 0x77, 0x5f, + 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x1a, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, + 0x4e, 0x65, 0x77, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x42, 0x0a, + 0x1e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x5f, + 0x6e, 0x65, 0x77, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1a, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x42, 0x72, 0x61, 0x6e, + 0x63, 0x68, 0x65, 0x73, 0x4e, 0x65, 0x77, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x22, 0xa9, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x2c, 0x0a, + 0x12, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x72, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, + 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, + 0x64, 0x65, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, + 0x64, 0x12, 0x29, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, + 0x64, 0x65, 0x72, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x66, 0x69, 0x6c, + 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, + 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x53, 0x6b, 0x69, + 0x70, 0x70, 0x65, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x5f, 0x73, + 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x53, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, + 0x65, 0x64, 0x12, 0x32, 0x0a, 0x15, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x5f, 0x73, 0x6b, 0x69, + 0x70, 0x70, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x13, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x74, + 0x63, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x67, 0x72, 0x61, 0x6d, + 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, + 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x04, + 0x77, 0x61, 0x69, 0x74, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x77, 0x61, 0x69, 0x74, 0x12, 0x51, 0x0a, 0x17, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x15, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x72, + 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x45, + 0x0a, 0x11, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x72, 0x65, 0x65, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, 0x67, 0x65, 0x78, 0x70, 0x73, + 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x65, 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x11, 0x72, 0x65, 0x67, 0x65, 0x78, 0x70, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, + 0x65, 0x72, 0x65, 0x64, 0x12, 0x42, 0x0a, 0x0c, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x5f, 0x72, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x7a, 0x6f, 0x65, + 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x0b, 0x66, 0x6c, 0x75, + 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x67, 0x72, 0x61, + 0x6d, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0c, 0x6e, 0x67, 0x72, 0x61, 0x6d, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x73, 0x22, 0x58, 0x0a, + 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x30, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x65, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x50, + 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x22, 0xb9, 0x04, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, + 0x65, 0x62, 0x75, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, 0x65, 0x62, 0x75, + 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, + 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1a, + 0x0a, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x73, 0x12, 0x40, 0x0a, 0x0c, 0x6c, 0x69, + 0x6e, 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, + 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0d, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x07, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x52, 0x0c, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, + 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x6f, 0x72, 0x79, 0x49, 0x64, 0x12, 0x2f, 0x0a, 0x13, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x12, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x50, + 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x12, 0x1a, 0x0a, + 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x73, 0x75, 0x62, + 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x73, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x73, 0x75, 0x62, + 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x73, 0x75, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x22, 0xca, 0x02, 0x0a, 0x09, 0x4c, 0x69, 0x6e, 0x65, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6c, 0x69, 0x6e, 0x65, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x65, 0x6e, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6c, 0x69, 0x6e, 0x65, 0x45, 0x6e, 0x64, 0x12, + 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6c, 0x69, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x12, 0x16, 0x0a, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x66, 0x74, 0x65, + 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x61, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1b, + 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, + 0x63, 0x6f, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, + 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, + 0x72, 0x65, 0x12, 0x4c, 0x0a, 0x0e, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x66, 0x72, 0x61, 0x67, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x7a, 0x6f, 0x65, + 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x52, 0x0d, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x22, 0xc5, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x6e, 0x65, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, + 0x74, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6c, 0x69, 0x6e, + 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, + 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x12, 0x44, 0x0a, 0x0b, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x5f, 0x69, 0x6e, 0x66, + 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, + 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, + 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x79, 0x6d, 0x62, 0x6f, + 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x73, 0x79, 0x6d, + 0x62, 0x6f, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x6b, 0x0a, 0x0a, 0x53, 0x79, 0x6d, 0x62, + 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x6d, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x6b, + 0x69, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x4b, 0x69, 0x6e, 0x64, 0x22, 0xb1, 0x02, 0x0a, 0x0a, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x41, + 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, + 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x31, + 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, + 0x73, 0x12, 0x3f, 0x0a, 0x0b, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, + 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x62, + 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x62, 0x75, + 0x67, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, + 0x65, 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x6b, 0x0a, 0x05, 0x52, 0x61, 0x6e, + 0x67, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2e, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x64, 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x4f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x6c, 0x69, 0x6e, 0x65, 0x4e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x2a, 0x8c, 0x01, 0x0a, + 0x0b, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x20, + 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, + 0x4f, 0x4e, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x52, 0x5f, 0x45, 0x58, 0x50, 0x49, 0x52, 0x45, 0x44, + 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, + 0x4f, 0x4e, 0x5f, 0x46, 0x49, 0x4e, 0x41, 0x4c, 0x5f, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x10, 0x02, + 0x12, 0x19, 0x0a, 0x15, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, + 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x10, 0x03, 0x32, 0x99, 0x02, 0x0a, 0x10, + 0x57, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x51, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x21, 0x2e, 0x7a, 0x6f, 0x65, + 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, - 0x0a, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x88, 0x01, 0x01, 0x42, 0x0e, - 0x0a, 0x0c, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x6b, - 0x0a, 0x0a, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, - 0x73, 0x79, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x6d, 0x12, 0x12, - 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, - 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4b, 0x69, 0x6e, 0x64, 0x22, 0xb1, 0x02, 0x0a, 0x0a, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, - 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, - 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0b, 0x73, 0x79, 0x6d, 0x62, 0x6f, - 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x7a, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x0c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x12, 0x27, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x73, 0x79, - 0x6d, 0x62, 0x6f, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, 0x62, 0x75, 0x67, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x22, - 0x6b, 0x0a, 0x05, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, - 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2e, 0x0a, 0x03, - 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, - 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x64, 0x0a, 0x08, - 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, - 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x62, - 0x79, 0x74, 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, - 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, - 0x6c, 0x69, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x2a, 0x8c, 0x01, 0x0a, 0x0b, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x52, 0x65, 0x61, 0x73, - 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, - 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x4c, 0x55, 0x53, - 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x52, 0x5f, 0x45, - 0x58, 0x50, 0x49, 0x52, 0x45, 0x44, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x4c, 0x55, 0x53, - 0x48, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x49, 0x4e, 0x41, 0x4c, 0x5f, 0x46, - 0x4c, 0x55, 0x53, 0x48, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x4c, 0x55, 0x53, 0x48, 0x5f, - 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x10, - 0x03, 0x32, 0x99, 0x02, 0x0a, 0x10, 0x57, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x12, 0x21, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x0c, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x27, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, - 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, - 0x12, 0x4b, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, - 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, - 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3d, 0x5a, - 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2f, 0x67, 0x72, - 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2f, - 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4b, 0x0a, 0x04, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2e, 0x77, 0x65, 0x62, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, + 0x68, 0x2f, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x73, 0x2f, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x2f, 0x77, 0x65, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/grpc/protos/zoekt/webserver/v1/webserver.proto b/grpc/protos/zoekt/webserver/v1/webserver.proto index 981a119dc..7ffe70081 100644 --- a/grpc/protos/zoekt/webserver/v1/webserver.proto +++ b/grpc/protos/zoekt/webserver/v1/webserver.proto @@ -107,10 +107,10 @@ message SearchOptions { // If set, the search results will contain debug information for scoring. bool debug_score = 14; - // EXPERIMENTAL. If true, use keyword-style scoring instead of the default scoring formula. + // EXPERIMENTAL. If true, use text search scoring instead of the default scoring formula. // Currently, this treats each match in a file as a term and computes an approximation to BM25. // When enabled, all other scoring signals are ignored, including document ranks. - bool use_keyword_scoring = 15; + bool use_bm25_scoring = 15; } message ListRequest { diff --git a/score.go b/score.go index 115eabd29..dc0f4c192 100644 --- a/score.go +++ b/score.go @@ -39,9 +39,9 @@ func (m *FileMatch) addScore(what string, computed float64, raw float64, debugSc m.Score += computed } -func (m *FileMatch) addKeywordScore(score float64, sumTf float64, L float64, debugScore bool) { +func (m *FileMatch) addBM25Score(score float64, sumTf float64, L float64, debugScore bool) { if debugScore { - m.Debug += fmt.Sprintf("keyword-score:%.2f (sum-tf: %.2f, length-ratio: %.2f)", score, sumTf, L) + m.Debug += fmt.Sprintf("bm25-score:%.2f (sum-tf: %.2f, length-ratio: %.2f)", score, sumTf, L) } m.Score += score } @@ -116,7 +116,7 @@ func (d *indexData) scoreFile(fileMatch *FileMatch, doc uint32, mt matchTree, kn } // scoreFileUsingBM25 computes a score for the file match using an approximation to BM25, the most common scoring -// algorithm for keyword search: https://en.wikipedia.org/wiki/Okapi_BM25. It implements all parts of the formula +// algorithm for text search: https://en.wikipedia.org/wiki/Okapi_BM25. It implements all parts of the formula // except inverse document frequency (idf), since we don't have access to global term frequency statistics. // // Filename matches count twice as much as content matches. This mimics a common text search strategy where you @@ -160,5 +160,5 @@ func (d *indexData) scoreFileUsingBM25(fileMatch *FileMatch, doc uint32, cands [ score += ((k + 1.0) * tf) / (k*(1.0-b+b*L) + tf) } - fileMatch.addKeywordScore(score, sumTf, L, opts.DebugScore) + fileMatch.addBM25Score(score, sumTf, L, opts.DebugScore) } diff --git a/shards/shards_test.go b/shards/shards_test.go index 5c4ddc735..4a9a3865b 100644 --- a/shards/shards_test.go +++ b/shards/shards_test.go @@ -1087,7 +1087,7 @@ func TestAtomCountScore(t *testing.T) { } } -func TestUseKeywordScoring(t *testing.T) { +func TestUseBM25Scoring(t *testing.T) { b := testIndexBuilder(t, &zoekt.Repository{}, zoekt.Document{Name: "f1", Content: []byte("one two two three")}, @@ -1103,7 +1103,7 @@ func TestUseKeywordScoring(t *testing.T) { &query.Substring{Pattern: "three"}) opts := zoekt.SearchOptions{ - UseKeywordScoring: true, + UseBM25Scoring: true, } results, err := ss.Search(context.Background(), q, &opts) From fe8f2a3d9cab2ca2c27a19f428f1aca823134e80 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 14 May 2024 10:00:04 -0700 Subject: [PATCH 15/31] Avoid overlapping trigrams in distanceHitIterator (#779) We select the two least frequent trigrams to create the candidate match iterator. It's common for these trigrams to overlap. This change shifts the first and last trigrams to avoid overlap, which is guaranteed to result in a smaller intersection. For frequent terms, this can substantially reduce the number of candidate matches we consider. --- bits.go | 4 ++-- index_test.go | 4 ++-- indexdata.go | 48 +++++++++++++++++++++++++++++++++++------------ indexdata_test.go | 28 +++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/bits.go b/bits.go index 0b4901745..7d1363102 100644 --- a/bits.go +++ b/bits.go @@ -110,7 +110,7 @@ func (n ngram) String() string { type runeNgramOff struct { ngram ngram // index is the original index inside of the returned array of splitNGrams - index uint32 + index int } func (a runeNgramOff) Compare(b runeNgramOff) int { @@ -149,7 +149,7 @@ func splitNGrams(str []byte) []runeNgramOff { ng := runesToNGram(runeGram) result = append(result, runeNgramOff{ ngram: ng, - index: uint32(len(result)), + index: len(result), }) } return result diff --git a/index_test.go b/index_test.go index 8608b5bfc..05401215f 100644 --- a/index_test.go +++ b/index_test.go @@ -441,7 +441,7 @@ func TestSearchStats(t *testing.T) { Want: Stats{ FilesLoaded: 1, ContentBytesLoaded: 22, - IndexBytesLoaded: 8, + IndexBytesLoaded: 10, NgramMatches: 3, // we look at doc 1, because it's max(0,1) due to AND NgramLookups: 104, MatchCount: 2, @@ -556,7 +556,7 @@ func TestSearchStats(t *testing.T) { }}, Want: Stats{ ContentBytesLoaded: 33, // we still have to run regex since "app" matches two documents - IndexBytesLoaded: 8, + IndexBytesLoaded: 10, FilesConsidered: 2, // important that we don't check 3 to ensure we are using the index FilesLoaded: 2, MatchCount: 0, // even though there is a match it doesn't align with a symbol diff --git a/indexdata.go b/indexdata.go index ddb8a95c1..2fdd6ef5a 100644 --- a/indexdata.go +++ b/indexdata.go @@ -336,9 +336,31 @@ func min2Index(xs []uint32) (idx0, idx1 int) { return } -// minFrequencyNgramOffsets returns the two lowest frequency ngrams to pass to -// the distance iterator. If they have the same frequency, we maximise the -// distance between them. first will always have a smaller index than last. +// findSelectiveNgrams returns two ngrams to pass to the distance iterator, chosen to +// produce a small file intersection. It finds the two lowest frequency ngrams, making +// sure to maximize the distance between them in case of ties. It avoids overlapping +// trigrams to keep their intersection as small as possible. +// +// Invariant: first will always have a smaller index than last. +func findSelectiveNgrams(ngramOffs []runeNgramOff, indexMap []int, frequencies []uint32) (first, last runeNgramOff) { + first, last = minFrequencyNgramOffsets(ngramOffs, frequencies) + + // If the trigrams are overlapping, then try to shift one to reduce overlap. + // This is guaranteed to produce a smaller intersection. + if last.index-first.index < ngramSize { + newFirstIndex := max(last.index-ngramSize, 0) + if newFirstIndex != first.index { + first = ngramOffs[indexMap[newFirstIndex]] + } + + newLastIndex := min(first.index+ngramSize, len(ngramOffs)-1) + if newLastIndex != last.index { + last = ngramOffs[indexMap[newLastIndex]] + } + } + return +} + func minFrequencyNgramOffsets(ngramOffs []runeNgramOff, frequencies []uint32) (first, last runeNgramOff) { firstI, lastI := min2Index(frequencies) // If the frequencies are equal lets maximise distance in the query @@ -357,13 +379,15 @@ func minFrequencyNgramOffsets(ngramOffs []runeNgramOff, frequencies []uint32) (f } } } + first = ngramOffs[firstI] last = ngramOffs[lastI] - // Ensure first appears before last to make distance logic below clean. + + // Ensure first appears before last as a helpful invariant. if first.index > last.index { last, first = first, last } - return first, last + return } func (data *indexData) ngrams(filename bool) btreeIndex { @@ -412,9 +436,10 @@ func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResult // bucket (which can cause disk IO). slices.SortFunc(ngramOffs, runeNgramOff.Compare) frequencies := make([]uint32, 0, len(ngramOffs)) + indexMap := make([]int, len(ngramOffs)) ngramLookups := 0 ngrams := d.ngrams(query.FileName) - for _, o := range ngramOffs { + for i, o := range ngramOffs { var freq uint32 if query.CaseSensitive { freq = ngrams.Get(o.ngram).sz @@ -438,15 +463,14 @@ func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResult } frequencies = append(frequencies, freq) + indexMap[o.index] = i } - // first and last are now the smallest trigram posting lists to iterate - // through. - first, last := minFrequencyNgramOffsets(ngramOffs, frequencies) + first, last := findSelectiveNgrams(ngramOffs, indexMap, frequencies) iter := &ngramDocIterator{ - leftPad: first.index, - rightPad: uint32(utf8.RuneCountInString(str)) - first.index, + leftPad: uint32(first.index), + rightPad: uint32(utf8.RuneCountInString(str) - first.index), ngramLookups: ngramLookups, } if query.FileName { @@ -456,7 +480,7 @@ func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResult } if first != last { - runeDist := last.index - first.index + runeDist := uint32(last.index - first.index) i, err := d.newDistanceTrigramIter(first.ngram, last.ngram, runeDist, query.CaseSensitive, query.FileName) if err != nil { return nil, err diff --git a/indexdata_test.go b/indexdata_test.go index d4f8e1182..1e8b07966 100644 --- a/indexdata_test.go +++ b/indexdata_test.go @@ -72,3 +72,31 @@ func TestMinFrequencyNgramOffsets(t *testing.T) { t.Fatal(err) } } + +func TestFindSelectiveNGrams(t *testing.T) { + if err := quick.Check(func(s string, maxFreq uint16) bool { + ngramOffs := splitNGrams([]byte(s)) + if len(ngramOffs) == 0 { + return true + } + + slices.SortFunc(ngramOffs, runeNgramOff.Compare) + indexMap := make([]int, len(ngramOffs)) + for i, n := range ngramOffs { + indexMap[n.index] = i + } + + frequencies := genFrequencies(ngramOffs, int(maxFreq)) + x0, x1 := findSelectiveNgrams(ngramOffs, indexMap, frequencies) + + if len(ngramOffs) <= 1 { + return true + } + + // Just assert the invariant that x0 is before x1. This test mostly checks + // for out-of-bounds errors. + return x0.index < x1.index + }, nil); err != nil { + t.Fatal(err) + } +} From df7a7e7162cf7d7af4d4cdde3701c57950830676 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Wed, 15 May 2024 09:08:24 -0700 Subject: [PATCH 16/31] Simplify trigram selection in distanceHitIterator (#782) Follow up to #779. This PR removes the logic for trigrams with the same frequency, because it will no longer have a big effect. --- indexdata.go | 52 ++++++++++++++++------------------------------------ 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/indexdata.go b/indexdata.go index 2fdd6ef5a..20ea48887 100644 --- a/indexdata.go +++ b/indexdata.go @@ -320,26 +320,9 @@ func (d *indexData) memoryUse() int { return sz } -const maxUInt32 = 0xffffffff - -func min2Index(xs []uint32) (idx0, idx1 int) { - min0, min1 := uint32(maxUInt32), uint32(maxUInt32) - for i, x := range xs { - if x <= min0 { - idx0, idx1 = i, idx0 - min0, min1 = x, min0 - } else if x <= min1 { - idx1 = i - min1 = x - } - } - return -} - // findSelectiveNgrams returns two ngrams to pass to the distance iterator, chosen to -// produce a small file intersection. It finds the two lowest frequency ngrams, making -// sure to maximize the distance between them in case of ties. It avoids overlapping -// trigrams to keep their intersection as small as possible. +// produce a small file intersection. It finds the two lowest frequency ngrams, but avoids +// overlapping trigrams to keep their intersection as small as possible. // // Invariant: first will always have a smaller index than last. func findSelectiveNgrams(ngramOffs []runeNgramOff, indexMap []int, frequencies []uint32) (first, last runeNgramOff) { @@ -361,27 +344,24 @@ func findSelectiveNgrams(ngramOffs []runeNgramOff, indexMap []int, frequencies [ return } +const maxUInt32 = 0xffffffff + func minFrequencyNgramOffsets(ngramOffs []runeNgramOff, frequencies []uint32) (first, last runeNgramOff) { - firstI, lastI := min2Index(frequencies) - // If the frequencies are equal lets maximise distance in the query - // string. This optimization normally triggers for long repeated trigrams - // in a string, eg a query like "AAAAA..." - if frequencies[firstI] == frequencies[lastI] { - for i, freq := range frequencies { - if freq != frequencies[firstI] { - continue - } - if ngramOffs[i].index < ngramOffs[firstI].index { - firstI = i - } - if ngramOffs[i].index > ngramOffs[lastI].index { - lastI = i - } + // Find the two lowest frequency ngrams. + idx0, idx1 := 0, 0 + min0, min1 := uint32(maxUInt32), uint32(maxUInt32) + for i, x := range frequencies { + if x <= min0 { + idx0, idx1 = i, idx0 + min0, min1 = x, min0 + } else if x <= min1 { + idx1 = i + min1 = x } } - first = ngramOffs[firstI] - last = ngramOffs[lastI] + first = ngramOffs[idx0] + last = ngramOffs[idx1] // Ensure first appears before last as a helpful invariant. if first.index > last.index { From 640102a4a30e3913b4d498a465bb2ecd4f9cb7a3 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 28 May 2024 14:51:34 -0700 Subject: [PATCH 17/31] Increase filename boost (#785) When we introduced filename boosting in BM25, we set it to a very conservative weight. This PR increases the weight from 2.0 -> 5.0, which improves results on relevant evals. Relates to SPLF-88 --- build/scoring_test.go | 6 +++--- score.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/scoring_test.go b/build/scoring_test.go index 03a139281..e4e2e51e6 100644 --- a/build/scoring_test.go +++ b/build/scoring_test.go @@ -78,7 +78,7 @@ func TestBM25(t *testing.T) { content: exampleJava, language: "Java", // bm25-score:1.69 (sum-tf: 7.00, length-ratio: 2.00) - wantScore: 1.69, + wantScore: 1.82, }, { // Matches only on content fileName: "example.java", @@ -99,7 +99,7 @@ func TestBM25(t *testing.T) { content: exampleJava, language: "Java", // bm25-score:1.07 (sum-tf: 2.00, length-ratio: 2.00) - wantScore: 1.07, + wantScore: 1.55, }, { // Matches only on filename, and content is missing @@ -107,7 +107,7 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "config.go"}, language: "Go", // bm25-score:1.91 (sum-tf: 2.00, length-ratio: 0.00) - wantScore: 1.91, + wantScore: 2.08, }, } diff --git a/score.go b/score.go index dc0f4c192..9bcf1bbcd 100644 --- a/score.go +++ b/score.go @@ -132,7 +132,7 @@ func (d *indexData) scoreFileUsingBM25(fileMatch *FileMatch, doc uint32, cands [ term := string(cand.substrLowered) if cand.fileName { - termFreqs[term] += 2 + termFreqs[term] += 5 } else { termFreqs[term]++ } From f9834a41e90073c49a727b1219dec51fca882edb Mon Sep 17 00:00:00 2001 From: Ian Kerins Date: Wed, 29 May 2024 12:09:29 -0400 Subject: [PATCH 18/31] Make IndexGitRepo signal whether there were changes (#781) As described, we'd like this information to be externally available to drive metrics for incremental indexing. Closes #780. --- cmd/zoekt-git-index/main.go | 2 +- gitindex/ignore_test.go | 2 +- gitindex/index.go | 32 ++++++++++++++++++-------------- gitindex/index_test.go | 4 ++-- gitindex/tree_test.go | 16 ++++++++-------- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cmd/zoekt-git-index/main.go b/cmd/zoekt-git-index/main.go index 1d7cfe88b..c1aa5d1ad 100644 --- a/cmd/zoekt-git-index/main.go +++ b/cmd/zoekt-git-index/main.go @@ -122,7 +122,7 @@ func run() int { DeltaShardNumberFallbackThreshold: *deltaShardNumberFallbackThreshold, } - if err := gitindex.IndexGitRepo(gitOpts); err != nil { + if _, err := gitindex.IndexGitRepo(gitOpts); err != nil { log.Printf("indexGitRepo(%s, delta=%t): %v", dir, gitOpts.BuildOptions.IsDelta, err) exitStatus = 1 } diff --git a/gitindex/ignore_test.go b/gitindex/ignore_test.go index ca19613d9..862507753 100644 --- a/gitindex/ignore_test.go +++ b/gitindex/ignore_test.go @@ -73,7 +73,7 @@ func TestIgnore(t *testing.T) { Submodules: true, Incremental: true, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } diff --git a/gitindex/index.go b/gitindex/index.go index eb2588b3c..5a295d05c 100644 --- a/gitindex/index.go +++ b/gitindex/index.go @@ -388,12 +388,16 @@ func expandBranches(repo *git.Repository, bs []string, prefix string) ([]string, } // IndexGitRepo indexes the git repository as specified by the options. -func IndexGitRepo(opts Options) error { +// The returned bool indicates whether the index was updated as a result. This +// can be informative if doing incremental indexing. +func IndexGitRepo(opts Options) (bool, error) { return indexGitRepo(opts, gitIndexConfig{}) } // indexGitRepo indexes the git repository as specified by the options and the provided gitIndexConfig. -func indexGitRepo(opts Options, config gitIndexConfig) error { +// The returned bool indicates whether the index was updated as a result. This +// can be informative if doing incremental indexing. +func indexGitRepo(opts Options, config gitIndexConfig) (bool, error) { prepareDeltaBuild := prepareDeltaBuild if config.prepareDeltaBuild != nil { prepareDeltaBuild = config.prepareDeltaBuild @@ -407,13 +411,13 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { // Set max thresholds, since we use them in this function. opts.BuildOptions.SetDefaults() if opts.RepoDir == "" { - return fmt.Errorf("gitindex: must set RepoDir") + return false, fmt.Errorf("gitindex: must set RepoDir") } opts.BuildOptions.RepositoryDescription.Source = opts.RepoDir repo, err := git.PlainOpen(opts.RepoDir) if err != nil { - return fmt.Errorf("git.PlainOpen: %w", err) + return false, fmt.Errorf("git.PlainOpen: %w", err) } if err := setTemplatesFromConfig(&opts.BuildOptions.RepositoryDescription, opts.RepoDir); err != nil { @@ -422,7 +426,7 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { branches, err := expandBranches(repo, opts.Branches, opts.BranchPrefix) if err != nil { - return fmt.Errorf("expandBranches: %w", err) + return false, fmt.Errorf("expandBranches: %w", err) } for _, b := range branches { commit, err := getCommit(repo, opts.BranchPrefix, b) @@ -431,7 +435,7 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { continue } - return fmt.Errorf("getCommit(%q, %q): %w", opts.BranchPrefix, b, err) + return false, fmt.Errorf("getCommit(%q, %q): %w", opts.BranchPrefix, b, err) } opts.BuildOptions.RepositoryDescription.Branches = append(opts.BuildOptions.RepositoryDescription.Branches, zoekt.RepositoryBranch{ @@ -445,7 +449,7 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { } if opts.Incremental && opts.BuildOptions.IncrementalSkipIndexing() { - return nil + return false, nil } // branch => (path, sha1) => repo. @@ -474,7 +478,7 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { if !opts.BuildOptions.IsDelta { repos, branchMap, branchVersions, err = prepareNormalBuild(opts, repo) if err != nil { - return fmt.Errorf("preparing normal build: %w", err) + return false, fmt.Errorf("preparing normal build: %w", err) } } @@ -507,7 +511,7 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { builder, err := build.NewBuilder(opts.BuildOptions) if err != nil { - return fmt.Errorf("build.NewBuilder: %w", err) + return false, fmt.Errorf("build.NewBuilder: %w", err) } var ranks repoPathRanks @@ -515,12 +519,12 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { if opts.BuildOptions.DocumentRanksPath != "" { data, err := os.ReadFile(opts.BuildOptions.DocumentRanksPath) if err != nil { - return err + return false, err } err = json.Unmarshal(data, &ranks) if err != nil { - return err + return false, err } // Compute the mean rank for this repository. Note: we overwrite the rank @@ -564,16 +568,16 @@ func indexGitRepo(opts Options, config gitIndexConfig) error { for _, key := range keys { doc, err := createDocument(key, repos, branchMap, ranks, opts.BuildOptions) if err != nil { - return err + return false, err } if err := builder.Add(doc); err != nil { - return fmt.Errorf("error adding document with name %s: %w", key.FullPath(), err) + return false, fmt.Errorf("error adding document with name %s: %w", key.FullPath(), err) } } } - return builder.Finish() + return true, builder.Finish() } type repoPathRanks struct { diff --git a/gitindex/index_test.go b/gitindex/index_test.go index bf5993717..63f5899a7 100644 --- a/gitindex/index_test.go +++ b/gitindex/index_test.go @@ -56,7 +56,7 @@ func TestIndexEmptyRepo(t *testing.T) { }, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } } @@ -619,7 +619,7 @@ func TestIndexDeltaBasic(t *testing.T) { } // run test - err := indexGitRepo(options, gitIndexConfig{ + _, err := indexGitRepo(options, gitIndexConfig{ prepareDeltaBuild: prepareDeltaSpy, prepareNormalBuild: prepareNormalSpy, }) diff --git a/gitindex/tree_test.go b/gitindex/tree_test.go index 64fe07dc1..406c4b175 100644 --- a/gitindex/tree_test.go +++ b/gitindex/tree_test.go @@ -213,7 +213,7 @@ func TestSubmoduleIndex(t *testing.T) { Incremental: true, RepoCacheDir: dir, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } @@ -317,7 +317,7 @@ func TestSearchSymlinkByContent(t *testing.T) { Incremental: true, RepoCacheDir: dir, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } @@ -375,11 +375,11 @@ func TestAllowMissingBranch(t *testing.T) { Incremental: true, RepoCacheDir: dir, } - if err := IndexGitRepo(opts); err == nil { + if _, err := IndexGitRepo(opts); err == nil { t.Fatalf("IndexGitRepo(nonexist) succeeded") } opts.AllowMissingBranch = true - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo(nonexist, allow): %v", err) } } @@ -444,7 +444,7 @@ func TestBranchWildcard(t *testing.T) { Submodules: true, Incremental: true, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } @@ -492,7 +492,7 @@ func TestSkipSubmodules(t *testing.T) { Branches: []string{"master"}, Submodules: false, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } } @@ -523,7 +523,7 @@ func TestFullAndShortRefNames(t *testing.T) { Incremental: false, AllowMissingBranch: false, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } @@ -573,7 +573,7 @@ func TestLatestCommit(t *testing.T) { BranchPrefix: "refs/heads", Branches: []string{"branchdir/a", "branchdir/b"}, } - if err := IndexGitRepo(opts); err != nil { + if _, err := IndexGitRepo(opts); err != nil { t.Fatalf("IndexGitRepo: %v", err) } From b8ea92de2f6c3fc952e9ec3d1df1f58be9a40d80 Mon Sep 17 00:00:00 2001 From: Max McDonnell Date: Fri, 31 May 2024 04:04:30 -0400 Subject: [PATCH 19/31] Allow wasm compilation (#786) --- build/builder_unix.go | 4 ++-- tombstones_unix.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/builder_unix.go b/build/builder_unix.go index cd50632dd..edbd87b06 100644 --- a/build/builder_unix.go +++ b/build/builder_unix.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -//go:build !windows -// +build !windows +//go:build !windows && !wasm +// +build !windows,!wasm package build diff --git a/tombstones_unix.go b/tombstones_unix.go index e21f42c5f..f983b5096 100644 --- a/tombstones_unix.go +++ b/tombstones_unix.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !wasm package zoekt From b41ceb2c666034c5916a14f5b3670709e07c5803 Mon Sep 17 00:00:00 2001 From: Matthew Manela Date: Fri, 7 Jun 2024 10:40:31 -0400 Subject: [PATCH 20/31] Update VSCode launch.json to enable easier debugging (#789) * Update VSCode launch.json to enable easier debugging * Add index path --- .vscode/launch.json | 58 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index dffa3c19e..6c254f63b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,14 +1,46 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Attach to Process (from list)", - "type": "go", - "request": "attach", - "mode": "local" - }, - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Index folder", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "cmd/zoekt-git-index", + "cwd": "${workspaceFolder}", + "args": ["-index", "${input:indexPath}", "${input:path}"] + }, + { + "name": "Webserver", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "cmd/zoekt-webserver", + "cwd": "${workspaceFolder}", + "args": ["-index", "${input:indexPath}"] + }, + { + "name": "Attach to Process (from list)", + "type": "go", + "request": "attach", + "mode": "local" + } + ], + "inputs": [ + { + "id": "path", + "description": "Please enter the path to the project to index", + "default": "", + "type": "promptString" + }, + { + "id": "indexPath", + "description": "Enter the path where indexes are stored", + "default": "${userHome}/.zoekt", + "type": "promptString" + } + ] +} From 376af3a6d529e59b4acad8128477f3cfcb57fd92 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Mon, 10 Jun 2024 12:31:42 +0200 Subject: [PATCH 21/31] ranking: add IDF to BM25 score calculation (#788) So far, we didn't include IDF in our BM25 score function. Zoekt uses a trigram index and hence doesn't compute document frequency during indexing. We could add this information to the index, but it is not immediately obvious how to tokenize code in a way that is compatible with tokens from a natural language query. Here we calulate the document frequency at query time under the assumption that we visit all documents containing any of the query terms. Notes: Also fixed an off-by-1 bug with how we count documents. Test plan: - Updated unit test - Context evaluation results are slightly worse with a decrease from 64/89 to 63/89 --- api.go | 12 ++++-- build/scoring_test.go | 16 ++++---- eval.go | 32 +++++++++++++-- score.go | 94 +++++++++++++++++++++++++++---------------- score_test.go | 51 +++++++++++++++++++++++ 5 files changed, 156 insertions(+), 49 deletions(-) create mode 100644 score_test.go diff --git a/api.go b/api.go index 192b6a834..1e478dfa7 100644 --- a/api.go +++ b/api.go @@ -946,9 +946,15 @@ type SearchOptions struct { // will be used. This option is temporary and is only exposed for testing/ tuning purposes. DocumentRanksWeight float64 - // EXPERIMENTAL. If true, use text-search style scoring instead of the default scoring formula. - // The scoring algorithm treats each match in a file as a term and computes an approximation to - // BM25. When enabled, all other scoring signals are ignored, including document ranks. + // EXPERIMENTAL. If true, use text-search style scoring instead of the default + // scoring formula. The scoring algorithm treats each match in a file as a term + // and computes an approximation to BM25. + // + // The calculation of IDF assumes that Zoekt visits all documents containing any + // of the query terms during evaluation. This is true, for example, if all query + // terms are ORed together. + // + // When enabled, all other scoring signals are ignored, including document ranks. UseBM25Scoring bool // Trace turns on opentracing for this request if true and if the Jaeger address was provided as diff --git a/build/scoring_test.go b/build/scoring_test.go index e4e2e51e6..8dc2f2f93 100644 --- a/build/scoring_test.go +++ b/build/scoring_test.go @@ -77,8 +77,8 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "example"}, content: exampleJava, language: "Java", - // bm25-score:1.69 (sum-tf: 7.00, length-ratio: 2.00) - wantScore: 1.82, + // bm25-score: 0.57 <- sum-termFrequencyScore: 10.00, length-ratio: 1.00 + wantScore: 0.57, }, { // Matches only on content fileName: "example.java", @@ -89,8 +89,8 @@ func TestBM25(t *testing.T) { }}, content: exampleJava, language: "Java", - // bm25-score:5.75 (sum-tf: 56.00, length-ratio: 2.00) - wantScore: 5.75, + // bm25-score: 1.75 <- sum-termFrequencyScore: 56.00, length-ratio: 1.00 + wantScore: 1.75, }, { // Matches only on filename @@ -98,16 +98,16 @@ func TestBM25(t *testing.T) { query: &query.Substring{Pattern: "java"}, content: exampleJava, language: "Java", - // bm25-score:1.07 (sum-tf: 2.00, length-ratio: 2.00) - wantScore: 1.55, + // bm25-score: 0.51 <- sum-termFrequencyScore: 5.00, length-ratio: 1.00 + wantScore: 0.51, }, { // Matches only on filename, and content is missing fileName: "a/b/c/config.go", query: &query.Substring{Pattern: "config.go"}, language: "Go", - // bm25-score:1.91 (sum-tf: 2.00, length-ratio: 0.00) - wantScore: 2.08, + // bm25-score: 0.60 <- sum-termFrequencyScore: 5.00, length-ratio: 0.00 + wantScore: 0.60, }, } diff --git a/eval.go b/eval.go index 0d8ec91bd..af637f01e 100644 --- a/eval.go +++ b/eval.go @@ -197,6 +197,12 @@ func (d *indexData) Search(ctx context.Context, q query.Q, opts *SearchOptions) docCount := uint32(len(d.fileBranchMasks)) lastDoc := int(-1) + // document frequency per term + df := make(termDocumentFrequency) + + // term frequency per file match + var tfs []termFrequency + nextFileMatch: for { canceled := false @@ -317,8 +323,14 @@ nextFileMatch: fileMatch.LineMatches = cp.fillMatches(finalCands, opts.NumContextLines, fileMatch.Language, opts.DebugScore) } + var tf map[string]int if opts.UseBM25Scoring { - d.scoreFileUsingBM25(&fileMatch, nextDoc, finalCands, opts) + // For BM25 scoring, the calculation of the score is split in two parts. Here we + // calculate the term frequencies for the current document and update the + // document frequencies. Since we don't store document frequencies in the index, + // we have to defer the calculation of the final BM25 score to after the whole + // shard has been processed. + tf = calculateTermFrequency(finalCands, df) } else { // Use the standard, non-experimental scoring method by default d.scoreFile(&fileMatch, nextDoc, mt, known, opts) @@ -339,16 +351,28 @@ nextFileMatch: repoMatchCount += len(fileMatch.LineMatches) repoMatchCount += matchedChunkRanges - if opts.DebugScore { - fileMatch.Debug = fmt.Sprintf("score:%.2f <- %s", fileMatch.Score, fileMatch.Debug) + if opts.UseBM25Scoring { + // Invariant: tfs[i] belongs to res.Files[i] + tfs = append(tfs, termFrequency{ + doc: nextDoc, + tf: tf, + }) } - res.Files = append(res.Files, fileMatch) + res.Stats.MatchCount += len(fileMatch.LineMatches) res.Stats.MatchCount += matchedChunkRanges res.Stats.FileCount++ } + // Calculate BM25 score for all file matches in the shard. We assume that we + // have seen all documents containing any of the terms in the query so that df + // correctly reflects the document frequencies. This is true, for example, if + // all terms in the query are ORed together. + if opts.UseBM25Scoring { + d.scoreFilesUsingBM25(res.Files, tfs, df, opts) + } + for _, md := range d.repoMetaData { r := md addRepo(&res, &r) diff --git a/score.go b/score.go index 9bcf1bbcd..a2579df2b 100644 --- a/score.go +++ b/score.go @@ -39,13 +39,6 @@ func (m *FileMatch) addScore(what string, computed float64, raw float64, debugSc m.Score += computed } -func (m *FileMatch) addBM25Score(score float64, sumTf float64, L float64, debugScore bool) { - if debugScore { - m.Debug += fmt.Sprintf("bm25-score:%.2f (sum-tf: %.2f, length-ratio: %.2f)", score, sumTf, L) - } - m.Score += score -} - // scoreFile computes a score for the file match using various scoring signals, like // whether there's an exact match on a symbol, the number of query clauses that matched, etc. func (d *indexData) scoreFile(fileMatch *FileMatch, doc uint32, mt matchTree, known map[matchTree]bool, opts *SearchOptions) { @@ -111,26 +104,20 @@ func (d *indexData) scoreFile(fileMatch *FileMatch, doc uint32, mt matchTree, kn addScore("repo-rank", scoreRepoRankFactor*float64(md.Rank)/maxUInt16) if opts.DebugScore { - fileMatch.Debug = strings.TrimSuffix(fileMatch.Debug, ", ") + fileMatch.Debug = fmt.Sprintf("score: %.2f <- %s", fileMatch.Score, strings.TrimSuffix(fileMatch.Debug, ", ")) } } -// scoreFileUsingBM25 computes a score for the file match using an approximation to BM25, the most common scoring -// algorithm for text search: https://en.wikipedia.org/wiki/Okapi_BM25. It implements all parts of the formula -// except inverse document frequency (idf), since we don't have access to global term frequency statistics. -// -// Filename matches count twice as much as content matches. This mimics a common text search strategy where you -// 'boost' matches on document titles. +// calculateTermFrequency computes the term frequency for the file match. // -// This scoring strategy ignores all other signals including document ranks. This keeps things simple for now, -// since BM25 is not normalized and can be tricky to combine with other scoring signals. -func (d *indexData) scoreFileUsingBM25(fileMatch *FileMatch, doc uint32, cands []*candidateMatch, opts *SearchOptions) { +// Filename matches count more than content matches. This mimics a common text +// search strategy where you 'boost' matches on document titles. +func calculateTermFrequency(cands []*candidateMatch, df termDocumentFrequency) map[string]int { // Treat each candidate match as a term and compute the frequencies. For now, ignore case // sensitivity and treat filenames and symbols the same as content. termFreqs := map[string]int{} for _, cand := range cands { term := string(cand.substrLowered) - if cand.fileName { termFreqs[term] += 5 } else { @@ -138,27 +125,66 @@ func (d *indexData) scoreFileUsingBM25(fileMatch *FileMatch, doc uint32, cands [ } } - // Compute the file length ratio. Usually the calculation would be based on terms, but using - // bytes should work fine, as we're just computing a ratio. - fileLength := float64(d.boundaries[doc+1] - d.boundaries[doc]) - numFiles := len(d.boundaries) - averageFileLength := float64(d.boundaries[numFiles-1]) / float64(numFiles) + for term := range termFreqs { + df[term] += 1 + } + + return termFreqs +} + +// idf computes the inverse document frequency for a term. nq is the number of +// documents that contain the term and documentCount is the total number of +// documents in the corpus. +func idf(nq, documentCount int) float64 { + return math.Log(1.0 + ((float64(documentCount) - float64(nq) + 0.5) / (float64(nq) + 0.5))) +} + +// termDocumentFrequency is a map "term" -> "number of documents that contain the term" +type termDocumentFrequency map[string]int +// termFrequency stores the term frequencies for doc. +type termFrequency struct { + doc uint32 + tf map[string]int +} + +// scoreFilesUsingBM25 computes the score according to BM25, the most common +// scoring algorithm for text search: https://en.wikipedia.org/wiki/Okapi_BM25. +// +// This scoring strategy ignores all other signals including document ranks. +// This keeps things simple for now, since BM25 is not normalized and can be +// tricky to combine with other scoring signals. +func (d *indexData) scoreFilesUsingBM25(fileMatches []FileMatch, tfs []termFrequency, df termDocumentFrequency, opts *SearchOptions) { + // Use standard parameter defaults (used in Lucene and academic papers) + k, b := 1.2, 0.75 + + averageFileLength := float64(d.boundaries[d.numDocs()]) / float64(d.numDocs()) // This is very unlikely, but explicitly guard against division by zero. if averageFileLength == 0 { averageFileLength++ } - L := fileLength / averageFileLength - // Use standard parameter defaults (used in Lucene and academic papers) - k, b := 1.2, 0.75 - sumTf := 0.0 // Just for debugging - score := 0.0 - for _, freq := range termFreqs { - tf := float64(freq) - sumTf += tf - score += ((k + 1.0) * tf) / (k*(1.0-b+b*L) + tf) - } + for i := range tfs { + score := 0.0 + + // Compute the file length ratio. Usually the calculation would be based on terms, but using + // bytes should work fine, as we're just computing a ratio. + doc := tfs[i].doc + fileLength := float64(d.boundaries[doc+1] - d.boundaries[doc]) - fileMatch.addBM25Score(score, sumTf, L, opts.DebugScore) + L := fileLength / averageFileLength + + sumTF := 0 // Just for debugging + for term, f := range tfs[i].tf { + sumTF += f + tfScore := ((k + 1.0) * float64(f)) / (k*(1.0-b+b*L) + float64(f)) + score += idf(df[term], int(d.numDocs())) * tfScore + } + + fileMatches[i].Score = score + + if opts.DebugScore { + fileMatches[i].Debug = fmt.Sprintf("bm25-score: %.2f <- sum-termFrequencies: %d, length-ratio: %.2f", score, sumTF, L) + } + } } diff --git a/score_test.go b/score_test.go new file mode 100644 index 000000000..2e3b13840 --- /dev/null +++ b/score_test.go @@ -0,0 +1,51 @@ +package zoekt + +import ( + "maps" + "testing" +) + +func TestCalculateTermFrequency(t *testing.T) { + cases := []struct { + cands []*candidateMatch + wantDF termDocumentFrequency + wantTermFrequencies map[string]int + }{{ + cands: []*candidateMatch{ + {substrLowered: []byte("foo")}, + {substrLowered: []byte("foo")}, + {substrLowered: []byte("bar")}, + { + substrLowered: []byte("bas"), + fileName: true, + }, + }, + wantDF: termDocumentFrequency{ + "foo": 1, + "bar": 1, + "bas": 1, + }, + wantTermFrequencies: map[string]int{ + "foo": 2, + "bar": 1, + "bas": 5, + }, + }, + } + + for _, c := range cases { + t.Run("", func(t *testing.T) { + fm := FileMatch{} + df := make(termDocumentFrequency) + tf := calculateTermFrequency(c.cands, df) + + if !maps.Equal(df, c.wantDF) { + t.Errorf("got %v, want %v", df, c.wantDF) + } + + if !maps.Equal(tf, c.wantTermFrequencies) { + t.Errorf("got %v, want %v", fm, c.wantTermFrequencies) + } + }) + } +} From c21df410093acfc887b458f901ada70aded67aee Mon Sep 17 00:00:00 2001 From: Matthew Manela Date: Tue, 11 Jun 2024 13:41:02 -0400 Subject: [PATCH 22/31] Add support for Magik and PKl languages that are not handled by Linguist (#790) Add fallbacks for languages not supported yet by linguist or go-enry --- index_test.go | 4 ++ indexbuilder.go | 4 +- internal/languages/language.go | 66 ++++++++++++++++++++ internal/languages/language_test.go | 95 +++++++++++++++++++++++++++++ query/parse.go | 4 +- 5 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 internal/languages/language.go create mode 100644 internal/languages/language_test.go diff --git a/index_test.go b/index_test.go index 05401215f..5aeec11ff 100644 --- a/index_test.go +++ b/index_test.go @@ -3449,6 +3449,7 @@ func TestSearchTypeLanguage(t *testing.T) { Document{Name: "apex.cls", Content: []byte("public class Car extends Vehicle {")}, Document{Name: "tex.cls", Content: []byte(`\DeclareOption*{`)}, Document{Name: "hello.h", Content: []byte(`#include `)}, + Document{Name: "be.magik", Content: []byte(`_package unicorn`)}, ) t.Log(b.languageMap) @@ -3486,6 +3487,9 @@ func TestSearchTypeLanguage(t *testing.T) { res = searchForTest(t, b, &query.Language{Language: "C"}) wantSingleMatch(res, "hello.h") + res = searchForTest(t, b, &query.Language{Language: "Magik"}) + wantSingleMatch(res, "be.magik") + // test fallback language search by pretending it's an older index version res = searchForTest(t, b, &query.Language{Language: "C++"}) if len(res.Files) != 0 { diff --git a/indexbuilder.go b/indexbuilder.go index 6a7b5ea1a..026fd7e8d 100644 --- a/indexbuilder.go +++ b/indexbuilder.go @@ -27,7 +27,7 @@ import ( "time" "unicode/utf8" - "github.com/go-enry/go-enry/v2" + "github.com/sourcegraph/zoekt/internal/languages" ) var _ = log.Println @@ -397,7 +397,7 @@ func (b *IndexBuilder) addSymbols(symbols []*Symbol) { func DetermineLanguageIfUnknown(doc *Document) { if doc.Language == "" { - doc.Language = enry.GetLanguage(doc.Name, doc.Content) + doc.Language = languages.GetLanguage(doc.Name, doc.Content) } } diff --git a/internal/languages/language.go b/internal/languages/language.go new file mode 100644 index 000000000..7f1110757 --- /dev/null +++ b/internal/languages/language.go @@ -0,0 +1,66 @@ +// This file wraps the logic of go-enry (https://github.com/go-enry/go-enry) to support additional languages. +// go-enry is based off of a package called Linguist (https://github.com/github/linguist) +// and sometimes programming languages may not be supported by Linguist +// or may take a while to get merged in and make it into go-enry. This wrapper +// gives us flexibility to support languages in those cases. We list additional languages +// in this file and remove them once they make it into Linguist and go-enry. +// This logic is similar to what we have in the sourcegraph/sourcegraph repo, in the future +// we plan to refactor both into a common library to share between the two repos. +package languages + +import ( + "path/filepath" + "strings" + + "github.com/go-enry/go-enry/v2" +) + +var unsupportedByLinguistAliasMap = map[string]string{ + // Pkl Configuration Language (https://pkl-lang.org/) + // Add to linguist on 6/7/24 + // can remove once go-enry package updates + // to that linguist version + "pkl": "Pkl", + // Magik Language + "magik": "Magik", +} + +var unsupportedByLinguistExtensionToNameMap = map[string]string{ + // Pkl Configuration Language (https://pkl-lang.org/) + ".pkl": "Pkl", + // Magik Language + ".magik": "Magik", +} + +// getLanguagesByAlias is a replacement for enry.GetLanguagesByAlias +// It supports languages that are missing in linguist +func GetLanguageByAlias(alias string) (language string, ok bool) { + language, ok = enry.GetLanguageByAlias(alias) + if !ok { + normalizedAlias := strings.ToLower(alias) + language, ok = unsupportedByLinguistAliasMap[normalizedAlias] + } + + return +} + +// GetLanguage is a replacement for enry.GetLanguage +// to find out the most probable language to return but includes support +// for languages missing from linguist +func GetLanguage(filename string, content []byte) (language string) { + language = enry.GetLanguage(filename, content) + + // If go-enry failed to find language, fall back on our + // internal check for languages missing in linguist + if language == "" { + ext := filepath.Ext(filename) + normalizedExt := strings.ToLower(ext) + if ext == "" { + return + } + if lang, ok := unsupportedByLinguistExtensionToNameMap[normalizedExt]; ok { + language = lang + } + } + return +} diff --git a/internal/languages/language_test.go b/internal/languages/language_test.go new file mode 100644 index 000000000..294c67776 --- /dev/null +++ b/internal/languages/language_test.go @@ -0,0 +1,95 @@ +package languages + +import "testing" + +func TestGetLanguageByAlias(t *testing.T) { + tests := []struct { + name string + alias string + want string + wantOk bool + }{ + { + name: "empty alias", + alias: "", + want: "", + wantOk: false, + }, + { + name: "unknown alias", + alias: "unknown", + want: "", + wantOk: false, + }, + { + name: "supported alias", + alias: "go", + want: "Go", + wantOk: true, + }, + { + name: "unsupported by linguist alias", + alias: "magik", + want: "Magik", + wantOk: true, + }, + { + name: "unsupported by linguist alias normalized", + alias: "mAgIk", + want: "Magik", + wantOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := GetLanguageByAlias(tt.alias) + if got != tt.want || ok != tt.wantOk { + t.Errorf("GetLanguageByAlias(%q) = %q, %t, want %q, %t", tt.alias, got, ok, tt.want, tt.wantOk) + } + }) + } +} + +func TestGetLanguage(t *testing.T) { + tests := []struct { + name string + filename string + content []byte + want string + }{ + { + name: "empty filename", + filename: "", + content: []byte(""), + want: "", + }, + { + name: "unknown extension", + filename: "file.unknown", + content: []byte(""), + want: "", + }, + { + name: "supported extension", + filename: "file.go", + content: []byte("package main"), + want: "Go", + }, + { + name: "unsupported by linguist extension", + filename: "file.magik", + content: []byte(""), + want: "Magik", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetLanguage(tt.filename, tt.content) + if got != tt.want { + t.Errorf("GetLanguage(%q, %q) = %q, want %q", tt.filename, tt.content, got, tt.want) + } + }) + } +} diff --git a/query/parse.go b/query/parse.go index 028017692..d8762f191 100644 --- a/query/parse.go +++ b/query/parse.go @@ -20,8 +20,8 @@ import ( "log" "regexp/syntax" - "github.com/go-enry/go-enry/v2" "github.com/grafana/regexp" + "github.com/sourcegraph/zoekt/internal/languages" ) var _ = log.Printf @@ -172,7 +172,7 @@ func parseExpr(in []byte) (Q, int, error) { } expr = q case tokLang: - canonical, ok := enry.GetLanguageByAlias(text) + canonical, ok := languages.GetLanguageByAlias(text) if !ok { expr = &Const{false} } else { From bc9f29b09a8ec1531f3d8d733555eef0afd864fe Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Thu, 20 Jun 2024 11:13:31 +0800 Subject: [PATCH 23/31] chore: Bump enry version in sync with Sourcegraph (#793) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 116179420..393760e08 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/fgprof v0.9.3 github.com/fsnotify/fsnotify v1.6.0 github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751 - github.com/go-enry/go-enry/v2 v2.8.4 + github.com/go-enry/go-enry/v2 v2.8.8 github.com/go-git/go-git/v5 v5.11.0 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.6.0 diff --git a/go.sum b/go.sum index cfa9b329c..4d94a1775 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-enry/v2 v2.8.8 h1:EhfxWpw4DQ3WEFB1Y77X8vKqZL0D0EDUUWYDUAIv9/4= +github.com/go-enry/go-enry/v2 v2.8.8/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= From 5ac92b1a7d4ab7b0dbeeaa9df77abb13d555e16b Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Thu, 20 Jun 2024 10:45:26 +0200 Subject: [PATCH 24/31] gomod: go get github.com/grafana/regexp@speedup (#794) Updates grafana/regexp to the latest version of the speedup branch. This incorporates the changes from upstream go that have happened. Test Plan: go test ./... --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 393760e08..b95ae577e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/go-github/v27 v27.0.6 github.com/google/slothfs v0.0.0-20190717100203-59c1163fd173 - github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db + github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f diff --git a/go.sum b/go.sum index 4d94a1775..fb0cfe1c0 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,6 @@ github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751 h1:Ea58sAu github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751/go.mod h1:IqOZzks2wlWCIai0esXnZPdPwxF2yOz0HcCYw5I4pCg= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= -github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= -github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-enry/v2 v2.8.8 h1:EhfxWpw4DQ3WEFB1Y77X8vKqZL0D0EDUUWYDUAIv9/4= github.com/go-enry/go-enry/v2 v2.8.8/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= @@ -183,8 +181,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA= -github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da h1:BML5sNe+bw2uO8t8cQSwe5QhvoP04eHPF7bnaQma0Kw= +github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= From 12ce07a298aed45c4ee9fa92f9acfdcb83f0836f Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Fri, 26 Jul 2024 18:01:08 +0200 Subject: [PATCH 25/31] index: experiment to limit ngram lookups for large snippets (#795) This introduces an experiment where we can stop looking up ngrams at a certain limit. The insight here is that for large substrings we spend more time finding the smallest ngram frequency than the time a normal search takes. So instead we can try and find a good balance between looking for a good (two) ngrams and actually searching the corpus. The plan is to set different values for SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT in sourcegraph production and see how it affects performance of attribution search service. Test Plan: ran all tests with the envvar set to 2. I expected tests that assert on stats to fail, but everything else to pass. This was the case. SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT=2 go test ./... --- bits.go | 7 ++++++- indexdata.go | 30 +++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/bits.go b/bits.go index 7d1363102..c3d17afee 100644 --- a/bits.go +++ b/bits.go @@ -124,6 +124,11 @@ func (a runeNgramOff) Compare(b runeNgramOff) int { } func splitNGrams(str []byte) []runeNgramOff { + // len(maxNgrams) >= the number of ngrams in str => no limit + return splitNGramsLimit(str, len(str)) +} + +func splitNGramsLimit(str []byte, maxNgrams int) []runeNgramOff { var runeGram [3]rune var off [3]uint32 var runeCount int @@ -131,7 +136,7 @@ func splitNGrams(str []byte) []runeNgramOff { result := make([]runeNgramOff, 0, len(str)) var i uint32 - for len(str) > 0 { + for len(str) > 0 && len(result) < maxNgrams { r, sz := utf8.DecodeRune(str) str = str[sz:] runeGram[0] = runeGram[1] diff --git a/indexdata.go b/indexdata.go index 20ea48887..91314f12f 100644 --- a/indexdata.go +++ b/indexdata.go @@ -21,7 +21,9 @@ import ( "hash/crc64" "log" "math/bits" + "os" "slices" + "strconv" "unicode/utf8" "github.com/sourcegraph/zoekt/query" @@ -401,11 +403,37 @@ func (r *ngramIterationResults) candidates() []*candidateMatch { return cs } +// experimentIterateNgramLookupLimit when non-zero will only lookup this many +// ngrams from a query string. Note: that if case-insensitive, this only +// limits the input. So we will still lookup the case folding. +// +// This experiment is targetting looking up large snippets. If it is +// successful, we will likely hardcode the value we use in production. +// +// Future note: if we find cases where this works badly, we can consider only +// searching a random subset of the query string to avoid bad strings. +var experimentIterateNgramLookupLimit = getEnvInt("SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT") + +func getEnvInt(k string) int { + v, _ := strconv.Atoi(os.Getenv(k)) + if v != 0 { + log.Printf("%s = %d\n", k, v) + } + return v +} + func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResults, error) { str := query.Pattern // Find the 2 least common ngrams from the string. - ngramOffs := splitNGrams([]byte(query.Pattern)) + var ngramOffs []runeNgramOff + if ngramLimit := experimentIterateNgramLookupLimit; ngramLimit > 0 { + // Note: we can't just do str = str[:ngramLimit] due to utf-8 and str + // length is asked later on for other optimizations. + ngramOffs = splitNGramsLimit([]byte(str), ngramLimit) + } else { + ngramOffs = splitNGrams([]byte(str)) + } // protect against accidental searching of empty strings if len(ngramOffs) == 0 { From 04e7057ffe6b07323fdd605f65bddabfcb041398 Mon Sep 17 00:00:00 2001 From: po3rin Date: Mon, 29 Jul 2024 15:33:26 +0900 Subject: [PATCH 26/31] Enabling numContextLines in non-json format (#796) --- web/api.go | 1 + web/server.go | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/api.go b/web/api.go index 8d07ae307..4f8c1468c 100644 --- a/web/api.go +++ b/web/api.go @@ -28,6 +28,7 @@ type ApiSearchResult struct { type LastInput struct { Query string Num int + Ctx int // If set, focus on the search box. AutoFocus bool diff --git a/web/server.go b/web/server.go index 75b631fd2..89c9fed27 100644 --- a/web/server.go +++ b/web/server.go @@ -280,12 +280,10 @@ func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) { } numCtxLines := 0 - if qvals.Get("format") == "json" { - if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" { - numCtxLines, err = strconv.Atoi(ctxLinesStr) - if err != nil || numCtxLines < 0 || numCtxLines > 10 { - return nil, fmt.Errorf("Number of context lines must be between 0 and 10") - } + if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" { + numCtxLines, err = strconv.Atoi(ctxLinesStr) + if err != nil || numCtxLines < 0 || numCtxLines > 10 { + return nil, fmt.Errorf("Number of context lines must be between 0 and 10") } } sOpts.NumContextLines = numCtxLines @@ -313,6 +311,7 @@ func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) { Last: LastInput{ Query: queryStr, Num: num, + Ctx: numCtxLines, AutoFocus: true, }, Stats: result.Stats, From ebb3ca24249dfa7d3be2bb48899b86e68bed2f6c Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Mon, 29 Jul 2024 14:53:13 +0200 Subject: [PATCH 27/31] index: use a random sample of ngrams when limiting (#797) The first bit of data I am getting back indicates this strategy of limiting the number of ngrams we lookup isn't working. I am still experimenting with different limits, but in the meantime it is easy to implement a strategy which picks a random subset. This is so that the first N ngrams of a query aren't the only ones being consulted. Test Plan: ran all tests with the envvar set to 2. I expected tests that assert on stats to fail, but everything else to pass. This was the case. SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT=2 go test ./... --- bits.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/bits.go b/bits.go index c3d17afee..5fc3e2889 100644 --- a/bits.go +++ b/bits.go @@ -18,6 +18,8 @@ import ( "cmp" "encoding/binary" "math" + "math/rand/v2" + "slices" "sort" "unicode" "unicode/utf8" @@ -136,7 +138,7 @@ func splitNGramsLimit(str []byte, maxNgrams int) []runeNgramOff { result := make([]runeNgramOff, 0, len(str)) var i uint32 - for len(str) > 0 && len(result) < maxNgrams { + for len(str) > 0 { r, sz := utf8.DecodeRune(str) str = str[sz:] runeGram[0] = runeGram[1] @@ -157,6 +159,22 @@ func splitNGramsLimit(str []byte, maxNgrams int) []runeNgramOff { index: len(result), }) } + + // We return a random subset of size maxNgrams. This is to prevent the start + // of the string biasing ngram selection. + if maxNgrams < len(result) { + // Deterministic seed for tests. Additionally makes comparing repeated + // queries performance easier. + r := rand.New(rand.NewPCG(uint64(maxNgrams), 0)) + + // Pick random subset via a shuffle + r.Shuffle(maxNgrams, func(i, j int) { result[i], result[j] = result[j], result[i] }) + result = result[:maxNgrams] + + // Caller expects ngrams in order of appearance. + slices.SortFunc(result, runeNgramOff.Compare) + } + return result } From c01b6c7778a6cd60b7a6be8db0a9f140560060d8 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Thu, 1 Aug 2024 16:12:43 +0200 Subject: [PATCH 28/31] remove SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT (#800) After defaulting to shard merging for all inactive repos, this in fact makes searches slightly slower. So we can remove the experiment. Test Plan: go test --- bits.go | 22 ---------------------- indexdata.go | 30 +----------------------------- 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/bits.go b/bits.go index 5fc3e2889..d438cbf15 100644 --- a/bits.go +++ b/bits.go @@ -18,8 +18,6 @@ import ( "cmp" "encoding/binary" "math" - "math/rand/v2" - "slices" "sort" "unicode" "unicode/utf8" @@ -126,11 +124,6 @@ func (a runeNgramOff) Compare(b runeNgramOff) int { } func splitNGrams(str []byte) []runeNgramOff { - // len(maxNgrams) >= the number of ngrams in str => no limit - return splitNGramsLimit(str, len(str)) -} - -func splitNGramsLimit(str []byte, maxNgrams int) []runeNgramOff { var runeGram [3]rune var off [3]uint32 var runeCount int @@ -160,21 +153,6 @@ func splitNGramsLimit(str []byte, maxNgrams int) []runeNgramOff { }) } - // We return a random subset of size maxNgrams. This is to prevent the start - // of the string biasing ngram selection. - if maxNgrams < len(result) { - // Deterministic seed for tests. Additionally makes comparing repeated - // queries performance easier. - r := rand.New(rand.NewPCG(uint64(maxNgrams), 0)) - - // Pick random subset via a shuffle - r.Shuffle(maxNgrams, func(i, j int) { result[i], result[j] = result[j], result[i] }) - result = result[:maxNgrams] - - // Caller expects ngrams in order of appearance. - slices.SortFunc(result, runeNgramOff.Compare) - } - return result } diff --git a/indexdata.go b/indexdata.go index 91314f12f..7dd728941 100644 --- a/indexdata.go +++ b/indexdata.go @@ -21,9 +21,7 @@ import ( "hash/crc64" "log" "math/bits" - "os" "slices" - "strconv" "unicode/utf8" "github.com/sourcegraph/zoekt/query" @@ -403,37 +401,11 @@ func (r *ngramIterationResults) candidates() []*candidateMatch { return cs } -// experimentIterateNgramLookupLimit when non-zero will only lookup this many -// ngrams from a query string. Note: that if case-insensitive, this only -// limits the input. So we will still lookup the case folding. -// -// This experiment is targetting looking up large snippets. If it is -// successful, we will likely hardcode the value we use in production. -// -// Future note: if we find cases where this works badly, we can consider only -// searching a random subset of the query string to avoid bad strings. -var experimentIterateNgramLookupLimit = getEnvInt("SRC_EXPERIMENT_ITERATE_NGRAM_LOOKUP_LIMIT") - -func getEnvInt(k string) int { - v, _ := strconv.Atoi(os.Getenv(k)) - if v != 0 { - log.Printf("%s = %d\n", k, v) - } - return v -} - func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResults, error) { str := query.Pattern // Find the 2 least common ngrams from the string. - var ngramOffs []runeNgramOff - if ngramLimit := experimentIterateNgramLookupLimit; ngramLimit > 0 { - // Note: we can't just do str = str[:ngramLimit] due to utf-8 and str - // length is asked later on for other optimizations. - ngramOffs = splitNGramsLimit([]byte(str), ngramLimit) - } else { - ngramOffs = splitNGrams([]byte(str)) - } + ngramOffs := splitNGrams([]byte(str)) // protect against accidental searching of empty strings if len(ngramOffs) == 0 { From bbd1fedfcd9d439681fc70defdbb4d75ac251245 Mon Sep 17 00:00:00 2001 From: Matthew Manela Date: Thu, 1 Aug 2024 10:49:14 -0400 Subject: [PATCH 29/31] feat(Search): Add support for all Apex language extensions (#799) * feat(Search): Add support for all Apex language extensions * clean up comment * Fix typo --- internal/languages/language.go | 8 ++++++++ internal/languages/language_test.go | 14 +++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/languages/language.go b/internal/languages/language.go index 7f1110757..ec76d9458 100644 --- a/internal/languages/language.go +++ b/internal/languages/language.go @@ -16,6 +16,9 @@ import ( ) var unsupportedByLinguistAliasMap = map[string]string{ + // Extensions for the Apex programming language + // See https://developer.salesforce.com/docs/atlas.en-us.apexcode.meta/apexcode/apex_dev_guide.htm + "apex": "Apex", // Pkl Configuration Language (https://pkl-lang.org/) // Add to linguist on 6/7/24 // can remove once go-enry package updates @@ -26,6 +29,11 @@ var unsupportedByLinguistAliasMap = map[string]string{ } var unsupportedByLinguistExtensionToNameMap = map[string]string{ + ".apex": "Apex", + ".apxt": "Apex", + ".apxc": "Apex", + ".cls": "Apex", + ".trigger": "Apex", // Pkl Configuration Language (https://pkl-lang.org/) ".pkl": "Pkl", // Magik Language diff --git a/internal/languages/language_test.go b/internal/languages/language_test.go index 294c67776..25e2382a0 100644 --- a/internal/languages/language_test.go +++ b/internal/languages/language_test.go @@ -39,6 +39,12 @@ func TestGetLanguageByAlias(t *testing.T) { want: "Magik", wantOk: true, }, + { + name: "apex example unsupported by linguist alias", + alias: "apex", + want: "Apex", + wantOk: true, + }, } for _, tt := range tests { @@ -77,11 +83,17 @@ func TestGetLanguage(t *testing.T) { want: "Go", }, { - name: "unsupported by linguist extension", + name: "magik: unsupported by linguist extension", filename: "file.magik", content: []byte(""), want: "Magik", }, + { + name: "apex: unsupported by linguist extension", + filename: "file.apxc", + content: []byte(""), + want: "Apex", + }, } for _, tt := range tests { From 764fe4f9de0e56efa9112385a520de1f96526684 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Thu, 1 Aug 2024 17:41:29 +0200 Subject: [PATCH 30/31] index: enable shard merging by default (#798) This enables shard merging by default for zoekt-sourcegraph-indexserver. Sourcegraph has been using shard merging in production for several years. We have recently confirmed significant performance improvements for queries which are bound by matchTree construction. I also remove -merge_max_priority because we have stopped using it. Use SRC_DISABLE_SHARD_MERGING to disable shard merging. Test plan: mostly CI, I did some manual testing to confirm that shard merging is enabled by default for zoekt-sourcegraph-indexserver. --- build/builder.go | 11 +++++++- cmd/zoekt-sourcegraph-indexserver/index.go | 5 ++++ cmd/zoekt-sourcegraph-indexserver/main.go | 31 +++++++++++++++------- cmd/zoekt-sourcegraph-indexserver/merge.go | 7 ----- tombstones.go | 8 ------ 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/build/builder.go b/build/builder.go index 7932dcf32..dfbfcf7fa 100644 --- a/build/builder.go +++ b/build/builder.go @@ -116,6 +116,10 @@ type Options struct { changedOrRemovedFiles []string LanguageMap ctags.LanguageMap + + // ShardMerging is true if builder should respect compound shards. This is a + // Sourcegraph specific option. + ShardMerging bool } // HashOptions contains only the options in Options that upon modification leads to IndexState of IndexStateMismatch during the next index building. @@ -194,6 +198,7 @@ func (o *Options) Flags(fs *flag.FlagSet) { // Sourcegraph specific fs.BoolVar(&o.DisableCTags, "disable_ctags", x.DisableCTags, "If set, ctags will not be called.") + fs.BoolVar(&o.ShardMerging, "shard_merging", x.ShardMerging, "If set, builder will respect compound shards.") } // Args generates command line arguments for o. It is the "inverse" of Flags. @@ -233,6 +238,10 @@ func (o *Options) Args() []string { args = append(args, "-disable_ctags") } + if o.ShardMerging { + args = append(args, "-shard_merging") + } + return args } @@ -774,7 +783,7 @@ func (b *Builder) Finish() error { for p := range toDelete { // Don't delete compound shards, set tombstones instead. - if zoekt.ShardMergingEnabled() && strings.HasPrefix(filepath.Base(p), "compound-") { + if b.opts.ShardMerging && strings.HasPrefix(filepath.Base(p), "compound-") { if !strings.HasSuffix(p, ".zoekt") { continue } diff --git a/cmd/zoekt-sourcegraph-indexserver/index.go b/cmd/zoekt-sourcegraph-indexserver/index.go index 39453987d..a07cf29df 100644 --- a/cmd/zoekt-sourcegraph-indexserver/index.go +++ b/cmd/zoekt-sourcegraph-indexserver/index.go @@ -98,6 +98,9 @@ type indexArgs struct { // DeltaShardNumberFallbackThreshold is an upper limit on the number of preexisting shards that can exist // before attempting a delta build. DeltaShardNumberFallbackThreshold uint64 + + // ShardMerging is true if we want zoekt-git-index to respect compound shards. + ShardMerging bool } // BuildOptions returns a build.Options represented by indexArgs. Note: it @@ -131,6 +134,8 @@ func (o *indexArgs) BuildOptions() *build.Options { DocumentRanksVersion: o.DocumentRanksVersion, LanguageMap: o.LanguageMap, + + ShardMerging: o.ShardMerging, } } diff --git a/cmd/zoekt-sourcegraph-indexserver/main.go b/cmd/zoekt-sourcegraph-indexserver/main.go index d26ae5ef4..a639894a5 100644 --- a/cmd/zoekt-sourcegraph-indexserver/main.go +++ b/cmd/zoekt-sourcegraph-indexserver/main.go @@ -645,6 +645,8 @@ func (s *Server) indexArgs(opts IndexOptions) *indexArgs { // 1 MB; match https://sourcegraph.sgdev.org/github.com/sourcegraph/sourcegraph/-/blob/cmd/symbols/internal/symbols/search.go#L22 FileLimit: 1 << 20, + + ShardMerging: s.shardMerging, } } @@ -1065,6 +1067,18 @@ func srcLogLevelIsDebug() bool { return strings.EqualFold(lvl, "dbug") || strings.EqualFold(lvl, "debug") } +func getEnvWithDefaultBool(k string, defaultVal bool) bool { + v := os.Getenv(k) + if v == "" { + return defaultVal + } + b, err := strconv.ParseBool(v) + if err != nil { + log.Fatalf("error parsing ENV %s to int64: %s", k, err) + } + return b +} + func getEnvWithDefaultInt64(k string, defaultVal int64) int64 { v := os.Getenv(k) if v == "" { @@ -1196,12 +1210,12 @@ type rootConfig struct { blockProfileRate int // config values related to shard merging - vacuumInterval time.Duration - mergeInterval time.Duration - targetSize int64 - minSize int64 - minAgeDays int - maxPriority float64 + disableShardMerging bool + vacuumInterval time.Duration + mergeInterval time.Duration + targetSize int64 + minSize int64 + minAgeDays int // config values related to backoff indexing repos with one or more consecutive failures backoffDuration time.Duration @@ -1221,12 +1235,12 @@ func (rc *rootConfig) registerRootFlags(fs *flag.FlagSet) { fs.DurationVar(&rc.maxBackoffDuration, "max_backoff_duration", getEnvWithDefaultDuration("MAX_BACKOFF_DURATION", 120*time.Minute), "the maximum duration to backoff from enqueueing a repo for indexing. A negative value disables indexing backoff.") // flags related to shard merging + fs.BoolVar(&rc.disableShardMerging, "shard_merging", getEnvWithDefaultBool("SRC_DISABLE_SHARD_MERGING", false), "disable shard merging") fs.DurationVar(&rc.vacuumInterval, "vacuum_interval", getEnvWithDefaultDuration("SRC_VACUUM_INTERVAL", 24*time.Hour), "run vacuum this often") fs.DurationVar(&rc.mergeInterval, "merge_interval", getEnvWithDefaultDuration("SRC_MERGE_INTERVAL", 8*time.Hour), "run merge this often") fs.Int64Var(&rc.targetSize, "merge_target_size", getEnvWithDefaultInt64("SRC_MERGE_TARGET_SIZE", 2000), "the target size of compound shards in MiB") fs.Int64Var(&rc.minSize, "merge_min_size", getEnvWithDefaultInt64("SRC_MERGE_MIN_SIZE", 1800), "the minimum size of a compound shard in MiB") fs.IntVar(&rc.minAgeDays, "merge_min_age", getEnvWithDefaultInt("SRC_MERGE_MIN_AGE", 7), "the time since the last commit in days. Shards with newer commits are excluded from merging.") - fs.Float64Var(&rc.maxPriority, "merge_max_priority", getEnvWithDefaultFloat64("SRC_MERGE_MAX_PRIORITY", 100), "the maximum priority a shard can have to be considered for merging.") } func startServer(conf rootConfig) error { @@ -1428,7 +1442,7 @@ func newServer(conf rootConfig) (*Server, error) { Interval: conf.interval, CPUCount: cpuCount, queue: *q, - shardMerging: zoekt.ShardMergingEnabled(), + shardMerging: !conf.disableShardMerging, deltaBuildRepositoriesAllowList: deltaBuildRepositoriesAllowList, deltaShardNumberFallbackThreshold: deltaShardNumberFallbackThreshold, repositoriesSkipSymbolsCalculationAllowList: reposShouldSkipSymbolsCalculation, @@ -1439,7 +1453,6 @@ func newServer(conf rootConfig) (*Server, error) { targetSizeBytes: conf.targetSize * 1024 * 1024, minSizeBytes: conf.minSize * 1024 * 1024, minAgeDays: conf.minAgeDays, - maxPriority: conf.maxPriority, }, timeout: indexingTimeout, }, err diff --git a/cmd/zoekt-sourcegraph-indexserver/merge.go b/cmd/zoekt-sourcegraph-indexserver/merge.go index 40c87fcff..9d3cf8533 100644 --- a/cmd/zoekt-sourcegraph-indexserver/merge.go +++ b/cmd/zoekt-sourcegraph-indexserver/merge.go @@ -180,9 +180,6 @@ type mergeOpts struct { // merging. For example, a value of 7 means that only repos that have been // inactive for 7 days will be considered for merging. minAgeDays int - - // the MAX priority a shard can have to be considered for merging. - maxPriority float64 } // isExcluded returns true if a shard should not be merged, false otherwise. @@ -213,10 +210,6 @@ func isExcluded(path string, fi os.FileInfo, opts mergeOpts) bool { return true } - if priority, err := strconv.ParseFloat(repos[0].RawConfig["priority"], 64); err == nil && priority > opts.maxPriority { - return true - } - return false } diff --git a/tombstones.go b/tombstones.go index 77c3a5bd1..a44eb37b1 100644 --- a/tombstones.go +++ b/tombstones.go @@ -5,16 +5,8 @@ import ( "fmt" "os" "path/filepath" - "strconv" ) -// ShardMergingEnabled returns true if SRC_ENABLE_SHARD_MERGING is set to true. -func ShardMergingEnabled() bool { - t := os.Getenv("SRC_ENABLE_SHARD_MERGING") - enabled, _ := strconv.ParseBool(t) - return enabled -} - var mockRepos []*Repository // SetTombstone idempotently sets a tombstone for repoName in .meta. From acacc5eda188f7a4168459b08f7c5f8ea4a078e8 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Fri, 2 Aug 2024 12:00:04 +0200 Subject: [PATCH 31/31] shards: only trigger rescan on .zoekt files changing (#801) Any write to the index dir triggered a scan. This means on busy instances we are constantly rescanning, leading to an over-representation in CPU profiles around watch. The events are normally writes to our temporary files. By only considering events for .zoekt files (which is what scan reads) we can avoid the constant scan calls. Just in case we also introduce a re-scan every minute in case we miss an event. There is error handling around this, but I thought it is just more reliable to call scan every once in a while. Note: this doesn't represent significant CPU use, but it does muddy the CPU profiler output. So this makes it easier to understand trends in our continuous cpu profiling. Test Plan: CI --- shards/watcher.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/shards/watcher.go b/shards/watcher.go index e8cde2e53..16613f701 100644 --- a/shards/watcher.go +++ b/shards/watcher.go @@ -117,6 +117,8 @@ func versionFromPath(path string) (string, int) { } func (s *DirectoryWatcher) scan() error { + // NOTE: if you change which file extensions are read, please update the + // watch implementation. fs, err := filepath.Glob(filepath.Join(s.dir, "*.zoekt")) if err != nil { return err @@ -216,21 +218,38 @@ func (s *DirectoryWatcher) watch() error { signal := make(chan struct{}, 1) go func() { + notify := func() { + select { + case signal <- struct{}{}: + default: + } + } + + ticker := time.NewTicker(time.Minute) + for { select { - case <-watcher.Events: - select { - case signal <- struct{}{}: - default: + case event := <-watcher.Events: + // Only notify if a file we read in has changed. This is important to + // avoid all the events writing to temporary files. + if strings.HasSuffix(event.Name, ".zoekt") || strings.HasSuffix(event.Name, ".meta") { + notify() } + + case <-ticker.C: + // Periodically just double check the disk + notify() + case err := <-watcher.Errors: // Ignore ErrEventOverflow since we rely on the presence of events so // safe to ignore. if err != nil && err != fsnotify.ErrEventOverflow { log.Println("watcher error:", err) } + case <-s.quit: watcher.Close() + ticker.Stop() close(signal) return }