Merge remote-tracking branch 'upstream/main' into jtibs/parser

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.2-alpine3.18 AS builder
+FROM golang:1.21.4-alpine3.18 AS builder
 
 RUN apk add --no-cache ca-certificates
 

build/builder.go

@@ -39,7 +39,6 @@ import (
 	"github.com/bmatcuk/doublestar"
 	"github.com/grafana/regexp"
 	"github.com/rs/xid"
-	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/sourcegraph/zoekt"
 	"github.com/sourcegraph/zoekt/ctags"
@@ -248,7 +247,7 @@ type Builder struct {
 	todo         []*zoekt.Document
 	docChecker   zoekt.DocChecker
 	size         int
-
+	
 	parserBins ctags.ParserBinMap
 	building   sync.WaitGroup
 
@@ -259,8 +258,6 @@ type Builder struct {
 	// them once all shards succeed to avoid Frankstein corpuses.
 	finishedShards map[string]string
 
-	shardLogger io.WriteCloser
-
 	// indexTime is set by tests for doing reproducible builds.
 	indexTime time.Time
 
@@ -575,12 +572,6 @@ func NewBuilder(opts Options) (*Builder, error) {
 
 	b.parserBins = parserBins
 
-	b.shardLogger = &lumberjack.Logger{
-		Filename:   filepath.Join(opts.IndexDir, "zoekt-builder-shard-log.tsv"),
-		MaxSize:    100, // Megabyte
-		MaxBackups: 5,
-	}
-
 	if opts.IsDelta {
 		// Delta shards build on top of previously existing shards.
 		// As a consequence, the shardNum for delta shards starts from
@@ -747,8 +738,6 @@ func (b *Builder) Finish() error {
 		return b.buildError
 	}
 
-	defer b.shardLogger.Close()
-
 	// Collect a map of the old shards on disk. For each new shard we replace we
 	// delete it from toDelete. Anything remaining in toDelete will be removed
 	// after we have renamed everything into place.
@@ -779,8 +768,6 @@ func (b *Builder) Finish() error {
 		}
 
 		delete(toDelete, final)
-
-		b.shardLog("upsert", final, b.opts.RepositoryDescription.Name)
 	}
 
 	b.finishedShards = map[string]string{}
@@ -791,13 +778,11 @@ func (b *Builder) Finish() error {
 			if !strings.HasSuffix(p, ".zoekt") {
 				continue
 			}
-			b.shardLog("tomb", p, b.opts.RepositoryDescription.Name)
 			err := zoekt.SetTombstone(p, b.opts.RepositoryDescription.ID)
 			b.buildError = err
 			continue
 		}
 		log.Printf("removing old shard file: %s", p)
-		b.shardLog("remove", p, b.opts.RepositoryDescription.Name)
 		if err := os.Remove(p); err != nil {
 			b.buildError = err
 		}
@@ -878,15 +863,6 @@ func (b *Builder) flush() error {
 	return nil
 }
 
-func (b *Builder) shardLog(action, shard string, repoName string) {
-	shard = filepath.Base(shard)
-	var shardSize int64
-	if fi, err := os.Stat(filepath.Join(b.opts.IndexDir, shard)); err == nil {
-		shardSize = fi.Size()
-	}
-	_, _ = fmt.Fprintf(b.shardLogger, "%s\t%s\t%s\t%d\t%s\n", time.Now().UTC().Format(time.RFC3339), action, shard, shardSize, repoName)
-}
-
 var profileNumber int
 
 func (b *Builder) writeMemProfile(name string) {

cmd/zoekt-sourcegraph-indexserver/cleanup.go

@@ -13,7 +13,6 @@ import (
 	"github.com/grafana/regexp"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/sourcegraph/zoekt"
 )
@@ -97,18 +96,17 @@ func cleanup(indexDir string, repos []uint32, now time.Time, shardMerging bool)
 		simple := shards[:0]
 		for _, s := range shards {
 			if shardMerging && maybeSetTombstone([]shard{s}, repo) {
-				shardsLog(indexDir, "tombname", []shard{s})
-			} else {
-				simple = append(simple, s)
+				continue
 			}
+
+			simple = append(simple, s)
 		}
 
 		if len(simple) == 0 {
 			continue
 		}
 
 		removeAll(simple...)
-		shardsLog(indexDir, "removename", simple)
 	}
 
 	// index: Move missing repos from trash into index
@@ -121,7 +119,6 @@ func cleanup(indexDir string, repos []uint32, now time.Time, shardMerging bool)
 		if shards, ok := trash[repo]; ok {
 			log.Printf("restoring shards from trash for %v", repo)
 			moveAll(indexDir, shards)
-			shardsLog(indexDir, "restore", shards)
 			continue
 		}
 
@@ -130,8 +127,6 @@ func cleanup(indexDir string, repos []uint32, now time.Time, shardMerging bool)
 			err := zoekt.UnsetTombstone(s.Path, repo)
 			if err != nil {
 				log.Printf("error removing tombstone for %v: %s", repo, err)
-			} else {
-				shardsLog(indexDir, "untomb", []shard{s})
 			}
 		}
 	}
@@ -145,11 +140,9 @@ func cleanup(indexDir string, repos []uint32, now time.Time, shardMerging bool)
 		}
 
 		if shardMerging && maybeSetTombstone(shards, repo) {
-			shardsLog(indexDir, "tomb", shards)
 			continue
 		}
 		moveAll(trashDir, shards)
-		shardsLog(indexDir, "remove", shards)
 	}
 
 	// Remove .tmp files from crashed indexer runs-- for example, if an indexer
@@ -382,24 +375,6 @@ func maybeSetTombstone(shards []shard, repoID uint32) bool {
 	return true
 }
 
-func shardsLog(indexDir, action string, shards []shard) {
-	shardLogger := &lumberjack.Logger{
-		Filename:   filepath.Join(indexDir, "zoekt-indexserver-shard-log.tsv"),
-		MaxSize:    100, // Megabyte
-		MaxBackups: 5,
-	}
-	defer shardLogger.Close()
-
-	for _, s := range shards {
-		shardName := filepath.Base(s.Path)
-		var shardSize int64
-		if fi, err := os.Stat(filepath.Join(indexDir, shardName)); err == nil {
-			shardSize = fi.Size()
-		}
-		_, _ = fmt.Fprintf(shardLogger, "%s\t%s\t%s\t%d\t%s\t%d\n", time.Now().UTC().Format(time.RFC3339), action, shardName, shardSize, s.RepoName, s.RepoID)
-	}
-}
-
 var metricVacuumRunning = promauto.NewGauge(prometheus.GaugeOpts{
 	Name: "index_vacuum_running",
 	Help: "Set to 1 if indexserver's vacuum job is running.",
@@ -448,28 +423,17 @@ func (s *Server) vacuum() {
 
 			if err != nil {
 				debug.Printf("failed to explode compound shard %s: %s", path, string(b))
-			} else {
-				shardsLog(s.IndexDir, "explode", []shard{{Path: path}})
 			}
 			continue
 		}
 
-		var removed []*zoekt.Repository
 		s.muIndexDir.Global(func() {
-			removed, err = removeTombstones(path)
+			_, err = removeTombstones(path)
 		})
 
 		if err != nil {
 			debug.Printf("error while removing tombstones in %s: %s", fn, err)
 		}
-		for _, repo := range removed {
-			shardsLog(s.IndexDir, "vac", []shard{{
-				RepoID:   repo.ID,
-				RepoName: repo.Name,
-				Path:     filepath.Join(s.IndexDir, fn),
-				ModTime:  info.ModTime(),
-			}})
-		}
 	}
 }
 

cmd/zoekt-sourcegraph-indexserver/merge.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"log"
 	"os"
 	"os/exec"
@@ -13,7 +12,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/atomic"
-	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/sourcegraph/zoekt"
 )
@@ -71,12 +69,6 @@ func (s *Server) merge(mergeCmd func(args ...string) *exec.Cmd) {
 	metricShardMergingRunning.Set(1)
 	defer metricShardMergingRunning.Set(0)
 
-	wc := &lumberjack.Logger{
-		Filename:   filepath.Join(s.IndexDir, "zoekt-merge-log.tsv"),
-		MaxSize:    100, // Megabyte
-		MaxBackups: 5,
-	}
-
 	// We keep creating compound shards until we run out of shards to merge or until
 	// we encounter an error during merging.
 	next := true
@@ -107,12 +99,6 @@ func (s *Server) merge(mergeCmd func(args ...string) *exec.Cmd) {
 				return
 			}
 
-			newCompoundName := reCompound.Find(out)
-			now := time.Now()
-			for _, s := range c.shards {
-				_, _ = fmt.Fprintf(wc, "%s\t%s\t%s\t%s\n", now.UTC().Format(time.RFC3339), "merge", filepath.Base(s.path), string(newCompoundName))
-			}
-
 			next = true
 		})
 	}

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/andygrunwald/go-gerrit v0.0.0-20230628115649-c44fe2fbf2ca
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/dustin/go-humanize v1.0.1
-	github.com/edsrzf/mmap-go v1.1.0
 	github.com/felixge/fgprof v0.9.3
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751
@@ -57,7 +56,6 @@ require (
 	golang.org/x/sys v0.11.0
 	google.golang.org/grpc v1.56.1
 	google.golang.org/protobuf v1.31.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (
@@ -142,4 +140,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.18
+go 1.21