e2e: introduce optional shard cache (#713)

While iterating on the ranking tests it is frustrating to wait for the
index to be built. This commit adds a "-shard_cache" flag to reuse the
computed shards. It defaults to off since this could lead to unexpected
results if we change how we index.

Test Plan: run go test several times with the flag, only the first run
should be slow. Then confirm without flag it is "slow".

  $ go test -shard_cache
  PASS
  ok      github.com/sourcegraph/zoekt/internal/e2e       23.307s
  $ go test -shard_cache
  PASS
  ok      github.com/sourcegraph/zoekt/internal/e2e       0.235s
  $ go test -shard_cache
  PASS
  ok      github.com/sourcegraph/zoekt/internal/e2e       0.218s
  $ go test
  PASS
  ok      github.com/sourcegraph/zoekt/internal/e2e       23.416s

internal/e2e/e2e_rank_test.go

@@ -24,6 +24,8 @@ import (
 
 var update = flag.Bool("update", false, "update golden file")
 
+var useShardCache = flag.Bool("shard_cache", false, "cache computed shards for faster test runs")
+
 // debugScore can be set to include much more output. Do not commit the
 // updated golden files, this is purely used for debugging in a local
 // environment.
@@ -58,7 +60,13 @@ func TestRanking(t *testing.T) {
 		"r:cody sourcegraph url",
 	}
 
-	indexDir := t.TempDir()
+	var indexDir string
+	if *useShardCache {
+		t.Logf("reusing index dir to speed up testing. If you have unexpected results remove %s", shardCache)
+		indexDir = shardCache
+	} else {
+		indexDir = t.TempDir()
+	}
 
 	for _, u := range archiveURLs {
 		if err := indexURL(indexDir, u); err != nil {
@@ -130,14 +138,16 @@ func TestRanking(t *testing.T) {
 }
 
 var tarballCache = "/tmp/zoekt-test-ranking-tarballs-" + os.Getenv("USER")
+var shardCache = "/tmp/zoekt-test-ranking-shards-" + os.Getenv("USER")
 
 func indexURL(indexDir, u string) error {
 	if err := os.MkdirAll(tarballCache, 0700); err != nil {
 		return err
 	}
 
 	opts := archive.Options{
-		Archive: u,
+		Archive:     u,
+		Incremental: true,
 	}
 	opts.SetDefaults() // sets metadata like Name and the codeload URL
 	u = opts.Archive