feat: optimistically compile go modules while extracting schema (#3385)

### Description
- The two longest parts to build a go module is schema extraction and
compilation.
- Some code changes (like a verb changing name, or sumtype changes)
require compilation to happen after schema extraction so that we can
generate code based on the latest schema.
- During development though, most builds will be done with compilation
not dependant on schema extraction.
- This PR optimises for this case by optimistically compiling while
schema extraction occurs if previous code generation is present. After
the schema is extracted we then generate code and check if anything has
changed that requires re-compilation.

### Performance
Ran `ftl dev examples/go/time`, waiting for the initial builds to
complete, and then modified the module code 20 times to get an average.
| Case |  Small Change | Verb Change |
-- | -- | --
Last week's code | 2.24s | 3.01s
[With smart `go mod tidy`
detection](fa14481)
| 1.76s (-27%) | 2.99s (~0%)
With optimistic compile (This PR) | 1.30s (-72%) | 3.14s (+5%)

go-runtime/compile/build.go

@@ -15,6 +15,7 @@ import (
 	"unicode"
 
 	"github.com/alecthomas/types/optional"
+	"github.com/alecthomas/types/result"
 	"github.com/block/scaffolder"
 	sets "github.com/deckarep/golang-set/v2"
 	"golang.org/x/exp/maps"
@@ -355,19 +356,12 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		return moduleSch, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
 	}
 
-	projectName := ""
-	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
-		pc, err := projectconfig.Load(ctx, pcpath)
-		if err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to load project config: %w", err)
-		}
-		projectName = pc.Name
-	}
-
 	logger := log.FromContext(ctx)
 	funcs := maps.Clone(scaffoldFuncs)
 
 	buildDir := buildDir(config.Dir)
+	mainDir := filepath.Join(buildDir, "go", "main")
+
 	err = os.MkdirAll(buildDir, 0750)
 	if err != nil {
 		return moduleSch, nil, fmt.Errorf("failed to create build directory: %w", err)
@@ -389,41 +383,139 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		return moduleSch, nil, fmt.Errorf("failed to scaffold zip: %w", err)
 	}
 
-	logger.Debugf("Extracting schema")
-	result, err := extract.Extract(config.Dir)
+	// In parallel, extract schema and optimistically compile.
+	// These two steps take the longest, and only sometimes depend on each other.
+	// After both have completed, we will scaffold out the build template and only use the optimistic compile
+	// if the extracted schema has not caused any changes.
+	extractResultChan := make(chan result.Result[extract.Result], 1)
+	go func() {
+		logger.Debugf("Extracting schema")
+		extractResultChan <- result.From(extract.Extract(config.Dir))
+	}()
+	optimisticHashesChan := make(chan watch.FileHashes, 1)
+	optimisticCompileChan := make(chan error, 1)
+	go func() {
+		hashes, err := fileHashesForOptimisticCompilation(config)
+		if err != nil {
+			optimisticHashesChan <- watch.FileHashes{}
+			return
+		}
+		optimisticHashesChan <- hashes
+
+		logger.Debugf("Optimistically compiling")
+		optimisticCompileChan <- compile(ctx, mainDir, buildEnv, devMode)
+	}()
+
+	// wait for schema extraction to complete
+	extractResult, err := (<-extractResultChan).Result()
 	if err != nil {
 		return moduleSch, nil, fmt.Errorf("could not extract schema: %w", err)
 	}
-
-	if builderrors.ContainsTerminalError(result.Errors) {
+	if builderrors.ContainsTerminalError(extractResult.Errors) {
 		// Only bail if schema errors contain elements at level ERROR.
 		// If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used.
-		return moduleSch, result.Errors, nil
+		return moduleSch, extractResult.Errors, nil
+	}
+
+	logger.Debugf("Generating main package")
+	projectName := ""
+	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
+		pc, err := projectconfig.Load(ctx, pcpath)
+		if err != nil {
+			return moduleSch, nil, fmt.Errorf("failed to load project config: %w", err)
+		}
+		projectName = pc.Name
+	}
+	mctx, err := buildMainModuleContext(sch, extractResult, goModVersion, projectName, sharedModulesPaths, replacements)
+	if err != nil {
+		return moduleSch, nil, err
+	}
+	mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
+	if err := scaffoldBuildTemplateAndTidy(ctx, config, mainDir, importsChanged, mainModuleCtxChanged, mctx, funcs, filesTransaction); err != nil {
+		return moduleSch, nil, err // nolint:wrapcheck
+	}
+
+	logger.Debugf("Writing launch script")
+	if err := writeLaunchScript(buildDir); err != nil {
+		return moduleSch, nil, err
+	}
+
+	// Compare main package hashes to when we optimistically compiled
+	if originalHashes := (<-optimisticHashesChan); len(originalHashes) > 0 {
+		currentHashes, err := fileHashesForOptimisticCompilation(config)
+		if err == nil {
+			changes := watch.CompareFileHashes(originalHashes, currentHashes)
+			// Wait for optimistic compile to complete if there has been no changes
+			if len(changes) == 0 && (<-optimisticCompileChan) == nil {
+				logger.Debugf("Accepting optimistic compilation")
+				return optional.Some(extractResult.Module), extractResult.Errors, nil
+			}
+			logger.Debugf("Discarding optimistic compilation due to file changes: %s", strings.Join(islices.Map(changes, func(change watch.FileChange) string {
+				p, err := filepath.Rel(config.Dir, change.Path)
+				if err != nil {
+					p = change.Path
+				}
+				return fmt.Sprintf("%s%s", change.Change, p)
+			}), ", "))
+		}
 	}
 
-	logger.Debugf("Generating main module")
-	mctx, err := buildMainModuleContext(sch, result, goModVersion, projectName, sharedModulesPaths,
-		replacements)
+	logger.Debugf("Compiling")
+	err = compile(ctx, mainDir, buildEnv, devMode)
 	if err != nil {
 		return moduleSch, nil, err
 	}
+	return optional.Some(extractResult.Module), extractResult.Errors, nil
+}
 
-	mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
+func fileHashesForOptimisticCompilation(config moduleconfig.AbsModuleConfig) (watch.FileHashes, error) {
+	// Include every file that may change while scaffolding the build template or tidying.
+	hashes, err := watch.ComputeFileHashes(config.Dir, false, []string{filepath.Join(buildDirName, "go", "main", "*"), "go.mod", "go.tidy", ftlTypesFilename})
+	if err != nil {
+		return nil, fmt.Errorf("could not calculate hashes for optimistic compilation: %w", err)
+	}
+	if _, ok := hashes[filepath.Join(config.Dir, buildDirName, "go", "main", "main.go")]; !ok {
+		return nil, fmt.Errorf("main package not scaffolded yet")
+	}
+	return hashes, nil
+}
+
+func compile(ctx context.Context, mainDir string, buildEnv []string, devMode bool) error {
+	args := []string{"build", "-o", "../../main", "."}
+	if devMode {
+		args = []string{"build", "-gcflags=all=-N -l", "-o", "../../main", "."}
+	}
+	// We have seen lots of upstream HTTP/2 failures that make CI unstable.
+	// Disable HTTP/2 for now during the build. This can probably be removed later
+	buildEnv = slices.Clone(buildEnv)
+	buildEnv = append(buildEnv, "GODEBUG=http2client=0")
+	err := exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunStderrError(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to compile: %w", err)
+	}
+	return nil
+}
+
+func scaffoldBuildTemplateAndTidy(ctx context.Context, config moduleconfig.AbsModuleConfig, mainDir string, importsChanged,
+	mainModuleCtxChanged bool, mctx mainModuleContext, funcs scaffolder.FuncMap, filesTransaction watch.ModifyFilesTransaction) error {
+	logger := log.FromContext(ctx)
 	if mainModuleCtxChanged {
 		if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
 			scaffolder.Functions(funcs)); err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
+			return fmt.Errorf("failed to scaffold build template: %w", err)
 		}
 		if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
+			return fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
 		}
+	} else {
+		logger.Debugf("Skipped scaffolding build template")
 	}
 	logger.Debugf("Tidying go.mod files")
 	wg, wgctx := errgroup.WithContext(ctx)
 
 	wg.Go(func() error {
 		if !importsChanged {
-			log.FromContext(ctx).Debugf("skipped go mod tidy (module dir)")
+			log.FromContext(ctx).Debugf("Skipped go mod tidy (module dir)")
 			return nil
 		}
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
@@ -432,12 +524,14 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "fmt", ftlTypesFilename).RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to format module dir: %w", config.Dir, err)
 		}
-		return filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename))
+		if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
+			return fmt.Errorf("could not files as modified after tidying module package: %w", err)
+		}
+		return nil
 	})
-	mainDir := filepath.Join(buildDir, "go", "main")
 	wg.Go(func() error {
 		if !mainModuleCtxChanged {
-			log.FromContext(ctx).Debugf("skipped go mod tidy (build dir)")
+			log.FromContext(ctx).Debugf("Skipped go mod tidy (build dir)")
 			return nil
 		}
 		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
@@ -446,36 +540,12 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "fmt", "./...").RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to format main dir: %w", mainDir, err)
 		}
-		return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(config.Dir, "go.sum"))
+		if err := filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(mainDir, "go.sum")); err != nil {
+			return fmt.Errorf("could not files as modified after tidying main package: %w", err)
+		}
+		return nil
 	})
-	if err := wg.Wait(); err != nil {
-		return moduleSch, nil, err // nolint:wrapcheck
-	}
-
-	logger.Debugf("Compiling")
-	args := []string{"build", "-o", "../../main", "."}
-	if devMode {
-		args = []string{"build", "-gcflags=all=-N -l", "-o", "../../main", "."}
-	}
-	// We have seen lots of upstream HTTP/2 failures that make CI unstable.
-	// Disable HTTP/2 for now during the build. This can probably be removed later
-	buildEnv = slices.Clone(buildEnv)
-	buildEnv = append(buildEnv, "GODEBUG=http2client=0")
-	err = exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunStderrError(ctx)
-	if err != nil {
-		return moduleSch, nil, fmt.Errorf("failed to compile: %w", err)
-	}
-	err = os.WriteFile(filepath.Join(mainDir, "../../launch"), []byte(`#!/bin/bash
-	if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then
-	    dlv --listen=localhost:$FTL_DEBUG_PORT --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive exec --continue ./main
-	else
-		exec ./main
-	fi
-	`), 0770) // #nosec
-	if err != nil {
-		return moduleSch, nil, fmt.Errorf("failed to write launch script: %w", err)
-	}
-	return optional.Some(result.Module), result.Errors, nil
+	return wg.Wait() //nolint:wrapcheck
 }
 
 type mainModuleContextBuilder struct {
@@ -547,6 +617,20 @@ func (b *mainModuleContextBuilder) build(goModVersion, ftlVersion, projectName s
 	return *ctx, nil
 }
 
+func writeLaunchScript(buildDir string) error {
+	err := os.WriteFile(filepath.Join(buildDir, "launch"), []byte(`#!/bin/bash
+	if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then
+	    dlv --listen=localhost:$FTL_DEBUG_PORT --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive exec --continue ./main
+	else
+		exec ./main
+	fi
+	`), 0770) // #nosec
+	if err != nil {
+		return fmt.Errorf("failed to write launch script: %w", err)
+	}
+	return nil
+}
+
 func (b *mainModuleContextBuilder) visit(
 	ctx *mainModuleContext,
 	module *schema.Module,

go-runtime/compile/dependencies.go

@@ -24,7 +24,7 @@ func extractDependenciesAndImports(config moduleconfig.AbsModuleConfig) (deps []
 	importsMap := map[string]bool{}
 	dependencies := map[string]bool{}
 	fset := token.NewFileSet()
-	err = watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
+	err = watch.WalkDir(config.Dir, true, func(path string, d fs.DirEntry) error {
 		if !d.IsDir() {
 			return nil
 		}

internal/buildengine/languageplugin/plugin_integration_test.go

@@ -575,7 +575,7 @@ func modifyVerbName(moduleName, old, new string) in.Action {
 
 func walkWatchedFiles(t testing.TB, ic in.TestContext, moduleName string, visit func(path string)) error {
 	path := filepath.Join(ic.WorkingDir(), moduleName)
-	return watch.WalkDir(path, func(srcPath string, entry fs.DirEntry) error {
+	return watch.WalkDir(path, true, func(srcPath string, entry fs.DirEntry) error {
 		if entry.IsDir() {
 			return nil
 		}

internal/watch/discover.go

@@ -40,7 +40,7 @@ func discoverModules(dirs ...string) ([]moduleconfig.UnvalidatedModuleConfig, er
 	}
 	out := []moduleconfig.UnvalidatedModuleConfig{}
 	for _, dir := range dirs {
-		err := WalkDir(dir, func(path string, d fs.DirEntry) error {
+		err := WalkDir(dir, true, func(path string, d fs.DirEntry) error {
 			if filepath.Base(path) != "ftl.toml" {
 				return nil
 			}

internal/watch/filehash.go

@@ -60,15 +60,16 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) []FileChange {
 	return changes
 }
 
-// ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in
-// the given directory.
-func computeFileHashes(dir string, patterns []string) (FileHashes, error) {
+// ComputeFileHashes computes the SHA256 hash of all files in the given directory.
+//
+// If skipGitIgnoredFiles is true, files that are ignored by git will be skipped.
+func ComputeFileHashes(dir string, skipGitIgnoredFiles bool, patterns []string) (FileHashes, error) {
 	// Watch paths are allowed to be outside the deploy directory.
 	fileHashes := make(FileHashes)
 	rootDirs := computeRootDirs(dir, patterns)
 
 	for _, rootDir := range rootDirs {
-		err := WalkDir(rootDir, func(srcPath string, entry fs.DirEntry) error {
+		err := WalkDir(rootDir, skipGitIgnoredFiles, func(srcPath string, entry fs.DirEntry) error {
 			if entry.IsDir() {
 				return nil
 			}
@@ -77,6 +78,9 @@ func computeFileHashes(dir string, patterns []string) (FileHashes, error) {
 				return err
 			}
 			if !matched {
+				if patterns[0] == "*" {
+					return fmt.Errorf("file %s:%s does not match any: %s", rootDir, srcPath, patterns)
+				}
 				return nil
 			}
 			fileHashes[srcPath] = hash

internal/watch/walk.go

@@ -22,8 +22,12 @@ var ErrSkip = errors.New("skip directory")
 //
 // It will adhere to .gitignore files. The callback "fn" can return ErrSkip to
 // skip recursion.
-func WalkDir(dir string, fn func(path string, d fs.DirEntry) error) error {
-	return walkDir(dir, initGitIgnore(dir), fn)
+func WalkDir(dir string, skipGitIgnoredFiles bool, fn func(path string, d fs.DirEntry) error) error {
+	var ignores []string
+	if skipGitIgnoredFiles {
+		ignores = initGitIgnore(dir)
+	}
+	return walkDir(dir, ignores, fn)
 }
 
 // Depth-first walk of dir executing fn after each entry.

internal/watch/watch.go

@@ -157,7 +157,7 @@ func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs []
 					continue
 				}
 				existingModule, haveExistingModule := w.existingModules[config.Dir]
-				hashes, err := computeFileHashes(config.Dir, w.patterns)
+				hashes, err := ComputeFileHashes(config.Dir, true, w.patterns)
 				if err != nil {
 					logger.Tracef("error computing file hashes for %s: %v", config.Dir, err)
 					continue