feat: optimistically compile go modules while extracting schema

go-runtime/compile/build.go

@@ -15,6 +15,7 @@ import (
 	"unicode"
 
 	"github.com/alecthomas/types/optional"
+	"github.com/alecthomas/types/result"
 	"github.com/block/scaffolder"
 	sets "github.com/deckarep/golang-set/v2"
 	"golang.org/x/exp/maps"
@@ -355,19 +356,12 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		return moduleSch, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
 	}
 
-	projectName := ""
-	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
-		pc, err := projectconfig.Load(ctx, pcpath)
-		if err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to load project config: %w", err)
-		}
-		projectName = pc.Name
-	}
-
 	logger := log.FromContext(ctx)
 	funcs := maps.Clone(scaffoldFuncs)
 
 	buildDir := buildDir(config.Dir)
+	mainDir := filepath.Join(buildDir, "go", "main")
+
 	err = os.MkdirAll(buildDir, 0750)
 	if err != nil {
 		return moduleSch, nil, fmt.Errorf("failed to create build directory: %w", err)
@@ -389,41 +383,139 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		return moduleSch, nil, fmt.Errorf("failed to scaffold zip: %w", err)
 	}
 
-	logger.Debugf("Extracting schema")
-	result, err := extract.Extract(config.Dir)
+	// In parallel, extract schema and optimistically compile.
+	// These two steps take the longest, and only sometimes depend on each other.
+	// After both have completed, we will scaffold out the build template and only use the optimistic compile
+	// if the extracted schema has not caused any changes.
+	extractResultChan := make(chan result.Result[extract.Result], 1)
+	go func() {
+		logger.Debugf("Extracting schema")
+		extractResultChan <- result.From(extract.Extract(config.Dir))
+	}()
+	optimisticHashesChan := make(chan watch.FileHashes, 1)
+	optimisticCompileChan := make(chan error, 1)
+	go func() {
+		hashes, err := fileHashesForOptimisticCompilation(config)
+		if err != nil {
+			optimisticHashesChan <- watch.FileHashes{}
+			return
+		}
+		optimisticHashesChan <- hashes
+
+		logger.Debugf("Optimistically compiling")
+		optimisticCompileChan <- compile(ctx, mainDir, buildEnv, devMode)
+	}()
+
+	// wait for schema extraction to complete
+	extractResult, err := (<-extractResultChan).Result()
 	if err != nil {
 		return moduleSch, nil, fmt.Errorf("could not extract schema: %w", err)
 	}
-
-	if builderrors.ContainsTerminalError(result.Errors) {
+	if builderrors.ContainsTerminalError(extractResult.Errors) {
 		// Only bail if schema errors contain elements at level ERROR.
 		// If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used.
-		return moduleSch, result.Errors, nil
+		return moduleSch, extractResult.Errors, nil
+	}
+
+	logger.Debugf("Generating main package")
+	projectName := ""
+	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
+		pc, err := projectconfig.Load(ctx, pcpath)
+		if err != nil {
+			return moduleSch, nil, fmt.Errorf("failed to load project config: %w", err)
+		}
+		projectName = pc.Name
+	}
+	mctx, err := buildMainModuleContext(sch, extractResult, goModVersion, projectName, sharedModulesPaths, replacements)
+	if err != nil {
+		return moduleSch, nil, err
+	}
+	mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
+	if err := scaffoldBuildTemplateAndTidy(ctx, config, mainDir, importsChanged, mainModuleCtxChanged, mctx, funcs, filesTransaction); err != nil {
+		return moduleSch, nil, err // nolint:wrapcheck
+	}
+
+	logger.Debugf("Writing launch script")
+	if err := writeLaunchScript(buildDir); err != nil {
+		return moduleSch, nil, err
+	}
+
+	// Compare main package hashes to when we optimistically compiled
+	if originalHashes := (<-optimisticHashesChan); len(originalHashes) > 0 {
+		currentHashes, err := fileHashesForOptimisticCompilation(config)
+		if err == nil {
+			changes := watch.CompareFileHashes(originalHashes, currentHashes)
+			// Wait for optimistic compile to complete if there has been no changes
+			if len(changes) == 0 && (<-optimisticCompileChan) == nil {
+				logger.Debugf("Accepting optimistic compilation")
+				return optional.Some(extractResult.Module), extractResult.Errors, nil
+			}
+			logger.Debugf("Discarding optimistic compilation due to file changes: %s", strings.Join(islices.Map(changes, func(change watch.FileChange) string {
+				p, err := filepath.Rel(config.Dir, change.Path)
+				if err != nil {
+					p = change.Path
+				}
+				return fmt.Sprintf("%s%s", change.Change, p)
+			}), ", "))
+		}
 	}
 
-	logger.Debugf("Generating main module")
-	mctx, err := buildMainModuleContext(sch, result, goModVersion, projectName, sharedModulesPaths,
-		replacements)
+	logger.Debugf("Compiling")
+	err = compile(ctx, mainDir, buildEnv, devMode)
 	if err != nil {
 		return moduleSch, nil, err
 	}
+	return optional.Some(extractResult.Module), extractResult.Errors, nil
+}
 
-	mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
+func fileHashesForOptimisticCompilation(config moduleconfig.AbsModuleConfig) (watch.FileHashes, error) {
+	// Include every file that may change while scaffolding the build template or tidying.
+	hashes, err := watch.ComputeFileHashes(config.Dir, false, []string{filepath.Join(buildDirName, "go", "main", "*"), "go.mod", "go.tidy", ftlTypesFilename})
+	if err != nil {
+		return nil, fmt.Errorf("could not calculate hashes for optimistic compilation: %w", err)
+	}
+	if _, ok := hashes[filepath.Join(config.Dir, buildDirName, "go", "main", "main.go")]; !ok {
+		return nil, fmt.Errorf("main package not scaffolded yet")
+	}
+	return hashes, nil
+}
+
+func compile(ctx context.Context, mainDir string, buildEnv []string, devMode bool) error {
+	args := []string{"build", "-o", "../../main", "."}
+	if devMode {
+		args = []string{"build", "-gcflags=all=-N -l", "-o", "../../main", "."}
+	}
+	// We have seen lots of upstream HTTP/2 failures that make CI unstable.
+	// Disable HTTP/2 for now during the build. This can probably be removed later
+	buildEnv = slices.Clone(buildEnv)
+	buildEnv = append(buildEnv, "GODEBUG=http2client=0")
+	err := exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunStderrError(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to compile: %w", err)
+	}
+	return nil
+}
+
+func scaffoldBuildTemplateAndTidy(ctx context.Context, config moduleconfig.AbsModuleConfig, mainDir string, importsChanged,
+	mainModuleCtxChanged bool, mctx mainModuleContext, funcs scaffolder.FuncMap, filesTransaction watch.ModifyFilesTransaction) error {
+	logger := log.FromContext(ctx)
 	if mainModuleCtxChanged {
 		if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
 			scaffolder.Functions(funcs)); err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
+			return fmt.Errorf("failed to scaffold build template: %w", err)
 		}
 		if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
-			return moduleSch, nil, fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
+			return fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
 		}
+	} else {
+		logger.Debugf("Skipped scaffolding build template")
 	}
 	logger.Debugf("Tidying go.mod files")
 	wg, wgctx := errgroup.WithContext(ctx)
 
 	wg.Go(func() error {
 		if !importsChanged {
-			log.FromContext(ctx).Debugf("skipped go mod tidy (module dir)")
+			log.FromContext(ctx).Debugf("Skipped go mod tidy (module dir)")
 			return nil
 		}
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
@@ -432,12 +524,14 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "fmt", ftlTypesFilename).RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to format module dir: %w", config.Dir, err)
 		}
-		return filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename))
+		if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
+			return fmt.Errorf("could not files as modified after tidying module package: %w", err)
+		}
+		return nil
 	})
-	mainDir := filepath.Join(buildDir, "go", "main")
 	wg.Go(func() error {
 		if !mainModuleCtxChanged {
-			log.FromContext(ctx).Debugf("skipped go mod tidy (build dir)")
+			log.FromContext(ctx).Debugf("Skipped go mod tidy (build dir)")
 			return nil
 		}
 		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
@@ -446,36 +540,12 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "fmt", "./...").RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to format main dir: %w", mainDir, err)
 		}
-		return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(config.Dir, "go.sum"))
+		if err := filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(mainDir, "go.sum")); err != nil {
+			return fmt.Errorf("could not files as modified after tidying main package: %w", err)
+		}
+		return nil
 	})
-	if err := wg.Wait(); err != nil {
-		return moduleSch, nil, err // nolint:wrapcheck
-	}
-
-	logger.Debugf("Compiling")
-	args := []string{"build", "-o", "../../main", "."}
-	if devMode {
-		args = []string{"build", "-gcflags=all=-N -l", "-o", "../../main", "."}
-	}
-	// We have seen lots of upstream HTTP/2 failures that make CI unstable.
-	// Disable HTTP/2 for now during the build. This can probably be removed later
-	buildEnv = slices.Clone(buildEnv)
-	buildEnv = append(buildEnv, "GODEBUG=http2client=0")
-	err = exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunStderrError(ctx)
-	if err != nil {
-		return moduleSch, nil, fmt.Errorf("failed to compile: %w", err)
-	}
-	err = os.WriteFile(filepath.Join(mainDir, "../../launch"), []byte(`#!/bin/bash
-	if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then
-	    dlv --listen=localhost:$FTL_DEBUG_PORT --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive exec --continue ./main
-	else
-		exec ./main
-	fi
-	`), 0770) // #nosec
-	if err != nil {
-		return moduleSch, nil, fmt.Errorf("failed to write launch script: %w", err)
-	}
-	return optional.Some(result.Module), result.Errors, nil
+	return wg.Wait() //nolint:wrapcheck
 }
 
 type mainModuleContextBuilder struct {
@@ -547,6 +617,20 @@ func (b *mainModuleContextBuilder) build(goModVersion, ftlVersion, projectName s
 	return *ctx, nil
 }
 
+func writeLaunchScript(buildDir string) error {
+	err := os.WriteFile(filepath.Join(buildDir, "launch"), []byte(`#!/bin/bash
+	if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then
+	    dlv --listen=localhost:$FTL_DEBUG_PORT --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive exec --continue ./main
+	else
+		exec ./main
+	fi
+	`), 0770) // #nosec
+	if err != nil {
+		return fmt.Errorf("failed to write launch script: %w", err)
+	}
+	return nil
+}
+
 func (b *mainModuleContextBuilder) visit(
 	ctx *mainModuleContext,
 	module *schema.Module,

go-runtime/compile/dependencies.go

@@ -24,7 +24,7 @@ func extractDependenciesAndImports(config moduleconfig.AbsModuleConfig) (deps []
 	importsMap := map[string]bool{}
 	dependencies := map[string]bool{}
 	fset := token.NewFileSet()
-	err = watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
+	err = watch.WalkDir(config.Dir, true, func(path string, d fs.DirEntry) error {
 		if !d.IsDir() {
 			return nil
 		}

internal/buildengine/languageplugin/plugin_integration_test.go

@@ -575,7 +575,7 @@ func modifyVerbName(moduleName, old, new string) in.Action {
 
 func walkWatchedFiles(t testing.TB, ic in.TestContext, moduleName string, visit func(path string)) error {
 	path := filepath.Join(ic.WorkingDir(), moduleName)
-	return watch.WalkDir(path, func(srcPath string, entry fs.DirEntry) error {
+	return watch.WalkDir(path, true, func(srcPath string, entry fs.DirEntry) error {
 		if entry.IsDir() {
 			return nil
 		}

internal/watch/discover.go

@@ -40,7 +40,7 @@ func discoverModules(dirs ...string) ([]moduleconfig.UnvalidatedModuleConfig, er
 	}
 	out := []moduleconfig.UnvalidatedModuleConfig{}
 	for _, dir := range dirs {
-		err := WalkDir(dir, func(path string, d fs.DirEntry) error {
+		err := WalkDir(dir, true, func(path string, d fs.DirEntry) error {
 			if filepath.Base(path) != "ftl.toml" {
 				return nil
 			}

internal/watch/filehash.go

@@ -60,15 +60,16 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) []FileChange {
 	return changes
 }
 
-// ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in
-// the given directory.
-func computeFileHashes(dir string, patterns []string) (FileHashes, error) {
+// ComputeFileHashes computes the SHA256 hash of all files in the given directory.
+//
+// If skipGitIgnoredFiles is true, files that are ignored by git will be skipped.
+func ComputeFileHashes(dir string, skipGitIgnoredFiles bool, patterns []string) (FileHashes, error) {
 	// Watch paths are allowed to be outside the deploy directory.
 	fileHashes := make(FileHashes)
 	rootDirs := computeRootDirs(dir, patterns)
 
 	for _, rootDir := range rootDirs {
-		err := WalkDir(rootDir, func(srcPath string, entry fs.DirEntry) error {
+		err := WalkDir(rootDir, skipGitIgnoredFiles, func(srcPath string, entry fs.DirEntry) error {
 			if entry.IsDir() {
 				return nil
 			}
@@ -77,6 +78,9 @@ func computeFileHashes(dir string, patterns []string) (FileHashes, error) {
 				return err
 			}
 			if !matched {
+				if patterns[0] == "*" {
+					return fmt.Errorf("file %s:%s does not match any: %s", rootDir, srcPath, patterns)
+				}
 				return nil
 			}
 			fileHashes[srcPath] = hash

internal/watch/walk.go

@@ -22,8 +22,12 @@ var ErrSkip = errors.New("skip directory")
 //
 // It will adhere to .gitignore files. The callback "fn" can return ErrSkip to
 // skip recursion.
-func WalkDir(dir string, fn func(path string, d fs.DirEntry) error) error {
-	return walkDir(dir, initGitIgnore(dir), fn)
+func WalkDir(dir string, skipGitIgnoredFiles bool, fn func(path string, d fs.DirEntry) error) error {
+	var ignores []string
+	if skipGitIgnoredFiles {
+		ignores = initGitIgnore(dir)
+	}
+	return walkDir(dir, ignores, fn)
 }
 
 // Depth-first walk of dir executing fn after each entry.

internal/watch/watch.go

@@ -157,7 +157,7 @@ func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs []
 					continue
 				}
 				existingModule, haveExistingModule := w.existingModules[config.Dir]
-				hashes, err := computeFileHashes(config.Dir, w.patterns)
+				hashes, err := ComputeFileHashes(config.Dir, true, w.patterns)
 				if err != nil {
 					logger.Tracef("error computing file hashes for %s: %v", config.Dir, err)
 					continue