feat: detect when go mod tidy and build scaffolding is needed (#3360)

closes #3129

Increases performance of go builds by 0.5s-0.8s in some cases.

Changes:
- Track if any thing has changed that would require us to
- call `go mod tidy` (in either the main module directory, or in the
build directory)
- scaffold the build template (if we scaffold, we also need to run go
mod tidy)
- If a build fails for any reason, reset ongoing build state so next
time we always rebuild everything.

Trigger details:
- If we detect changes in `go.mod`, `go.sum` or `types.ftl.go`, always
rebuild fully
- If the list of imported packages has changed, `go mod tidy` is
required in the module's directory
- If the MainModuleContext has changed in any way, then the build
template needs to be scaffolded, and `go mod tidy` needs to be run in
`.ftl/go/main`

Performace:
- Scaffolding is super fast
- Running `go mod tidy` usually takes around 0.5-0.8s (but can take more
if network requests are required)
- The 2 `go mod tidy` executions are run in parallel, so if either is
required then there is no real performance gain. If neither is required
then we see the performance improvement.
- The rest of the build time is split between extracting schema
(0.5-1.5s) and compiling (0.5-1.5s)

go-runtime/compile/build.go

@@ -2,6 +2,7 @@ package compile
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -31,12 +32,17 @@ import (
 	"github.com/TBD54566975/ftl/internal/projectconfig"
 	"github.com/TBD54566975/ftl/internal/reflect"
 	"github.com/TBD54566975/ftl/internal/schema"
+	islices "github.com/TBD54566975/ftl/internal/slices"
 	"github.com/TBD54566975/ftl/internal/watch"
 )
 
+var ErrInvalidateDependencies = errors.New("dependencies need to be updated")
+var ftlTypesFilename = "types.ftl.go"
+
 type MainWorkContext struct {
 	GoVersion          string
 	SharedModulesPaths []string
+	IncludeMainPackage bool
 }
 
 type mainModuleContext struct {
@@ -78,7 +84,9 @@ func (c *mainModuleContext) generateMainImports() []string {
 	for _, e := range c.MainCtx.ExternalTypes {
 		imports.Add(e.importStatement())
 	}
-	return imports.ToSlice()
+	out := imports.ToSlice()
+	slices.Sort(out)
+	return out
 }
 
 func (c *mainModuleContext) generateTypesImports(mainModuleImport string) []string {
@@ -112,6 +120,7 @@ func (c *mainModuleContext) generateTypesImports(mainModuleImport string) []stri
 		}
 		filteredImports = append(filteredImports, im)
 	}
+	slices.Sort(filteredImports)
 	return filteredImports
 }
 
@@ -262,17 +271,80 @@ func buildDir(moduleDir string) string {
 	return filepath.Join(moduleDir, buildDirName)
 }
 
+// OngoingState maintains state between builds, allowing the Build function to skip steps if nothing has changed.
+type OngoingState struct {
+	imports   []string
+	moduleCtx mainModuleContext
+}
+
+func (s *OngoingState) checkIfImportsChanged(imports []string) (changed bool) {
+	if slices.Equal(s.imports, imports) {
+		return false
+	}
+	s.imports = imports
+	return true
+}
+
+func (s *OngoingState) checkIfMainModuleContextChanged(moduleCtx mainModuleContext) (changed bool) {
+	if stdreflect.DeepEqual(s.moduleCtx, moduleCtx) {
+		return false
+	}
+	s.moduleCtx = moduleCtx
+	return true
+}
+
+// DetectedFileChanges should be called whenever file changes are detected outside of the Build() function.
+// This allows the OngoingState to detect if files need to be reprocessed.
+func (s *OngoingState) DetectedFileChanges(config moduleconfig.AbsModuleConfig, changes []watch.FileChange) {
+	paths := []string{
+		filepath.Join(config.Dir, ftlTypesFilename),
+		filepath.Join(config.Dir, "go.mod"),
+		filepath.Join(config.Dir, "go.sum"),
+	}
+	for _, change := range changes {
+		if !slices.Contains(paths, change.Path) {
+			continue
+		}
+		// If files altered by Build() have been manually changed, reset state to make sure we correct them if needed.
+		s.reset()
+		return
+	}
+}
+
+func (s *OngoingState) reset() {
+	s.imports = nil
+	s.moduleCtx = mainModuleContext{}
+}
+
 // Build the given module.
-func Build(ctx context.Context, projectRootDir, stubsRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction watch.ModifyFilesTransaction, buildEnv []string, devMode bool) (moduleSch optional.Option[*schema.Module], buildErrors []builderrors.Error, err error) {
+func Build(ctx context.Context, projectRootDir, stubsRoot string, config moduleconfig.AbsModuleConfig,
+	sch *schema.Schema, deps, buildEnv []string, filesTransaction watch.ModifyFilesTransaction, ongoingState *OngoingState,
+	devMode bool) (moduleSch optional.Option[*schema.Module], buildErrors []builderrors.Error, err error) {
 	if err := filesTransaction.Begin(); err != nil {
 		return moduleSch, nil, fmt.Errorf("could not start a file transaction: %w", err)
 	}
 	defer func() {
 		if terr := filesTransaction.End(); terr != nil {
 			err = fmt.Errorf("failed to end file transaction: %w", terr)
 		}
+		if err != nil {
+			// If we failed, reset the state to ensure we don't skip steps on the next build.
+			// Example: If `go mod tidy` fails due to a network failure, we need to try again next time, even if nothing else has changed.
+			ongoingState.reset()
+		}
 	}()
 
+	// Check dependencies
+	newDeps, imports, err := extractDependenciesAndImports(config)
+	if err != nil {
+		return moduleSch, nil, fmt.Errorf("could not extract dependencies: %w", err)
+	}
+	importsChanged := ongoingState.checkIfImportsChanged(imports)
+	if !slices.Equal(islices.Sort(newDeps), islices.Sort(deps)) {
+		// dependencies have changed
+		return moduleSch, nil, ErrInvalidateDependencies
+	}
+
 	replacements, goModVersion, err := updateGoModule(filepath.Join(config.Dir, "go.mod"))
 	if err != nil {
 		return moduleSch, nil, err
@@ -283,11 +355,6 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 		return moduleSch, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
 	}
 
-	ftlVersion := ""
-	if ftl.IsRelease(ftl.Version) {
-		ftlVersion = ftl.Version
-	}
-
 	projectName := ""
 	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
 		pc, err := projectconfig.Load(ctx, pcpath)
@@ -317,6 +384,7 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 	if err := internal.ScaffoldZip(mainWorkTemplateFiles(), config.Dir, MainWorkContext{
 		GoVersion:          goModVersion,
 		SharedModulesPaths: sharedModulesPaths,
+		IncludeMainPackage: mainPackageExists(config),
 	}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
 		return moduleSch, nil, fmt.Errorf("failed to scaffold zip: %w", err)
 	}
@@ -334,32 +402,44 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
 	}
 
 	logger.Debugf("Generating main module")
-	mctx, err := buildMainModuleContext(sch, result, goModVersion, ftlVersion, projectName, sharedModulesPaths,
+	mctx, err := buildMainModuleContext(sch, result, goModVersion, projectName, sharedModulesPaths,
 		replacements)
 	if err != nil {
 		return moduleSch, nil, err
 	}
-	if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
-		scaffolder.Functions(funcs)); err != nil {
-		return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
-	}
 
+	mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
+	if mainModuleCtxChanged {
+		if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
+			scaffolder.Functions(funcs)); err != nil {
+			return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
+		}
+		if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
+			return moduleSch, nil, fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
+		}
+	}
 	logger.Debugf("Tidying go.mod files")
 	wg, wgctx := errgroup.WithContext(ctx)
 
-	ftlTypesFilename := "types.ftl.go"
 	wg.Go(func() error {
+		if !importsChanged {
+			log.FromContext(ctx).Debugf("skipped go mod tidy (module dir)\n")
+			return nil
+		}
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to tidy go.mod: %w", config.Dir, err)
 		}
-
 		if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "fmt", ftlTypesFilename).RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to format module dir: %w", config.Dir, err)
 		}
 		return filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename))
 	})
 	mainDir := filepath.Join(buildDir, "go", "main")
 	wg.Go(func() error {
+		if !mainModuleCtxChanged {
+			log.FromContext(ctx).Debugf("skipped go mod tidy (build dir)\n")
+			return nil
+		}
 		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
 			return fmt.Errorf("%s: failed to tidy go.mod: %w", mainDir, err)
 		}
@@ -405,8 +485,12 @@ type mainModuleContextBuilder struct {
 	imports     map[string]string
 }
 
-func buildMainModuleContext(sch *schema.Schema, result extract.Result, goModVersion, ftlVersion, projectName string,
+func buildMainModuleContext(sch *schema.Schema, result extract.Result, goModVersion, projectName string,
 	sharedModulesPaths []string, replacements []*modfile.Replace) (mainModuleContext, error) {
+	ftlVersion := ""
+	if ftl.IsRelease(ftl.Version) {
+		ftlVersion = ftl.Version
+	}
 	combinedSch := &schema.Schema{
 		Modules: append(sch.Modules, result.Module),
 	}

go-runtime/compile/dependencies.go

@@ -16,9 +16,15 @@ import (
 )
 
 func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error) {
+	deps, _, err := extractDependenciesAndImports(config)
+	return deps, err
+}
+
+func extractDependenciesAndImports(config moduleconfig.AbsModuleConfig) (deps []string, imports []string, err error) {
+	importsMap := map[string]bool{}
 	dependencies := map[string]bool{}
 	fset := token.NewFileSet()
-	err := watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
+	err = watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
 		if !d.IsDir() {
 			return nil
 		}
@@ -36,6 +42,7 @@ func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error)
 					if err != nil {
 						continue
 					}
+					importsMap[path] = true
 					if !strings.HasPrefix(path, "ftl/") {
 						continue
 					}
@@ -50,9 +57,11 @@ func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error)
 		return nil
 	})
 	if err != nil {
-		return nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", config.Module, err)
+		return nil, nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", config.Module, err)
 	}
 	modules := maps.Keys(dependencies)
 	sort.Strings(modules)
-	return modules, nil
+	imports = maps.Keys(importsMap)
+	sort.Strings(imports)
+	return modules, imports, nil
 }

go-runtime/compile/main-work-template/go.work.tmpl

@@ -5,4 +5,7 @@ use (
 {{- range .SharedModulesPaths }}
     {{ . }}
 {{- end }}
+{{ if .IncludeMainPackage }}
+    .ftl/go/main
+{{ end }}
 )

go-runtime/compile/stubs.go

@@ -2,7 +2,9 @@ package compile
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 
@@ -102,8 +104,15 @@ func SyncGeneratedStubReferences(ctx context.Context, config moduleconfig.AbsMod
 	if err := internal.ScaffoldZip(mainWorkTemplateFiles(), config.Dir, MainWorkContext{
 		GoVersion:          goModVersion,
 		SharedModulesPaths: sharedModulePaths,
+		IncludeMainPackage: mainPackageExists(config),
 	}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
 		return fmt.Errorf("failed to scaffold zip: %w", err)
 	}
 	return nil
 }
+
+func mainPackageExists(config moduleconfig.AbsModuleConfig) bool {
+	// check if main package exists, otherwise do not include it
+	_, err := os.Stat(filepath.Join(buildDir(config.Dir), "go", "main", "go.mod"))
+	return !errors.Is(err, os.ErrNotExist)
+}