Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: detect when go mod tidy and build scaffolding is needed #3360

Merged
merged 2 commits into from
Nov 8, 2024
Merged

feat: detect when go mod tidy and build scaffolding is needed #3360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c87d3e
improve file change logs
matt2e Nov 7, 2024
ed6a543
only do build template and go mod tidy on relevant changes
matt2e Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 99 additions & 15 deletions go-runtime/compile/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package compile

import (
"context"
"errors"
"fmt"
"os"
"path"
Expand Down Expand Up @@ -31,12 +32,17 @@ import (
"github.com/TBD54566975/ftl/internal/projectconfig"
"github.com/TBD54566975/ftl/internal/reflect"
"github.com/TBD54566975/ftl/internal/schema"
islices "github.com/TBD54566975/ftl/internal/slices"
"github.com/TBD54566975/ftl/internal/watch"
)

var ErrInvalidateDependencies = errors.New("dependencies need to be updated")
var ftlTypesFilename = "types.ftl.go"

type MainWorkContext struct {
GoVersion string
SharedModulesPaths []string
IncludeMainPackage bool
}

type mainModuleContext struct {
Expand Down Expand Up @@ -78,7 +84,9 @@ func (c *mainModuleContext) generateMainImports() []string {
for _, e := range c.MainCtx.ExternalTypes {
imports.Add(e.importStatement())
}
return imports.ToSlice()
out := imports.ToSlice()
slices.Sort(out)
return out
}

func (c *mainModuleContext) generateTypesImports(mainModuleImport string) []string {
Expand Down Expand Up @@ -112,6 +120,7 @@ func (c *mainModuleContext) generateTypesImports(mainModuleImport string) []stri
}
filteredImports = append(filteredImports, im)
}
slices.Sort(filteredImports)
return filteredImports
}

Expand Down Expand Up @@ -262,17 +271,80 @@ func buildDir(moduleDir string) string {
return filepath.Join(moduleDir, buildDirName)
}

// OngoingState maintains state between builds, allowing the Build function to skip steps if nothing has changed.
type OngoingState struct {
imports []string
moduleCtx mainModuleContext
}

func (s *OngoingState) checkIfImportsChanged(imports []string) (changed bool) {
if slices.Equal(s.imports, imports) {
return false
}
s.imports = imports
return true
}

func (s *OngoingState) checkIfMainModuleContextChanged(moduleCtx mainModuleContext) (changed bool) {
if stdreflect.DeepEqual(s.moduleCtx, moduleCtx) {
return false
}
s.moduleCtx = moduleCtx
return true
}

// DetectedFileChanges should be called whenever file changes are detected outside of the Build() function.
// This allows the OngoingState to detect if files need to be reprocessed.
func (s *OngoingState) DetectedFileChanges(config moduleconfig.AbsModuleConfig, changes []watch.FileChange) {
paths := []string{
filepath.Join(config.Dir, ftlTypesFilename),
filepath.Join(config.Dir, "go.mod"),
filepath.Join(config.Dir, "go.sum"),
}
for _, change := range changes {
if !slices.Contains(paths, change.Path) {
continue
}
// If files altered by Build() have been manually changed, reset state to make sure we correct them if needed.
s.reset()
return
}
}

func (s *OngoingState) reset() {
s.imports = nil
s.moduleCtx = mainModuleContext{}
}

// Build the given module.
func Build(ctx context.Context, projectRootDir, stubsRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction watch.ModifyFilesTransaction, buildEnv []string, devMode bool) (moduleSch optional.Option[*schema.Module], buildErrors []builderrors.Error, err error) {
func Build(ctx context.Context, projectRootDir, stubsRoot string, config moduleconfig.AbsModuleConfig,
sch *schema.Schema, deps, buildEnv []string, filesTransaction watch.ModifyFilesTransaction, ongoingState *OngoingState,
devMode bool) (moduleSch optional.Option[*schema.Module], buildErrors []builderrors.Error, err error) {
if err := filesTransaction.Begin(); err != nil {
return moduleSch, nil, fmt.Errorf("could not start a file transaction: %w", err)
}
defer func() {
if terr := filesTransaction.End(); terr != nil {
err = fmt.Errorf("failed to end file transaction: %w", terr)
}
if err != nil {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we log this error?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All errors returned from this function get logged by the build engine, so we don't need to double log here.

// If we failed, reset the state to ensure we don't skip steps on the next build.
// Example: If `go mod tidy` fails due to a network failure, we need to try again next time, even if nothing else has changed.
ongoingState.reset()
}
}()

// Check dependencies
newDeps, imports, err := extractDependenciesAndImports(config)
if err != nil {
return moduleSch, nil, fmt.Errorf("could not extract dependencies: %w", err)
}
importsChanged := ongoingState.checkIfImportsChanged(imports)
if !slices.Equal(islices.Sort(newDeps), islices.Sort(deps)) {
// dependencies have changed
return moduleSch, nil, ErrInvalidateDependencies
}

replacements, goModVersion, err := updateGoModule(filepath.Join(config.Dir, "go.mod"))
if err != nil {
return moduleSch, nil, err
Expand All @@ -283,11 +355,6 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
return moduleSch, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
}

ftlVersion := ""
if ftl.IsRelease(ftl.Version) {
ftlVersion = ftl.Version
}

projectName := ""
if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
pc, err := projectconfig.Load(ctx, pcpath)
Expand Down Expand Up @@ -317,6 +384,7 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
if err := internal.ScaffoldZip(mainWorkTemplateFiles(), config.Dir, MainWorkContext{
GoVersion: goModVersion,
SharedModulesPaths: sharedModulesPaths,
IncludeMainPackage: mainPackageExists(config),
}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
return moduleSch, nil, fmt.Errorf("failed to scaffold zip: %w", err)
}
Expand All @@ -334,32 +402,44 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
}

logger.Debugf("Generating main module")
mctx, err := buildMainModuleContext(sch, result, goModVersion, ftlVersion, projectName, sharedModulesPaths,
mctx, err := buildMainModuleContext(sch, result, goModVersion, projectName, sharedModulesPaths,
replacements)
if err != nil {
return moduleSch, nil, err
}
if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
scaffolder.Functions(funcs)); err != nil {
return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
}

mainModuleCtxChanged := ongoingState.checkIfMainModuleContextChanged(mctx)
if mainModuleCtxChanged {
if err := internal.ScaffoldZip(buildTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
scaffolder.Functions(funcs)); err != nil {
return moduleSch, nil, fmt.Errorf("failed to scaffold build template: %w", err)
}
if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
return moduleSch, nil, fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
}
}
logger.Debugf("Tidying go.mod files")
wg, wgctx := errgroup.WithContext(ctx)

ftlTypesFilename := "types.ftl.go"
wg.Go(func() error {
if !importsChanged {
log.FromContext(ctx).Debugf("skipped go mod tidy (module dir)\n")
return nil
}
if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
return fmt.Errorf("%s: failed to tidy go.mod: %w", config.Dir, err)
}

if err := exec.Command(wgctx, log.Debug, config.Dir, "go", "fmt", ftlTypesFilename).RunStderrError(wgctx); err != nil {
return fmt.Errorf("%s: failed to format module dir: %w", config.Dir, err)
}
return filesTransaction.ModifiedFiles(filepath.Join(config.Dir, "go.mod"), filepath.Join(config.Dir, "go.sum"), filepath.Join(config.Dir, ftlTypesFilename))
})
mainDir := filepath.Join(buildDir, "go", "main")
wg.Go(func() error {
if !mainModuleCtxChanged {
log.FromContext(ctx).Debugf("skipped go mod tidy (build dir)\n")
return nil
}
if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunStderrError(wgctx); err != nil {
return fmt.Errorf("%s: failed to tidy go.mod: %w", mainDir, err)
}
Expand Down Expand Up @@ -405,8 +485,12 @@ type mainModuleContextBuilder struct {
imports map[string]string
}

func buildMainModuleContext(sch *schema.Schema, result extract.Result, goModVersion, ftlVersion, projectName string,
func buildMainModuleContext(sch *schema.Schema, result extract.Result, goModVersion, projectName string,
sharedModulesPaths []string, replacements []*modfile.Replace) (mainModuleContext, error) {
ftlVersion := ""
if ftl.IsRelease(ftl.Version) {
ftlVersion = ftl.Version
}
combinedSch := &schema.Schema{
Modules: append(sch.Modules, result.Module),
}
Expand Down
15 changes: 12 additions & 3 deletions go-runtime/compile/dependencies.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,15 @@ import (
)

func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error) {
deps, _, err := extractDependenciesAndImports(config)
return deps, err
}

func extractDependenciesAndImports(config moduleconfig.AbsModuleConfig) (deps []string, imports []string, err error) {
importsMap := map[string]bool{}
dependencies := map[string]bool{}
fset := token.NewFileSet()
err := watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
err = watch.WalkDir(config.Dir, func(path string, d fs.DirEntry) error {
if !d.IsDir() {
return nil
}
Expand All @@ -36,6 +42,7 @@ func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error)
if err != nil {
continue
}
importsMap[path] = true
if !strings.HasPrefix(path, "ftl/") {
continue
}
Expand All @@ -50,9 +57,11 @@ func ExtractDependencies(config moduleconfig.AbsModuleConfig) ([]string, error)
return nil
})
if err != nil {
return nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", config.Module, err)
return nil, nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", config.Module, err)
}
modules := maps.Keys(dependencies)
sort.Strings(modules)
return modules, nil
imports = maps.Keys(importsMap)
sort.Strings(imports)
return modules, imports, nil
}
3 changes: 3 additions & 0 deletions go-runtime/compile/main-work-template/go.work.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,7 @@ use (
{{- range .SharedModulesPaths }}
{{ . }}
{{- end }}
{{ if .IncludeMainPackage }}
.ftl/go/main
{{ end }}
)
9 changes: 9 additions & 0 deletions go-runtime/compile/stubs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package compile

import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"

Expand Down Expand Up @@ -102,8 +104,15 @@ func SyncGeneratedStubReferences(ctx context.Context, config moduleconfig.AbsMod
if err := internal.ScaffoldZip(mainWorkTemplateFiles(), config.Dir, MainWorkContext{
GoVersion: goModVersion,
SharedModulesPaths: sharedModulePaths,
IncludeMainPackage: mainPackageExists(config),
}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
return fmt.Errorf("failed to scaffold zip: %w", err)
}
return nil
}

func mainPackageExists(config moduleconfig.AbsModuleConfig) bool {
// check if main package exists, otherwise do not include it
_, err := os.Stat(filepath.Join(buildDir(config.Dir), "go", "main", "go.mod"))
return !errors.Is(err, os.ErrNotExist)
}
Loading
Loading