From 4ded60b02f40cfb6af4830dae9e9e803e632dcbd Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Mon, 22 Apr 2024 16:39:27 +1000 Subject: [PATCH] fix: infinite loop caused by build updating files (#1274) fixes: https://github.com/TBD54566975/ftl/issues/1202 Changes: - file watch service now allows transactions that pause file update logic on modules, and allows specific file changes to be explicitly allowed (ie does not count as an update that triggers another build) - building a go module uses a transaction to handle `go mod tidy` commands --- buildengine/build.go | 8 +- buildengine/build_go.go | 4 +- buildengine/build_test.go | 16 +- buildengine/deploy_test.go | 2 +- buildengine/engine.go | 17 ++- buildengine/filehash.go | 77 ++++++---- buildengine/testdata/projects/another/go.sum | 12 +- buildengine/watch.go | 149 +++++++++++++++++-- buildengine/watch_test.go | 147 ++++++++++++++++-- go-runtime/compile/build.go | 26 +++- go-runtime/compile/testdata/one/go.sum | 12 +- go-runtime/compile/testdata/two/go.sum | 12 +- 12 files changed, 387 insertions(+), 95 deletions(-) diff --git a/buildengine/build.go b/buildengine/build.go index bae4ca96f..43ec874ca 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -19,10 +19,10 @@ import ( // Build a project in the given directory given the schema and project config. // For a module, this will build the module. For an external library, this will build stubs for imported modules. -func Build(ctx context.Context, sch *schema.Schema, project Project) error { +func Build(ctx context.Context, sch *schema.Schema, project Project, filesTransaction ModifyFilesTransaction) error { switch project := project.(type) { case Module: - return buildModule(ctx, sch, project) + return buildModule(ctx, sch, project, filesTransaction) case ExternalLibrary: return buildExternalLibrary(ctx, sch, project) default: @@ -30,7 +30,7 @@ func Build(ctx context.Context, sch *schema.Schema, project Project) error { } } -func buildModule(ctx context.Context, sch *schema.Schema, module Module) error { +func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { logger := log.FromContext(ctx).Scope(module.Module) ctx = log.ContextWithLogger(ctx, logger) @@ -43,7 +43,7 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module) error { var err error switch module.Language { case "go": - err = buildGoModule(ctx, sch, module) + err = buildGoModule(ctx, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) default: diff --git a/buildengine/build_go.go b/buildengine/build_go.go index 875fcd3da..ce7d6c677 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, sch *schema.Schema, module Module) error { - if err := compile.Build(ctx, module.Dir, sch); err != nil { +func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { + if err := compile.Build(ctx, module.Dir, sch, transaction); err != nil { return fmt.Errorf("failed to build module %q: %w", module.Config().Key, err) } return nil diff --git a/buildengine/build_test.go b/buildengine/build_test.go index bb544e22a..757f7d44d 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -21,6 +21,20 @@ type buildContext struct { type assertion func(t testing.TB, bctx buildContext) error +type mockModifyFilesTransaction struct{} + +func (t *mockModifyFilesTransaction) Begin() error { + return nil +} + +func (t *mockModifyFilesTransaction) ModifiedFiles(paths ...string) error { + return nil +} + +func (t *mockModifyFilesTransaction) End() error { + return nil +} + func testBuild( t *testing.T, bctx buildContext, @@ -33,7 +47,7 @@ func testBuild( assert.NoError(t, err, "Error getting absolute path for module directory") module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module) + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) if len(expectedBuildErrMsg) > 0 { assert.Error(t, err) assert.Contains(t, err.Error(), expectedBuildErrMsg) diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index 8186e1be7..d2b92e529 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -72,7 +72,7 @@ func TestDeploy(t *testing.T) { assert.NoError(t, err) // Build first to make sure the files are there. - err = Build(ctx, sch, module) + err = Build(ctx, sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) sum, err := sha256.SumFile(modulePath + "/_ftl/main") diff --git a/buildengine/engine.go b/buildengine/engine.go index 2c541d993..c8fb8f515 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -51,6 +51,7 @@ type Engine struct { projectMetas *xsync.MapOf[ProjectKey, projectMeta] moduleDirs []string externalDirs []string + watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] schemaChanges *pubsub.Topic[schemaChange] cancel func() @@ -88,6 +89,7 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, modul moduleDirs: moduleDirs, externalDirs: externalDirs, projectMetas: xsync.NewMapOf[ProjectKey, projectMeta](), + watcher: NewWatcher(), controllerSchema: xsync.NewMapOf[string, *schema.Module](), schemaChanges: pubsub.New[schemaChange](), parallelism: runtime.NumCPU(), @@ -242,13 +244,16 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration defer e.schemaChanges.Unsubscribe(schemaChanges) watchEvents := make(chan WatchEvent, 128) - watch := Watch(ctx, period, e.moduleDirs, e.externalDirs) - watch.Subscribe(watchEvents) - defer watch.Unsubscribe(watchEvents) - defer watch.Close() + topic, err := e.watcher.Watch(ctx, period, e.moduleDirs, e.externalDirs) + if err != nil { + return err + } + topic.Subscribe(watchEvents) + defer topic.Unsubscribe(watchEvents) + defer topic.Close() // Build and deploy all modules first. - err := e.buildAndDeploy(ctx, 1, true) + err = e.buildAndDeploy(ctx, 1, true) if err != nil { logger.Errorf(err, "initial deploy failed") } else { @@ -554,7 +559,7 @@ func (e *Engine) build(ctx context.Context, key ProjectKey, builtModules map[str if e.listener != nil { e.listener.OnBuildStarted(meta.project) } - err := Build(ctx, sch, meta.project) + err := Build(ctx, sch, meta.project, e.watcher.GetTransaction(meta.project.Config().Dir)) if err != nil { return err } diff --git a/buildengine/filehash.go b/buildengine/filehash.go index 53a3e85d0..34a932051 100644 --- a/buildengine/filehash.go +++ b/buildengine/filehash.go @@ -63,47 +63,62 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) (FileChangeType, string, b // ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in // the given directory. -func ComputeFileHashes(module Project) (FileHashes, error) { - config := module.Config() +func ComputeFileHashes(project Project) (FileHashes, error) { + config := project.Config() fileHashes := make(FileHashes) err := WalkDir(config.Dir, func(srcPath string, entry fs.DirEntry) error { - for _, pattern := range config.Watch { - relativePath, err := filepath.Rel(config.Dir, srcPath) - if err != nil { - return err - } + if entry.IsDir() { + return nil + } + hash, matched, err := ComputeFileHash(project, srcPath) + if err != nil { + return err + } + if !matched { + return nil + } + fileHashes[srcPath] = hash + return nil + }) + if err != nil { + return nil, err + } + + return fileHashes, err +} - match, err := doublestar.PathMatch(pattern, relativePath) +func ComputeFileHash(project Project, srcPath string) (hash []byte, matched bool, err error) { + config := project.Config() + + for _, pattern := range config.Watch { + relativePath, err := filepath.Rel(config.Dir, srcPath) + if err != nil { + return nil, false, err + } + match, err := doublestar.PathMatch(pattern, relativePath) + if err != nil { + return nil, false, err + } + if match { + file, err := os.Open(srcPath) if err != nil { - return err + return nil, false, err } - if match && !entry.IsDir() { - file, err := os.Open(srcPath) - if err != nil { - return err - } - - hasher := sha256.New() - if _, err := io.Copy(hasher, file); err != nil { - _ = file.Close() - return err - } + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + _ = file.Close() + return nil, false, err + } - fileHashes[srcPath] = hasher.Sum(nil) + hash := hasher.Sum(nil) - if err := file.Close(); err != nil { - return err - } + if err := file.Close(); err != nil { + return nil, false, err } + return hash, true, nil } - - return nil - }) - if err != nil { - return nil, err } - - return fileHashes, err + return nil, false, nil } diff --git a/buildengine/testdata/projects/another/go.sum b/buildengine/testdata/projects/another/go.sum index 7ac30c81b..9bddd3462 100644 --- a/buildengine/testdata/projects/another/go.sum +++ b/buildengine/testdata/projects/another/go.sum @@ -128,14 +128,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg= +modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8= +modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/buildengine/watch.go b/buildengine/watch.go index b68b81d19..4ad26c1a9 100644 --- a/buildengine/watch.go +++ b/buildengine/watch.go @@ -2,10 +2,13 @@ package buildengine import ( "context" + "fmt" + "sync" "time" "github.com/alecthomas/types/pubsub" + "github.com/TBD54566975/ftl/go-runtime/compile" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/maps" ) @@ -31,17 +34,48 @@ type WatchEventProjectChanged struct { func (WatchEventProjectChanged) watchEvent() {} +type projectHashes struct { + Hashes FileHashes + Project Project +} + +type Watcher struct { + isWatching bool + + // use mutex whenever accessing / modifying existingProjects or moduleTransactions + mutex sync.Mutex + existingProjects map[string]projectHashes + moduleTransactions map[string][]*modifyFilesTransaction +} + +func NewWatcher() *Watcher { + svc := &Watcher{ + existingProjects: map[string]projectHashes{}, + moduleTransactions: map[string][]*modifyFilesTransaction{}, + } + + return svc +} + +func (w *Watcher) GetTransaction(moduleDir string) ModifyFilesTransaction { + return &modifyFilesTransaction{ + watcher: w, + moduleDir: moduleDir, + } +} + // Watch the given directories for new projects, deleted projects, and changes to // existing projects, publishing a change event for each. -func Watch(ctx context.Context, period time.Duration, moduleDirs []string, externalLibDirs []string) *pubsub.Topic[WatchEvent] { +func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs []string, externalLibDirs []string) (*pubsub.Topic[WatchEvent], error) { + if w.isWatching { + return nil, fmt.Errorf("file watcher is already watching") + } + w.isWatching = true + logger := log.FromContext(ctx) topic := pubsub.New[WatchEvent]() + go func() { - type projectHashes struct { - Hashes FileHashes - Project Project - } - existingProjects := map[string]projectHashes{} wait := topic.Wait() for { select { @@ -61,20 +95,29 @@ func Watch(ctx context.Context, period time.Duration, moduleDirs []string, exter return project.Config().Dir, project }) + w.mutex.Lock() // Trigger events for removed projects. - for _, existingProject := range existingProjects { + for _, existingProject := range w.existingProjects { + if transactions, ok := w.moduleTransactions[existingProject.Project.Config().Dir]; ok && len(transactions) > 0 { + // Skip projects that currently have transactions + continue + } existingConfig := existingProject.Project.Config() if _, haveProject := projectsByDir[existingConfig.Dir]; !haveProject { logger.Debugf("removed %s %q", existingProject.Project.TypeString(), existingProject.Project.Config().Key) topic.Publish(WatchEventProjectRemoved{Project: existingProject.Project}) - delete(existingProjects, existingConfig.Dir) + delete(w.existingProjects, existingConfig.Dir) } } // Compare the projects to the existing projects. for _, project := range projectsByDir { + if transactions, ok := w.moduleTransactions[project.Config().Dir]; ok && len(transactions) > 0 { + // Skip projects that currently have transactions + continue + } config := project.Config() - existingProject, haveExistingProject := existingProjects[config.Dir] + existingProject, haveExistingProject := w.existingProjects[config.Dir] hashes, err := ComputeFileHashes(project) if err != nil { logger.Tracef("error computing file hashes for %s: %v", config.Dir, err) @@ -88,14 +131,96 @@ func Watch(ctx context.Context, period time.Duration, moduleDirs []string, exter } logger.Debugf("changed %s %q: %c%s", project.TypeString(), project.Config().Key, changeType, path) topic.Publish(WatchEventProjectChanged{Project: existingProject.Project, Change: changeType, Path: path, Time: time.Now()}) - existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: existingProject.Project} + w.existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: existingProject.Project} continue } logger.Debugf("added %s %q", project.TypeString(), project.Config().Key) topic.Publish(WatchEventProjectAdded{Project: project}) - existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: project} + w.existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: project} } + w.mutex.Unlock() } }() - return topic + return topic, nil +} + +// ModifyFilesTransaction allows builds to modify files in a module without triggering a watch event. +// This helps us avoid infinite loops with builds changing files, and those changes triggering new builds.as a no-op +type ModifyFilesTransaction interface { + Begin() error + ModifiedFiles(paths ...string) error + End() error +} + +// Implementation of ModifyFilesTransaction protocol +type modifyFilesTransaction struct { + watcher *Watcher + moduleDir string + isActive bool +} + +var _ ModifyFilesTransaction = (*modifyFilesTransaction)(nil) +var _ compile.ModifyFilesTransaction = (*modifyFilesTransaction)(nil) + +func (t *modifyFilesTransaction) Begin() error { + if t.isActive { + return fmt.Errorf("transaction is already active") + } + t.isActive = true + + t.watcher.mutex.Lock() + defer t.watcher.mutex.Unlock() + + t.watcher.moduleTransactions[t.moduleDir] = append(t.watcher.moduleTransactions[t.moduleDir], t) + + return nil +} + +func (t *modifyFilesTransaction) End() error { + if !t.isActive { + return fmt.Errorf("transaction is not active") + } + + t.watcher.mutex.Lock() + defer t.watcher.mutex.Unlock() + + for idx, transaction := range t.watcher.moduleTransactions[t.moduleDir] { + if transaction != t { + continue + } + t.isActive = false + t.watcher.moduleTransactions[t.moduleDir] = append(t.watcher.moduleTransactions[t.moduleDir][:idx], t.watcher.moduleTransactions[t.moduleDir][idx+1:]...) + return nil + } + return fmt.Errorf("could not end transaction because it was not found") +} + +func (t *modifyFilesTransaction) ModifiedFiles(paths ...string) error { + if !t.isActive { + return fmt.Errorf("can not modify file because transaction is not active: %v", paths) + } + + t.watcher.mutex.Lock() + defer t.watcher.mutex.Unlock() + + projectHashes, ok := t.watcher.existingProjects[t.moduleDir] + if !ok { + // skip updating hashes because we have not discovered this project yet + return nil + } + + for _, path := range paths { + hash, matched, err := ComputeFileHash(projectHashes.Project, path) + if err != nil { + return err + } + if !matched { + continue + } + + projectHashes.Hashes[path] = hash + } + t.watcher.existingProjects[t.moduleDir] = projectHashes + + return nil } diff --git a/buildengine/watch_test.go b/buildengine/watch_test.go index 39bc2b464..cf0560227 100644 --- a/buildengine/watch_test.go +++ b/buildengine/watch_test.go @@ -9,11 +9,15 @@ import ( "time" "github.com/alecthomas/assert/v2" + "github.com/alecthomas/types/pubsub" . "github.com/TBD54566975/ftl/buildengine" "github.com/TBD54566975/ftl/internal/log" ) +const pollFrequency = time.Millisecond * 500 +const halfPollFrequency = time.Millisecond * 250 + func TestWatch(t *testing.T) { if testing.Short() { t.SkipNow() @@ -22,32 +26,27 @@ func TestWatch(t *testing.T) { dir := t.TempDir() - // Start the watch - events := make(chan WatchEvent, 128) - watch := Watch(ctx, time.Millisecond*200, []string{dir}, nil) - watch.Subscribe(events) + w := NewWatcher() + events, topic := startWatching(ctx, t, w, dir) + + time.Sleep(pollFrequency + halfPollFrequency) // midway between file scans // Initiate a bunch of changes. err := ftl("init", "go", dir, "one") assert.NoError(t, err) err = ftl("init", "go", dir, "two") assert.NoError(t, err) - time.Sleep(time.Millisecond * 500) + time.Sleep(pollFrequency) // Delete a module err = os.RemoveAll(filepath.Join(dir, "two")) assert.NoError(t, err) // Change a module. - cmd := exec.Command("go", "mod", "edit", "-replace=github.com/TBD54566975/ftl=..") - cmd.Dir = filepath.Join(dir, "one") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - assert.NoError(t, err) + updateModFile(t, filepath.Join(dir, "one")) - time.Sleep(time.Millisecond * 500) - watch.Close() + time.Sleep(pollFrequency) + topic.Close() allEvents := []WatchEvent{} for event := range events { @@ -76,7 +75,117 @@ func TestWatch(t *testing.T) { } } } - assert.True(t, found >= 4, "expected at least 4 events, got %d", found) + assert.True(t, found >= 4, "expected at least 4 events, got %d:\n%v", found, allEvents) +} + +func TestWatchWithBuildModifyingFiles(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + ctx := log.ContextWithNewDefaultLogger(context.Background()) + + dir := t.TempDir() + + w := NewWatcher() + + // Initiate a module + err := ftl("init", "go", dir, "one") + assert.NoError(t, err) + + events, topic := startWatching(ctx, t, w, dir) + + time.Sleep(pollFrequency + halfPollFrequency) // midway between file scans + + // Change a file in a module, within a transaction + transaction := w.GetTransaction(filepath.Join(dir, "one")) + err = transaction.Begin() + assert.NoError(t, err) + updateModFile(t, filepath.Join(dir, "one")) + err = transaction.ModifiedFiles(filepath.Join(dir, "one", "go.mod")) + assert.NoError(t, err) + + err = transaction.End() + assert.NoError(t, err) + + time.Sleep(pollFrequency) + topic.Close() + + allEvents := []WatchEvent{} + for event := range events { + allEvents = append(allEvents, event) + } + for _, event := range allEvents { + event, wasAdded := event.(WatchEventProjectAdded) + assert.True(t, wasAdded, "expected only project added events, got %v", event) + } +} + +func TestWatchWithBuildAndUserModifyingFiles(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + ctx := log.ContextWithNewDefaultLogger(context.Background()) + + dir := t.TempDir() + + // Initiate a module + err := ftl("init", "go", dir, "one") + assert.NoError(t, err) + + w := NewWatcher() + events, topic := startWatching(ctx, t, w, dir) + + time.Sleep(pollFrequency + halfPollFrequency) // midway between file scans + + // Change a file in a module, within a transaction + transaction := w.GetTransaction(filepath.Join(dir, "one")) + err = transaction.Begin() + assert.NoError(t, err) + + updateModFile(t, filepath.Join(dir, "one")) + + // Change a file in a module, without a transaction (user change) + cmd := exec.Command("mv", "one.go", "one_.go") + cmd.Dir = filepath.Join(dir, "one") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + assert.NoError(t, err) + + err = transaction.End() + assert.NoError(t, err) + + time.Sleep(pollFrequency) + topic.Close() + + allEvents := []WatchEvent{} + for event := range events { + allEvents = append(allEvents, event) + } + foundChange := false + for _, event := range allEvents { + switch event := event.(type) { + case WatchEventProjectAdded: + // expected + case WatchEventProjectRemoved: + assert.False(t, true, "unexpected project removed event") + case WatchEventProjectChanged: + if event.Project.Config().Key == "one" { + foundChange = true + } + } + } + assert.True(t, foundChange, "expected project changed event") +} + +func startWatching(ctx context.Context, t *testing.T, w *Watcher, dir string) (chan WatchEvent, *pubsub.Topic[WatchEvent]) { + t.Helper() + events := make(chan WatchEvent, 128) + topic, err := w.Watch(ctx, pollFrequency, []string{dir}, nil) + assert.NoError(t, err) + topic.Subscribe(events) + + return events, topic } func ftl(args ...string) error { @@ -85,3 +194,13 @@ func ftl(args ...string) error { cmd.Stderr = os.Stderr return cmd.Run() } + +func updateModFile(t *testing.T, dir string) { + t.Helper() + cmd := exec.Command("go", "mod", "edit", "-replace=github.com/TBD54566975/ftl=..") + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + assert.NoError(t, err) +} diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index f9122773f..abbe4fc4e 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -51,6 +51,12 @@ type mainModuleContext struct { Replacements []*modfile.Replace } +type ModifyFilesTransaction interface { + Begin() error + ModifiedFiles(paths ...string) error + End() error +} + func (b ExternalModuleContext) NonMainModules() []*schema.Module { modules := make([]*schema.Module, 0, len(b.Modules)) for _, module := range b.Modules { @@ -69,7 +75,16 @@ func buildDir(moduleDir string) string { } // Build the given module. -func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { +func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { + if err := filesTransaction.Begin(); err != nil { + return err + } + defer func() { + if terr := filesTransaction.End(); terr != nil { + err = fmt.Errorf("failed to end file transaction: %w", terr) + } + }() + replacements, goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod")) if err != nil { return err @@ -155,28 +170,27 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { return err } - wg, wgctx := errgroup.WithContext(ctx) - logger.Debugf("Tidying go.mod files") + wg, wgctx := errgroup.WithContext(ctx) wg.Go(func() error { if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err) } - return nil + return filesTransaction.ModifiedFiles(filepath.Join(moduleDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) mainDir := filepath.Join(buildDir, "go", "main") wg.Go(func() error { if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", mainDir, err) } - return nil + return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) wg.Go(func() error { modulesDir := filepath.Join(buildDir, "go", "modules") if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err) } - return nil + return filesTransaction.ModifiedFiles(filepath.Join(modulesDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) if err := wg.Wait(); err != nil { return err diff --git a/go-runtime/compile/testdata/one/go.sum b/go-runtime/compile/testdata/one/go.sum index 7ac30c81b..9bddd3462 100644 --- a/go-runtime/compile/testdata/one/go.sum +++ b/go-runtime/compile/testdata/one/go.sum @@ -128,14 +128,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg= +modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8= +modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/go-runtime/compile/testdata/two/go.sum b/go-runtime/compile/testdata/two/go.sum index 7ac30c81b..9bddd3462 100644 --- a/go-runtime/compile/testdata/two/go.sum +++ b/go-runtime/compile/testdata/two/go.sum @@ -128,14 +128,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg= +modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8= +modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=