From ab853367387690af388bf2f451763230cc43e594 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Wed, 9 Oct 2024 08:59:42 +1100 Subject: [PATCH] feat: remove Error and Deploy from ModuleConfig (#3033) Remove things from ModuleConfig in preparation for language plugins: - `ModuleConfig.Errors` - Removed logic to ever write errors to disk (except Java plugin, which just hardcodes `errors.pb` until it moves to being an external plugin) - `ModuleConfig.Deploy` - This has been refactored to being part of a build result. This also allows language plugins to dynamically include files to deploy. --------- Co-authored-by: github-actions[bot] --- frontend/cli/cmd_box.go | 2 +- go-runtime/compile/build.go | 81 ++++--------------- internal/buildengine/build.go | 14 ++-- internal/buildengine/deploy.go | 14 +++- internal/buildengine/engine.go | 34 ++++++-- .../buildengine/languageplugin/go_plugin.go | 14 ++-- .../languageplugin/go_plugin_test.go | 8 -- .../buildengine/languageplugin/java_plugin.go | 54 +++++++++++-- .../languageplugin/java_plugin_test.go | 12 +-- internal/buildengine/languageplugin/plugin.go | 55 +++---------- .../buildengine/languageplugin/rust_plugin.go | 28 ++++++- internal/buildengine/module.go | 8 ++ internal/moduleconfig/moduleconfig.go | 35 -------- internal/moduleconfig/moduleconfig_test.go | 44 +--------- 14 files changed, 164 insertions(+), 239 deletions(-) diff --git a/frontend/cli/cmd_box.go b/frontend/cli/cmd_box.go index 245f39137..8a0e93f9a 100644 --- a/frontend/cli/cmd_box.go +++ b/frontend/cli/cmd_box.go @@ -148,7 +148,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC destDir := filepath.Join(workDir, "modules", config.Module) // Copy deployment artefacts. - files, err := buildengine.FindFilesToDeploy(config) + files, err := buildengine.FindFilesToDeploy(config, m.Deploy) if err != nil { return err } diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 660f2e384..116b58fbf 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -20,12 +20,8 @@ import ( "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/TBD54566975/ftl" - languagepb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/language" - schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema" extract "github.com/TBD54566975/ftl/go-runtime/schema" "github.com/TBD54566975/ftl/internal" "github.com/TBD54566975/ftl/internal/builderrors" @@ -262,9 +258,9 @@ func buildDir(moduleDir string) string { } // Build the given module. -func Build(ctx context.Context, projectRootDir, moduleDir string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction ModifyFilesTransaction, buildEnv []string, devMode bool) (err error) { +func Build(ctx context.Context, projectRootDir, moduleDir string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction ModifyFilesTransaction, buildEnv []string, devMode bool) (moduleSch *schema.Module, buildErrors []builderrors.Error, err error) { if err := filesTransaction.Begin(); err != nil { - return err + return nil, nil, fmt.Errorf("could not start a file transaction: %w", err) } defer func() { if terr := filesTransaction.End(); terr != nil { @@ -274,12 +270,12 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec replacements, goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod")) if err != nil { - return err + return nil, nil, err } goVersion := runtime.Version()[2:] if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 { - return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) + return nil, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) } ftlVersion := "" @@ -291,7 +287,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok { pc, err := projectconfig.Load(ctx, pcpath) if err != nil { - return fmt.Errorf("failed to load project config: %w", err) + return nil, nil, fmt.Errorf("failed to load project config: %w", err) } projectName = pc.Name } @@ -302,7 +298,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec buildDir := buildDir(moduleDir) err = os.MkdirAll(buildDir, 0750) if err != nil { - return fmt.Errorf("failed to create build directory: %w", err) + return nil, nil, fmt.Errorf("failed to create build directory: %w", err) } var sharedModulesPaths []string @@ -317,36 +313,30 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec GoVersion: goModVersion, SharedModulesPaths: sharedModulesPaths, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { - return fmt.Errorf("failed to scaffold zip: %w", err) + return nil, nil, fmt.Errorf("failed to scaffold zip: %w", err) } logger.Debugf("Extracting schema") result, err := extract.Extract(config.Dir) if err != nil { - return err + return nil, nil, fmt.Errorf("could not extract schema: %w", err) } - if err = writeSchemaErrors(config, result.Errors); err != nil { - return fmt.Errorf("failed to write schema errors: %w", err) - } if builderrors.ContainsTerminalError(result.Errors) { // Only bail if schema errors contain elements at level ERROR. // If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used. - return nil - } - if err = writeSchema(config, result.Module); err != nil { - return fmt.Errorf("failed to write schema: %w", err) + return nil, result.Errors, nil } logger.Debugf("Generating main module") mctx, err := buildMainModuleContext(sch, result, goModVersion, ftlVersion, projectName, sharedModulesPaths, replacements) if err != nil { - return err + return nil, nil, err } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mctx, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { - return err + return nil, nil, fmt.Errorf("failed to scaffold build template: %w", err) } logger.Debugf("Tidying go.mod files") @@ -374,7 +364,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) if err := wg.Wait(); err != nil { - return err + return nil, nil, err // nolint:wrapcheck } logger.Debugf("Compiling") @@ -387,7 +377,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec buildEnv = append(buildEnv, "GODEBUG=http2client=0") err = exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunBuffered(ctx) if err != nil { - return fmt.Errorf("failed to compile: %w", err) + return nil, nil, fmt.Errorf("failed to compile: %w", err) } err = os.WriteFile(filepath.Join(mainDir, "../../launch"), []byte(`#!/bin/bash if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then @@ -397,9 +387,9 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec fi `), 0770) // #nosec if err != nil { - return fmt.Errorf("failed to write launch script: %w", err) + return nil, nil, fmt.Errorf("failed to write launch script: %w", err) } - return nil + return result.Module, result.Errors, nil } // CleanStubs removes all generated stubs. @@ -1154,47 +1144,6 @@ func shouldUpdateVersion(goModfile *modfile.File) bool { return true } -func writeSchema(config moduleconfig.AbsModuleConfig, module *schema.Module) error { - modulepb := module.ToProto().(*schemapb.Module) //nolint:forcetypeassert - // If user has overridden GOOS and GOARCH we want to use those values. - goos, ok := os.LookupEnv("GOOS") - if !ok { - goos = runtime.GOOS - } - goarch, ok := os.LookupEnv("GOARCH") - if !ok { - goarch = runtime.GOARCH - } - - modulepb.Runtime = &schemapb.ModuleRuntime{ - CreateTime: timestamppb.Now(), - Language: "go", - Os: &goos, - Arch: &goarch, - } - schemaBytes, err := proto.Marshal(module.ToProto()) - if err != nil { - return fmt.Errorf("failed to marshal schema: %w", err) - } - err = os.WriteFile(config.Schema(), schemaBytes, 0600) - if err != nil { - return fmt.Errorf("could not write schema: %w", err) - } - return nil -} - -func writeSchemaErrors(config moduleconfig.AbsModuleConfig, errors []builderrors.Error) error { - elBytes, err := proto.Marshal(languagepb.ErrorsToProto(errors)) - if err != nil { - return fmt.Errorf("failed to marshal errors: %w", err) - } - err = os.WriteFile(config.Errors, elBytes, 0600) - if err != nil { - return fmt.Errorf("could not write build errors: %w", err) - } - return nil -} - // returns the import path and directory name for a Go type // package and directory names are the same (dir=bar, pkg=bar): "github.com/foo/bar.A" => "github.com/foo/bar", none // package and directory names differ (dir=bar, pkg=baz): "github.com/foo/bar.baz.A" => "github.com/foo/bar", "baz" diff --git a/internal/buildengine/build.go b/internal/buildengine/build.go index c0d0ae113..b87134b2e 100644 --- a/internal/buildengine/build.go +++ b/internal/buildengine/build.go @@ -20,7 +20,7 @@ import ( // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, sch *schema.Schema, config moduleconfig.ModuleConfig, buildEnv []string, devMode bool) (*schema.Module, error) { +func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, sch *schema.Schema, config moduleconfig.ModuleConfig, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) { logger := log.FromContext(ctx).Module(config.Module).Scope("build") ctx = log.ContextWithLogger(ctx, logger) @@ -34,14 +34,14 @@ func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRoo } // handleBuildResult processes the result of a build -func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult either.Either[languageplugin.BuildResult, error]) (*schema.Module, error) { +func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult either.Either[languageplugin.BuildResult, error]) (moduleSchema *schema.Module, deploy []string, err error) { logger := log.FromContext(ctx) config := c.Abs() var result languageplugin.BuildResult switch eitherResult := eitherResult.(type) { case either.Right[languageplugin.BuildResult, error]: - return nil, fmt.Errorf("failed to build module: %w", eitherResult.Get()) + return nil, nil, fmt.Errorf("failed to build module: %w", eitherResult.Get()) case either.Left[languageplugin.BuildResult, error]: result = eitherResult.Get() } @@ -56,7 +56,7 @@ func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherR } if len(errs) > 0 { - return nil, errors.Join(errs...) + return nil, nil, errors.Join(errs...) } logger.Infof("Module built (%.2fs)", time.Since(result.StartTime).Seconds()) @@ -64,10 +64,10 @@ func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherR // write schema proto to deploy directory schemaBytes, err := proto.Marshal(result.Schema.ToProto()) if err != nil { - return nil, fmt.Errorf("failed to marshal schema: %w", err) + return nil, nil, fmt.Errorf("failed to marshal schema: %w", err) } if err := os.WriteFile(config.Schema(), schemaBytes, 0600); err != nil { - return nil, fmt.Errorf("failed to write schema: %w", err) + return nil, nil, fmt.Errorf("failed to write schema: %w", err) } - return result.Schema, nil + return result.Schema, result.Deploy, nil } diff --git a/internal/buildengine/deploy.go b/internal/buildengine/deploy.go index abe9fe44e..374eb654e 100644 --- a/internal/buildengine/deploy.go +++ b/internal/buildengine/deploy.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "connectrpc.com/connect" @@ -38,13 +39,14 @@ type DeployClient interface { } // Deploy a module to the FTL controller with the given number of replicas. Optionally wait for the deployment to become ready. -func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnline bool, client DeployClient) error { +func Deploy(ctx context.Context, module Module, deploy []string, replicas int32, waitForDeployOnline bool, client DeployClient) error { + fmt.Printf("Deplying with arg: %v", deploy) logger := log.FromContext(ctx).Module(module.Config.Module).Scope("deploy") ctx = log.ContextWithLogger(ctx, logger) logger.Infof("Deploying module") moduleConfig := module.Config.Abs() - files, err := FindFilesToDeploy(moduleConfig) + files, err := FindFilesToDeploy(moduleConfig, deploy) if err != nil { logger.Errorf(err, "failed to find files in %s", moduleConfig) return err @@ -159,9 +161,13 @@ func loadProtoSchema(config moduleconfig.AbsModuleConfig, replicas int32) (*sche } // FindFilesToDeploy returns a list of files to deploy for the given module. -func FindFilesToDeploy(moduleConfig moduleconfig.AbsModuleConfig) ([]string, error) { +func FindFilesToDeploy(config moduleconfig.AbsModuleConfig, deploy []string) ([]string, error) { var out []string - for _, file := range moduleConfig.Deploy { + for _, f := range deploy { + file := filepath.Clean(filepath.Join(config.DeployDir, f)) + if !strings.HasPrefix(file, config.DeployDir) { + return nil, fmt.Errorf("deploy path %q is not beneath deploy directory %q", file, config.DeployDir) + } info, err := os.Stat(file) if err != nil { return nil, err diff --git a/internal/buildengine/engine.go b/internal/buildengine/engine.go index 47807e6f3..9461f74b7 100644 --- a/internal/buildengine/engine.go +++ b/internal/buildengine/engine.go @@ -338,11 +338,14 @@ func (e *Engine) Deploy(ctx context.Context, replicas int32, waitForDeployOnline continue } deployGroup.Go(func() error { - module, ok := e.moduleMetas.Load(moduleName) + meta, ok := e.moduleMetas.Load(moduleName) if !ok { return fmt.Errorf("module %q not found", moduleName) } - return Deploy(ctx, module.module, replicas, waitForDeployOnline, e.client) + if len(meta.module.Deploy) == 0 { + return fmt.Errorf("no files found to deploy for %q", moduleName) + } + return Deploy(ctx, meta.module, meta.module.Deploy, replicas, waitForDeployOnline, e.client) }) } if err := deployGroup.Wait(); err != nil { @@ -599,7 +602,8 @@ func (e *Engine) BuildAndDeploy(ctx context.Context, replicas int32, waitForDepl buildGroup.Go(func() error { e.modulesToBuild.Store(module.Config.Module, false) terminal.UpdateModuleState(ctx, module.Config.Module, terminal.BuildStateDeploying) - return Deploy(buildCtx, module, replicas, waitForDeployOnline, e.client) + + return Deploy(buildCtx, module, module.Deploy, replicas, waitForDeployOnline, e.client) }) return nil }, moduleNames...) @@ -758,7 +762,7 @@ func (e *Engine) tryBuild(ctx context.Context, mustBuild map[string]bool, module meta, ok := e.moduleMetas.Load(moduleName) if !ok { - return fmt.Errorf("Module %q not found", moduleName) + return fmt.Errorf("module %q not found", moduleName) } for _, dep := range meta.module.Dependencies { @@ -770,6 +774,11 @@ func (e *Engine) tryBuild(ctx context.Context, mustBuild map[string]bool, module err := e.build(ctx, moduleName, builtModules, schemas) if err == nil && callback != nil { + // load latest meta as it may have been updated + meta, ok = e.moduleMetas.Load(moduleName) + if !ok { + return fmt.Errorf("module %q not found", moduleName) + } return callback(ctx, meta.module) } @@ -802,11 +811,20 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ e.listener.OnBuildStarted(meta.module) } - moduleSchema, err := build(ctx, meta.plugin, e.projectRoot, sch, meta.module.Config, e.buildEnv, e.devMode) + moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectRoot, sch, meta.module.Config, e.buildEnv, e.devMode) if err != nil { terminal.UpdateModuleState(ctx, moduleName, terminal.BuildStateFailed) return err } + // update files to deploy + e.moduleMetas.Compute(moduleName, func(meta moduleMeta, exists bool) (out moduleMeta, shouldDelete bool) { + if !exists { + return moduleMeta{}, true + } + meta.module = meta.module.CopyWithDeploy(deploy) + return meta, false + }) + terminal.UpdateModuleState(ctx, moduleName, terminal.BuildStateBuilt) schemas <- moduleSchema return nil @@ -911,14 +929,16 @@ func (e *Engine) listenForBuildUpdates(originalCtx context.Context) { } case languageplugin.AutoRebuildEndedEvent: - if _, err := handleBuildResult(ctx, meta.module.Config, event.Result); err != nil { + _, deploy, err := handleBuildResult(ctx, meta.module.Config, event.Result) + if err != nil { logger.Errorf(err, "build failed") e.reportBuildFailed(err) terminal.UpdateModuleState(ctx, event.Module, terminal.BuildStateFailed) continue } + // TODO: update deploy dirs terminal.UpdateModuleState(ctx, event.Module, terminal.BuildStateDeploying) - if err := Deploy(ctx, meta.module, 1, true, e.client); err != nil { + if err := Deploy(ctx, meta.module, deploy, 1, true, e.client); err != nil { logger.Errorf(err, "deploy failed") e.reportBuildFailed(err) } else { diff --git a/internal/buildengine/languageplugin/go_plugin.go b/internal/buildengine/languageplugin/go_plugin.go index 4c7443828..5266c5d07 100644 --- a/internal/buildengine/languageplugin/go_plugin.go +++ b/internal/buildengine/languageplugin/go_plugin.go @@ -51,7 +51,6 @@ func (p *goPlugin) ModuleConfigDefaults(ctx context.Context, dir string) (module return moduleconfig.CustomDefaults{ Watch: watch, DeployDir: deployDir, - Deploy: []string{"main", "launch"}, }, nil } @@ -160,9 +159,14 @@ func (p *goPlugin) GetDependencies(ctx context.Context, config moduleconfig.Modu }) } -func buildGo(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) error { - if err := compile.Build(ctx, projectRoot, config.Dir, config, sch, transaction, buildEnv, devMode); err != nil { - return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", config.Module, err)} +func buildGo(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) (BuildResult, error) { + moduleSch, buildErrs, err := compile.Build(ctx, projectRoot, config.Dir, config, sch, transaction, buildEnv, devMode) + if err != nil { + return BuildResult{}, CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", config.Module, err)} } - return nil + return BuildResult{ + Errors: buildErrs, + Schema: moduleSch, + Deploy: []string{"main", "launch"}, + }, nil } diff --git a/internal/buildengine/languageplugin/go_plugin_test.go b/internal/buildengine/languageplugin/go_plugin_test.go index 56091e56f..1d699f69f 100644 --- a/internal/buildengine/languageplugin/go_plugin_test.go +++ b/internal/buildengine/languageplugin/go_plugin_test.go @@ -55,10 +55,6 @@ func TestGoConfigDefaults(t *testing.T) { { dir: "../testdata/alpha", expected: moduleconfig.CustomDefaults{ - Deploy: []string{ - "main", - "launch", - }, DeployDir: ".ftl", Watch: []string{ "**/*.go", @@ -71,10 +67,6 @@ func TestGoConfigDefaults(t *testing.T) { { dir: "../testdata/another", expected: moduleconfig.CustomDefaults{ - Deploy: []string{ - "main", - "launch", - }, DeployDir: ".ftl", Watch: []string{ "**/*.go", diff --git a/internal/buildengine/languageplugin/java_plugin.go b/internal/buildengine/languageplugin/java_plugin.go index bb10f76f2..14de6643e 100644 --- a/internal/buildengine/languageplugin/java_plugin.go +++ b/internal/buildengine/languageplugin/java_plugin.go @@ -17,9 +17,13 @@ import ( "github.com/beevik/etree" "github.com/go-viper/mapstructure/v2" "golang.org/x/exp/maps" + "google.golang.org/protobuf/proto" "github.com/TBD54566975/ftl" + languagepb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/language" "github.com/TBD54566975/ftl/internal" + "github.com/TBD54566975/ftl/internal/builderrors" + "github.com/TBD54566975/ftl/internal/errors" "github.com/TBD54566975/ftl/internal/exec" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/moduleconfig" @@ -63,7 +67,6 @@ func newJavaPlugin(ctx context.Context, language string) *javaPlugin { func (p *javaPlugin) ModuleConfigDefaults(ctx context.Context, dir string) (moduleconfig.CustomDefaults, error) { defaults := moduleconfig.CustomDefaults{ GeneratedSchemaDir: "src/main/ftl-module-schema", - Deploy: []string{"launch", "quarkus-app"}, // Watch defaults to files related to maven and gradle Watch: []string{"pom.xml", "src/**", "build/generated", "target/generated-sources"}, } @@ -244,11 +247,13 @@ func extractKotlinFTLImports(self, dir string) ([]string, error) { return modules, nil } -func buildJava(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) error { +func buildJava(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) (BuildResult, error) { + // TODO: add back + // Deploy: logger := log.FromContext(ctx) javaConfig, err := loadJavaConfig(config.LanguageConfig, config.Language) if err != nil { - return fmt.Errorf("failed to build module %q: %w", config.Module, err) + return BuildResult{}, fmt.Errorf("failed to build module %q: %w", config.Module, err) } if javaConfig.BuildTool == JavaBuildToolMaven { if err := setPOMProperties(ctx, config.Dir); err != nil { @@ -261,9 +266,29 @@ func buildJava(ctx context.Context, projectRoot string, config moduleconfig.AbsM command := exec.Command(ctx, log.Debug, config.Dir, "bash", "-c", config.Build) err = command.RunBuffered(ctx) if err != nil { - return fmt.Errorf("failed to build module %q: %w", config.Module, err) + return BuildResult{}, fmt.Errorf("failed to build module %q: %w", config.Module, err) } - return nil + + buildErrs, err := loadProtoErrors(config) + if err != nil { + return BuildResult{}, fmt.Errorf("failed to load build errors: %w", err) + } + result := BuildResult{ + Errors: buildErrs, + } + if builderrors.ContainsTerminalError(buildErrs) { + // skip reading schema + return result, nil + } + + moduleSchema, err := schema.ModuleFromProtoFile(config.Schema()) + if err != nil { + return BuildResult{}, fmt.Errorf("failed to read schema for module: %w", err) + } + + result.Schema = moduleSchema + result.Deploy = []string{"launch", "quarkus-app"} + return result, nil } // setPOMProperties updates the ftl.version properties in the @@ -326,3 +351,22 @@ func updatePomProperties(root *etree.Element, pomFile string, ftlVersion string) version.SetText(ftlVersion) return nil } + +func loadProtoErrors(config moduleconfig.AbsModuleConfig) ([]builderrors.Error, error) { + errorsPath := filepath.Join(config.DeployDir, "errors.pb") + if _, err := os.Stat(errorsPath); errors.Is(err, os.ErrNotExist) { + return nil, nil + } + + content, err := os.ReadFile(errorsPath) + if err != nil { + return nil, fmt.Errorf("could not load build errors file: %w", err) + } + + errorspb := &languagepb.ErrorList{} + err = proto.Unmarshal(content, errorspb) + if err != nil { + return nil, fmt.Errorf("could not unmarshal build errors %w", err) + } + return languagepb.ErrorsFromProto(errorspb), nil +} diff --git a/internal/buildengine/languageplugin/java_plugin_test.go b/internal/buildengine/languageplugin/java_plugin_test.go index b156cd2af..601047bca 100644 --- a/internal/buildengine/languageplugin/java_plugin_test.go +++ b/internal/buildengine/languageplugin/java_plugin_test.go @@ -32,11 +32,7 @@ func TestJavaConfigDefaults(t *testing.T) { language: "kotlin", dir: "testdata/echokotlin", expected: moduleconfig.CustomDefaults{ - Build: "mvn -B package", - Deploy: []string{ - "launch", - "quarkus-app", - }, + Build: "mvn -B package", DeployDir: "target", GeneratedSchemaDir: "src/main/ftl-module-schema", Watch: watch, @@ -49,11 +45,7 @@ func TestJavaConfigDefaults(t *testing.T) { language: "kotlin", dir: "testdata/externalkotlin", expected: moduleconfig.CustomDefaults{ - Build: "mvn -B package", - Deploy: []string{ - "launch", - "quarkus-app", - }, + Build: "mvn -B package", DeployDir: "target", GeneratedSchemaDir: "src/main/ftl-module-schema", Watch: watch, diff --git a/internal/buildengine/languageplugin/plugin.go b/internal/buildengine/languageplugin/plugin.go index 1964dddf9..23aa6e4cd 100644 --- a/internal/buildengine/languageplugin/plugin.go +++ b/internal/buildengine/languageplugin/plugin.go @@ -10,11 +10,8 @@ import ( "github.com/alecthomas/kong" "github.com/alecthomas/types/either" "github.com/alecthomas/types/pubsub" - "google.golang.org/protobuf/proto" - languagepb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/language" "github.com/TBD54566975/ftl/internal/builderrors" - "github.com/TBD54566975/ftl/internal/errors" "github.com/TBD54566975/ftl/internal/flock" "github.com/TBD54566975/ftl/internal/moduleconfig" "github.com/TBD54566975/ftl/internal/projectconfig" @@ -25,10 +22,13 @@ import ( const BuildLockTimeout = time.Minute type BuildResult struct { - Name string - Errors []builderrors.Error - Schema *schema.Module StartTime time.Time + + Schema *schema.Module + Errors []builderrors.Error + + // Files to deploy, relative to the module config's DeployDir + Deploy []string } // PluginEvent is used to notify of updates from the plugin. @@ -129,7 +129,7 @@ type getDependenciesCommand struct { func (getDependenciesCommand) pluginCmd() {} -type buildFunc = func(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) error +type buildFunc = func(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) (BuildResult, error) type CompilerBuildError struct { err error @@ -328,47 +328,10 @@ func buildAndLoadResult(ctx context.Context, projectRoot string, c moduleconfig. } transaction := watcher.GetTransaction(config.Dir) - err = build(ctx, projectRoot, config, sch, buildEnv, devMode, transaction) + result, err := build(ctx, projectRoot, config, sch, buildEnv, devMode, transaction) if err != nil { return BuildResult{}, err } - errors, err := loadProtoErrors(config) - if err != nil { - return BuildResult{}, fmt.Errorf("failed to read build errors for module: %w", err) - } - - result := BuildResult{ - Errors: errors, - StartTime: startTime, - } - - if builderrors.ContainsTerminalError(errors) { - // skip reading schema - return result, nil - } - - moduleSchema, err := schema.ModuleFromProtoFile(config.Schema()) - if err != nil { - return BuildResult{}, fmt.Errorf("failed to read schema for module: %w", err) - } - result.Schema = moduleSchema + result.StartTime = startTime return result, nil } - -func loadProtoErrors(config moduleconfig.AbsModuleConfig) ([]builderrors.Error, error) { - if _, err := os.Stat(config.Errors); errors.Is(err, os.ErrNotExist) { - return nil, nil - } - - content, err := os.ReadFile(config.Errors) - if err != nil { - return nil, fmt.Errorf("could not load build errors file: %w", err) - } - - errorspb := &languagepb.ErrorList{} - err = proto.Unmarshal(content, errorspb) - if err != nil { - return nil, fmt.Errorf("could not unmarshal build errors %w", err) - } - return languagepb.ErrorsFromProto(errorspb), nil -} diff --git a/internal/buildengine/languageplugin/rust_plugin.go b/internal/buildengine/languageplugin/rust_plugin.go index ad4612826..7bf7d4aca 100644 --- a/internal/buildengine/languageplugin/rust_plugin.go +++ b/internal/buildengine/languageplugin/rust_plugin.go @@ -6,6 +6,7 @@ import ( "github.com/alecthomas/kong" + "github.com/TBD54566975/ftl/internal/builderrors" "github.com/TBD54566975/ftl/internal/exec" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/moduleconfig" @@ -31,7 +32,6 @@ func (p *rustPlugin) ModuleConfigDefaults(ctx context.Context, dir string) (modu return moduleconfig.CustomDefaults{ Build: "cargo build", DeployDir: "_ftl/target/debug", - Deploy: []string{"main"}, Watch: []string{"**/*.rs", "Cargo.toml", "Cargo.lock"}, }, nil } @@ -48,12 +48,32 @@ func (p *rustPlugin) GetDependencies(ctx context.Context, config moduleconfig.Mo return nil, fmt.Errorf("not implemented") } -func buildRust(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) error { +func buildRust(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) (BuildResult, error) { logger := log.FromContext(ctx) logger.Debugf("Using build command '%s'", config.Build) err := exec.Command(ctx, log.Debug, config.Dir+"/_ftl", "bash", "-c", config.Build).RunBuffered(ctx) if err != nil { - return fmt.Errorf("failed to build module %q: %w", config.Module, err) + return BuildResult{}, fmt.Errorf("failed to build module %q: %w", config.Module, err) } - return nil + buildErrs, err := loadProtoErrors(config) + if err != nil { + return BuildResult{}, fmt.Errorf("failed to load build errors: %w", err) + } + result := BuildResult{ + Errors: buildErrs, + } + if builderrors.ContainsTerminalError(buildErrs) { + // skip reading schema + return result, nil + } + + moduleSchema, err := schema.ModuleFromProtoFile(config.Schema()) + if err != nil { + return BuildResult{}, fmt.Errorf("failed to read schema for module: %w", err) + } + + result.Schema = moduleSchema + result.Deploy = []string{"main"} + + return result, nil } diff --git a/internal/buildengine/module.go b/internal/buildengine/module.go index 31a725716..0f807e094 100644 --- a/internal/buildengine/module.go +++ b/internal/buildengine/module.go @@ -9,6 +9,8 @@ import ( type Module struct { Config moduleconfig.ModuleConfig Dependencies []string + // paths to deploy, relative to ModuleConfig.DeployDir + Deploy []string } func (m Module) CopyWithDependencies(dependencies []string) Module { @@ -16,3 +18,9 @@ func (m Module) CopyWithDependencies(dependencies []string) Module { module.Dependencies = dependencies return module } + +func (m Module) CopyWithDeploy(files []string) Module { + module := reflect.DeepCopy(m) + module.Deploy = files + return module +} diff --git a/internal/moduleconfig/moduleconfig.go b/internal/moduleconfig/moduleconfig.go index 65be0ad80..ff138f058 100644 --- a/internal/moduleconfig/moduleconfig.go +++ b/internal/moduleconfig/moduleconfig.go @@ -23,14 +23,10 @@ type ModuleConfig struct { Module string `toml:"module"` // Build is the command to build the module. Build string `toml:"build"` - // Deploy is the list of files to deploy relative to the DeployDir. - Deploy []string `toml:"deploy"` // DeployDir is the directory to deploy from, relative to the module directory. DeployDir string `toml:"deploy-dir"` // GeneratedSchemaDir is the directory to generate protobuf schema files into. These can be picked up by language specific build tools GeneratedSchemaDir string `toml:"generated-schema-dir"` - // Errors is the name of the error file relative to the DeployDir. - Errors string `toml:"errors"` // Watch is the list of files to watch for changes. Watch []string `toml:"watch"` @@ -56,10 +52,8 @@ type UnvalidatedModuleConfig ModuleConfig type CustomDefaults struct { Build string - Deploy []string DeployDir string GeneratedSchemaDir string - Errors string Watch []string // only the root keys in LanguageConfig are used to find missing values that can be defaulted @@ -128,17 +122,6 @@ func (c ModuleConfig) Abs() AbsModuleConfig { panic(fmt.Sprintf("generated-schema-dir %q is not beneath module directory %q", clone.GeneratedSchemaDir, clone.Dir)) } } - clone.Errors = filepath.Clean(filepath.Join(clone.DeployDir, clone.Errors)) - if !strings.HasPrefix(clone.Errors, clone.DeployDir) { - panic(fmt.Sprintf("errors %q is not beneath deploy directory %q", clone.Errors, clone.DeployDir)) - } - clone.Deploy = slices.Map(clone.Deploy, func(p string) string { - out := filepath.Clean(filepath.Join(clone.DeployDir, p)) - if !strings.HasPrefix(out, clone.DeployDir) { - panic(fmt.Sprintf("deploy path %q is not beneath deploy directory %q", out, clone.DeployDir)) - } - return out - }) // Watch paths are allowed to be outside the deploy directory. clone.Watch = slices.Map(clone.Watch, func(p string) string { return filepath.Clean(filepath.Join(clone.Dir, p)) @@ -153,9 +136,6 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe if c.Realm == "" { c.Realm = "home" } - if c.Errors == "" { - c.Errors = "errors.pb" - } // Custom defaults if c.Build == "" { @@ -164,9 +144,6 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe if c.DeployDir == "" { c.DeployDir = customDefaults.DeployDir } - if c.Deploy == nil { - c.Deploy = customDefaults.Deploy - } if c.GeneratedSchemaDir == "" { c.GeneratedSchemaDir = customDefaults.GeneratedSchemaDir } @@ -185,24 +162,12 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe } // Validate - if len(c.Deploy) == 0 { - return ModuleConfig{}, fmt.Errorf("no deploy files configured") - } if c.DeployDir == "" { return ModuleConfig{}, fmt.Errorf("no deploy directory configured") } - if c.Errors == "" { - return ModuleConfig{}, fmt.Errorf("no errors file configured") - } if !isBeneath(c.Dir, c.DeployDir) { return ModuleConfig{}, fmt.Errorf("deploy-dir %s must be relative to the module directory %s", c.DeployDir, c.Dir) } - for _, deploy := range c.Deploy { - if !isBeneath(c.Dir, deploy) { - return ModuleConfig{}, fmt.Errorf("deploy %s files must be relative to the module directory %s", deploy, c.Dir) - } - } - c.Deploy = slices.Sort(c.Deploy) c.Watch = slices.Sort(c.Watch) return ModuleConfig(c), nil } diff --git a/internal/moduleconfig/moduleconfig_test.go b/internal/moduleconfig/moduleconfig_test.go index 44da74ee4..e9a1b372c 100644 --- a/internal/moduleconfig/moduleconfig_test.go +++ b/internal/moduleconfig/moduleconfig_test.go @@ -22,10 +22,8 @@ func TestDefaulting(t *testing.T) { }, defaults: CustomDefaults{ Build: "build", - Deploy: []string{"deploy"}, DeployDir: "deploydir", GeneratedSchemaDir: "generatedschemadir", - Errors: "errors.pb", Watch: []string{"a", "b", "c"}, }, expected: ModuleConfig{ @@ -34,10 +32,8 @@ func TestDefaulting(t *testing.T) { Module: "nothingset", Language: "test", Build: "build", - Deploy: []string{"deploy"}, DeployDir: "deploydir", GeneratedSchemaDir: "generatedschemadir", - Errors: "errors.pb", Watch: []string{"a", "b", "c"}, }, }, @@ -47,10 +43,8 @@ func TestDefaulting(t *testing.T) { Module: "allset", Language: "test", Build: "custombuild", - Deploy: []string{"customdeploy"}, DeployDir: "customdeploydir", GeneratedSchemaDir: "customgeneratedschemadir", - Errors: "customerrors.pb", Watch: []string{"custom1"}, LanguageConfig: map[string]any{ "build-tool": "maven", @@ -59,10 +53,8 @@ func TestDefaulting(t *testing.T) { }, defaults: CustomDefaults{ Build: "build", - Deploy: []string{"deploy"}, DeployDir: "deploydir", GeneratedSchemaDir: "generatedschemadir", - Errors: "errors.pb", Watch: []string{"a", "b", "c"}, }, expected: ModuleConfig{ @@ -71,10 +63,8 @@ func TestDefaulting(t *testing.T) { Module: "allset", Language: "test", Build: "custombuild", - Deploy: []string{"customdeploy"}, DeployDir: "customdeploydir", GeneratedSchemaDir: "customgeneratedschemadir", - Errors: "customerrors.pb", Watch: []string{"custom1"}, LanguageConfig: map[string]any{ "build-tool": "maven", @@ -97,7 +87,6 @@ func TestDefaulting(t *testing.T) { }, }, defaults: CustomDefaults{ - Deploy: []string{"example"}, DeployDir: "deploydir", LanguageConfig: map[string]any{ "alreadyset": "incorrect", @@ -109,9 +98,7 @@ func TestDefaulting(t *testing.T) { }, }, expected: ModuleConfig{ - Deploy: []string{"example"}, DeployDir: "deploydir", - Errors: "errors.pb", Realm: "home", Dir: "b", Module: "languageconfig", @@ -128,39 +115,14 @@ func TestDefaulting(t *testing.T) { }, // Validation failures - { - config: UnvalidatedModuleConfig{ - Dir: "b", - Module: "nodeploy", - Language: "test", - }, - defaults: CustomDefaults{ - DeployDir: "deploydir", - }, - error: "no deploy files configured", - }, { config: UnvalidatedModuleConfig{ Dir: "b", Module: "nodeploydir", Language: "test", }, - defaults: CustomDefaults{ - Deploy: []string{"example"}, - }, - error: "no deploy directory configured", - }, - { - config: UnvalidatedModuleConfig{ - Dir: "b", - Module: "deploynotindir", - Language: "test", - }, - defaults: CustomDefaults{ - Deploy: []string{"example"}, - DeployDir: "../../deploydir", - }, - error: "must be relative to the module directory", + defaults: CustomDefaults{}, + error: "no deploy directory configured", }, } { t.Run(tt.config.Module, func(t *testing.T) { @@ -168,7 +130,7 @@ func TestDefaulting(t *testing.T) { config, err := tt.config.FillDefaultsAndValidate(tt.defaults) if tt.error != "" { - assert.Contains(t, err.Error(), tt.error) + assert.EqualError(t, err, tt.error) return } assert.NoError(t, err)