diff --git a/.gitignore b/.gitignore index 31dcf4448..e54ce3717 100644 --- a/.gitignore +++ b/.gitignore @@ -20,15 +20,9 @@ examples/**/go.work examples/**/go.work.sum testdata/**/go.work testdata/**/go.work.sum - -# Leaving old _ftl for now to avoid old stuff getting checked in **/testdata/**/_ftl **/examples/**/_ftl **/examples/**/types.ftl.go -**/testdata/**/.ftl -**/examples/**/.ftl -/.ftl - buildengine/.gitignore go.work* junit*.xml diff --git a/Justfile b/Justfile index 50af62da2..0f6d1eb3f 100644 --- a/Justfile +++ b/Justfile @@ -9,7 +9,7 @@ KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar" RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip" TIMESTAMP := `date +%s` SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto" -ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/compile/main-work-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" +ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" FRONTEND_OUT := "frontend/dist/index.html" EXTENSION_OUT := "extensions/vscode/dist/extension.js" PROTOS_IN := "backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/console/console.proto backend/protos/xyz/block/ftl/v1/ftl.proto backend/protos/xyz/block/ftl/v1/schema/runtime.proto" diff --git a/buildengine/build.go b/buildengine/build.go index 3a50e0522..298722399 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -22,11 +22,11 @@ const BuildLockTimeout = time.Minute // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - return buildModule(ctx, projectRootDir, sch, module, filesTransaction) +func Build(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + return buildModule(ctx, sch, module, filesTransaction) } -func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { +func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err @@ -46,7 +46,7 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, switch module.Config.Language { case "go": - err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) + err = buildGoModule(ctx, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) case "rust": diff --git a/buildengine/build_go.go b/buildengine/build_go.go index 8eba01a74..ee8fb1c98 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction); err != nil { +func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { + if err := compile.Build(ctx, module.Config.Dir, sch, transaction); err != nil { return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)} } return nil diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index cdfd9c821..498e26e43 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -11,6 +11,181 @@ import ( "github.com/TBD54566975/ftl/backend/schema" ) +func TestGenerateGoModule(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + sch := &schema.Schema{ + Modules: []*schema.Module{ + schema.Builtins(), + {Name: "other", Decls: []schema.Decl{ + &schema.Enum{ + Comments: []string{"This is an enum.", "", "It has 3 variants."}, + Name: "Color", + Export: true, + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + }, + }, + &schema.Enum{ + Name: "ColorInt", + Export: true, + Type: &schema.Int{}, + Variants: []*schema.EnumVariant{ + {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, + {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, + {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, + }, + }, + &schema.Enum{ + Comments: []string{"This is type enum."}, + Name: "TypeEnum", + Export: true, + Variants: []*schema.EnumVariant{ + {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, + {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, + }, + }, + &schema.Data{Name: "EchoRequest", Export: true}, + &schema.Data{ + Comments: []string{"This is an echo data response."}, + Name: "EchoResponse", Export: true}, + &schema.Verb{ + Name: "echo", + Export: true, + Request: &schema.Ref{Name: "EchoRequest"}, + Response: &schema.Ref{Name: "EchoResponse"}, + }, + &schema.Data{Name: "SinkReq", Export: true}, + &schema.Verb{ + Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, + Name: "sink", + Export: true, + Request: &schema.Ref{Name: "SinkReq"}, + Response: &schema.Unit{}, + }, + &schema.Data{Name: "SourceResp", Export: true}, + &schema.Verb{ + Name: "source", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Ref{Name: "SourceResp"}, + }, + &schema.Verb{ + Name: "nothing", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Unit{}, + }, + }}, + {Name: "test"}, + }, + } + expected := `// Code generated by FTL. DO NOT EDIT. + +package other + +import ( + "context" + + "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" +) + +var _ = context.Background + +// This is an enum. +// +// It has 3 variants. +// +//ftl:enum +type Color string +const ( + Red Color = "Red" + Blue Color = "Blue" + Green Color = "Green" +) + +//ftl:enum +type ColorInt int +const ( + RedInt ColorInt = 0 + BlueInt ColorInt = 1 + GreenInt ColorInt = 2 +) + +// This is type enum. +// +//ftl:enum +type TypeEnum interface { typeEnum() } + +type A int + +func (A) typeEnum() {} + +type B string + +func (B) typeEnum() {} + +type EchoRequest struct { +} + +// This is an echo data response. +// +type EchoResponse struct { +} + +//ftl:verb +func Echo(context.Context, EchoRequest) (EchoResponse, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} + +type SinkReq struct { +} + +// This is a sink verb. +// +// Here is another line for this comment! +// +//ftl:verb +func Sink(context.Context, SinkReq) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") +} + +type SourceResp struct { +} + +//ftl:verb +func Source(context.Context) (SourceResp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") +} + +//ftl:verb +func Nothing(context.Context) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") +} + +func init() { + reflection.Register( + reflection.SumType[TypeEnum]( + *new(A), + *new(B), + ), + ) +} +` + bctx := buildContext{ + moduleDir: "testdata/another", + buildDir: "_ftl", + sch: sch, + } + testBuild(t, bctx, "", []assertion{ + assertGeneratedModule("go/modules/other/external_module.go", expected), + }) +} + func TestGoBuildClearsBuildDir(t *testing.T) { if testing.Short() { t.SkipNow() @@ -23,22 +198,82 @@ func TestGoBuildClearsBuildDir(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/another", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } testBuildClearsBuildDir(t, bctx) } +func TestMetadataImportsExcluded(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + sch := &schema.Schema{ + Modules: []*schema.Module{ + schema.Builtins(), + {Name: "test", Decls: []schema.Decl{ + &schema.Data{ + Comments: []string{"Request data type."}, + Name: "Req", Export: true}, + &schema.Data{Name: "Resp", Export: true}, + &schema.Verb{ + Comments: []string{"This is a verb."}, + Name: "call", + Export: true, + Request: &schema.Ref{Name: "Req"}, + Response: &schema.Ref{Name: "Resp"}, + Metadata: []schema.Metadata{ + &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, + }, + }, + }}, + }, + } + expected := `// Code generated by FTL. DO NOT EDIT. + +package test + +import ( + "context" +) + +var _ = context.Background + +// Request data type. +// +type Req struct { +} + +type Resp struct { +} + +// This is a verb. +// +//ftl:verb +func Call(context.Context, Req) (Resp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} +` + bctx := buildContext{ + moduleDir: "testdata/another", + buildDir: "_ftl", + sch: sch, + } + testBuild(t, bctx, "", []assertion{ + assertGeneratedModule("go/modules/test/external_module.go", expected), + }) +} + func TestExternalType(t *testing.T) { if testing.Short() { t.SkipNow() } bctx := buildContext{ moduleDir: "testdata/external", - buildDir: ".ftl", + buildDir: "_ftl", sch: &schema.Schema{}, } - testBuild(t, bctx, "", "unsupported external type", []assertion{ + testBuild(t, bctx, "unsupported external type", []assertion{ assertBuildProtoErrors( "unsupported external type \"time.Month\"", "unsupported type \"time.Month\" for field \"Month\"", @@ -67,10 +302,10 @@ func TestGoModVersion(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/highgoversion", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } - testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), "", []assertion{}) + testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), []assertion{}) } func TestGeneratedTypeRegistry(t *testing.T) { @@ -110,10 +345,10 @@ func TestGeneratedTypeRegistry(t *testing.T) { assert.NoError(t, err) bctx := buildContext{ moduleDir: "testdata/other", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } - testBuild(t, bctx, "", "", []assertion{ + testBuild(t, bctx, "", []assertion{ assertGeneratedMain(string(expected)), }) } diff --git a/buildengine/build_test.go b/buildengine/build_test.go index e29e244ea..83a05b6c0 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -38,7 +38,6 @@ func (t *mockModifyFilesTransaction) End() error { func testBuild( t *testing.T, bctx buildContext, - expectedGeneratStubsErrMsg string, // emptystr if no error expected expectedBuildErrMsg string, // emptystr if no error expected assertions []assertion, ) { @@ -48,31 +47,12 @@ func testBuild( assert.NoError(t, err, "Error getting absolute path for module directory") module, err := LoadModule(abs) assert.NoError(t, err) - - projectRootDir := t.TempDir() - - configs := []moduleconfig.ModuleConfig{} - if bctx.moduleDir != "" { - config, err := moduleconfig.LoadModuleConfig(bctx.moduleDir) - assert.NoError(t, err, "Error loading project config") - configs = append(configs, config) - } - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRootDir, bctx.sch.Modules, configs) - if len(expectedGeneratStubsErrMsg) > 0 { + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + if len(expectedBuildErrMsg) > 0 { assert.Error(t, err) - assert.Contains(t, err.Error(), expectedGeneratStubsErrMsg) + assert.Contains(t, err.Error(), expectedBuildErrMsg) } else { assert.NoError(t, err) - - err = Build(ctx, projectRootDir, bctx.sch, module, &mockModifyFilesTransaction{}) - if len(expectedBuildErrMsg) > 0 { - assert.Error(t, err) - assert.Contains(t, err.Error(), expectedBuildErrMsg) - } else { - assert.NoError(t, err) - } } for _, a := range assertions { @@ -90,16 +70,10 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { abs, err := filepath.Abs(bctx.moduleDir) assert.NoError(t, err, "Error getting absolute path for module directory") - projectRoot := t.TempDir() - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir}}) - assert.NoError(t, err) - // build to generate the build directory module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // create a temporary file in the build directory @@ -111,7 +85,7 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { // build to clear the old build directory module, err = LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // ensure the temporary file was removed diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index 3550668d2..029f2d33a 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -10,7 +10,6 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/sha256" ) @@ -72,17 +71,11 @@ func TestDeploy(t *testing.T) { module, err := LoadModule(modulePath) assert.NoError(t, err) - projectRootDir := t.TempDir() - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRootDir, sch.Modules, []moduleconfig.ModuleConfig{module.Config}) - assert.NoError(t, err) - // Build first to make sure the files are there. - err = Build(ctx, projectRootDir, sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) - sum, err := sha256.SumFile(modulePath + "/.ftl/main") + sum, err := sha256.SumFile(modulePath + "/_ftl/main") assert.NoError(t, err) client := &mockDeployClient{ diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index 0aec6ccc9..b63d7d341 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -22,7 +22,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "alpha", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -35,7 +35,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "another", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -48,7 +48,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle1", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -61,7 +61,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle2", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -100,7 +100,7 @@ func TestDiscoverModules(t *testing.T) { Deploy: []string{ "main", }, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{ @@ -140,7 +140,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "highgoversion", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -153,7 +153,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "other", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, diff --git a/buildengine/engine.go b/buildengine/engine.go index dbc109e15..b9a7b4215 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -21,7 +21,6 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" "github.com/TBD54566975/ftl/internal/slices" @@ -66,7 +65,6 @@ type Listener interface { type Engine struct { client ftlv1connect.ControllerServiceClient moduleMetas *xsync.MapOf[string, moduleMeta] - projectRoot string moduleDirs []string watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] @@ -99,11 +97,10 @@ func WithListener(listener Listener) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, projectRoot string, moduleDirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, - projectRoot: projectRoot, moduleDirs: moduleDirs, moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: NewWatcher(), @@ -569,21 +566,6 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } errCh := make(chan error, 1024) for _, group := range topology { - groupSchemas := map[string]*schema.Module{} - metas, err := e.gatherGroupSchemas(builtModules, group, groupSchemas) - if err != nil { - return err - } - - moduleConfigs := make([]moduleconfig.ModuleConfig, len(metas)) - for i, meta := range metas { - moduleConfigs[i] = meta.module.Config - } - err = GenerateStubs(ctx, e.projectRoot, maps.Values(groupSchemas), moduleConfigs) - if err != nil { - return err - } - // Collect schemas to be inserted into "built" map for subsequent groups. schemas := make(chan *schema.Module, len(group)) @@ -682,7 +664,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ if e.listener != nil { e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) + err := Build(ctx, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) if err != nil { return err } @@ -695,29 +677,6 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ return nil } -// Construct a combined schema for a group of modules and their transitive dependencies. -func (e *Engine) gatherGroupSchemas( - moduleSchemas map[string]*schema.Module, - group []string, - out map[string]*schema.Module, -) ([]moduleMeta, error) { - var metas []moduleMeta - for _, module := range group { - if module == "builtin" { - continue // Skip the builtin module - } - - meta, ok := e.moduleMetas.Load(module) - if ok { - metas = append(metas, meta) - if err := e.gatherSchemas(moduleSchemas, meta.module, out); err != nil { - return nil, err - } - } - } - return metas, nil -} - // Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index a44ab8768..73a521134 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -16,7 +16,7 @@ func TestEngine(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/alpha", "testdata/other", "testdata/another"}) + engine, err := buildengine.New(ctx, nil, []string{"testdata/alpha", "testdata/other", "testdata/another"}) assert.NoError(t, err) defer engine.Close() @@ -64,7 +64,7 @@ func TestCycleDetection(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/depcycle1", "testdata/depcycle2"}) + engine, err := buildengine.New(ctx, nil, []string{"testdata/depcycle1", "testdata/depcycle2"}) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/stubs.go b/buildengine/stubs.go deleted file mode 100644 index 911d00d10..000000000 --- a/buildengine/stubs.go +++ /dev/null @@ -1,26 +0,0 @@ -package buildengine - -import ( - "context" - "fmt" - - "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" - "github.com/TBD54566975/ftl/go-runtime/compile" -) - -// GenerateStubs generates stubs for the given modules. -// -// Currently, only Go stubs are supported. Kotlin and other language stubs can be added in the future. -func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { - return generateGoStubs(ctx, projectRoot, modules, moduleConfigs) -} - -func generateGoStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { - sch := &schema.Schema{Modules: modules} - err := compile.GenerateStubsForModules(ctx, projectRoot, moduleConfigs, sch) - if err != nil { - return fmt.Errorf("failed to generate Go stubs: %w", err) - } - return nil -} diff --git a/buildengine/stubs_test.go b/buildengine/stubs_test.go deleted file mode 100644 index c4c1efe5b..000000000 --- a/buildengine/stubs_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package buildengine - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" - "github.com/TBD54566975/ftl/internal/log" - "github.com/alecthomas/assert/v2" -) - -func TestGenerateGoStubs(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - modules := []*schema.Module{ - schema.Builtins(), - {Name: "other", Decls: []schema.Decl{ - &schema.Enum{ - Comments: []string{"This is an enum.", "", "It has 3 variants."}, - Name: "Color", - Export: true, - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, - }, - &schema.Enum{ - Name: "ColorInt", - Export: true, - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, - {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, - {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, - }, - }, - &schema.Enum{ - Comments: []string{"This is type enum."}, - Name: "TypeEnum", - Export: true, - Variants: []*schema.EnumVariant{ - {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, - {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, - }, - }, - &schema.Data{Name: "EchoRequest", Export: true}, - &schema.Data{ - Comments: []string{"This is an echo data response."}, - Name: "EchoResponse", Export: true}, - &schema.Verb{ - Name: "echo", - Export: true, - Request: &schema.Ref{Name: "EchoRequest"}, - Response: &schema.Ref{Name: "EchoResponse"}, - }, - &schema.Data{Name: "SinkReq", Export: true}, - &schema.Verb{ - Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, - Name: "sink", - Export: true, - Request: &schema.Ref{Name: "SinkReq"}, - Response: &schema.Unit{}, - }, - &schema.Data{Name: "SourceResp", Export: true}, - &schema.Verb{ - Name: "source", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Ref{Name: "SourceResp"}, - }, - &schema.Verb{ - Name: "nothing", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Unit{}, - }, - }}, - {Name: "test"}, - } - - expected := `// Code generated by FTL. DO NOT EDIT. - -package other - -import ( - "context" - - "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" -) - -var _ = context.Background - -// This is an enum. -// -// It has 3 variants. -// -//ftl:enum -type Color string -const ( - Red Color = "Red" - Blue Color = "Blue" - Green Color = "Green" -) - -//ftl:enum -type ColorInt int -const ( - RedInt ColorInt = 0 - BlueInt ColorInt = 1 - GreenInt ColorInt = 2 -) - -// This is type enum. -// -//ftl:enum -type TypeEnum interface { typeEnum() } - -type A int - -func (A) typeEnum() {} - -type B string - -func (B) typeEnum() {} - -type EchoRequest struct { -} - -// This is an echo data response. -// -type EchoResponse struct { -} - -//ftl:verb -func Echo(context.Context, EchoRequest) (EchoResponse, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} - -type SinkReq struct { -} - -// This is a sink verb. -// -// Here is another line for this comment! -// -//ftl:verb -func Sink(context.Context, SinkReq) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") -} - -type SourceResp struct { -} - -//ftl:verb -func Source(context.Context) (SourceResp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") -} - -//ftl:verb -func Nothing(context.Context) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") -} - -func init() { - reflection.Register( - reflection.SumType[TypeEnum]( - *new(A), - *new(B), - ), - ) -} -` - - ctx := log.ContextWithNewDefaultLogger(context.Background()) - projectRoot := t.TempDir() - err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) - assert.NoError(t, err) - - generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/other/external_module.go") - fileContent, err := os.ReadFile(generatedPath) - assert.NoError(t, err) - assert.Equal(t, expected, string(fileContent)) -} - -func TestMetadataImportsExcluded(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - modules := []*schema.Module{ - schema.Builtins(), - {Name: "test", Decls: []schema.Decl{ - &schema.Data{ - Comments: []string{"Request data type."}, - Name: "Req", Export: true}, - &schema.Data{Name: "Resp", Export: true}, - &schema.Verb{ - Comments: []string{"This is a verb."}, - Name: "call", - Export: true, - Request: &schema.Ref{Name: "Req"}, - Response: &schema.Ref{Name: "Resp"}, - Metadata: []schema.Metadata{ - &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, - }, - }, - }}, - } - - expected := `// Code generated by FTL. DO NOT EDIT. - -package test - -import ( - "context" -) - -var _ = context.Background - -// Request data type. -// -type Req struct { -} - -type Resp struct { -} - -// This is a verb. -// -//ftl:verb -func Call(context.Context, Req) (Resp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} -` - ctx := log.ContextWithNewDefaultLogger(context.Background()) - projectRoot := t.TempDir() - err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) - assert.NoError(t, err) - - generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/test/external_module.go") - fileContent, err := os.ReadFile(generatedPath) - assert.NoError(t, err) - assert.Equal(t, expected, string(fileContent)) -} diff --git a/cmd/ftl/cmd_box.go b/cmd/ftl/cmd_box.go index 887eefb76..d09214c88 100644 --- a/cmd/ftl/cmd_box.go +++ b/cmd/ftl/cmd_box.go @@ -123,7 +123,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_box_run.go b/cmd/ftl/cmd_box_run.go index 0565edd54..0f9cb08b5 100644 --- a/cmd/ftl/cmd_box_run.go +++ b/cmd/ftl/cmd_box_run.go @@ -17,7 +17,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/buildengine" - "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -34,7 +33,7 @@ type boxRunCmd struct { ControllerTimeout time.Duration `help:"Timeout for Controller start." default:"30s"` } -func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { +func (b *boxRunCmd) Run(ctx context.Context) error { conn, err := databasetesting.CreateForDevel(ctx, b.DSN, b.Recreate) if err != nil { return fmt.Errorf("failed to create database: %w", err) @@ -75,7 +74,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er return fmt.Errorf("controller failed to start: %w", err) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), []string{b.Dir}) + engine, err := buildengine.New(ctx, client, []string{b.Dir}) if err != nil { return fmt.Errorf("failed to create build engine: %w", err) } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index a3f13869b..35e826e74 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -22,7 +22,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index a50c7f25f..0ecd47a6e 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -5,7 +5,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/buildengine" - "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -16,9 +15,9 @@ type deployCmd struct { NoWait bool `help:"Do not wait for deployment to complete." default:"false"` } -func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { +func (d *deployCmd) Run(ctx context.Context) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index e596d84fa..2741faecf 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -89,7 +89,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, opts...) + engine, err := buildengine.New(ctx, client, d.Dirs, opts...) if err != nil { return err } diff --git a/cmd/ftl/cmd_init.go b/cmd/ftl/cmd_init.go index b2cafb64e..b39463979 100644 --- a/cmd/ftl/cmd_init.go +++ b/cmd/ftl/cmd_init.go @@ -86,7 +86,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { scanner := bufio.NewScanner(f) for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "**/.ftl" { + if strings.TrimSpace(scanner.Text()) == "**/_ftl" { return nil } } @@ -96,7 +96,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { } // append if not already present - if _, err = f.WriteString("**/.ftl\n"); err != nil { + if _, err = f.WriteString("**/_ftl\n"); err != nil { return err } diff --git a/cmd/ftl/cmd_new.go b/cmd/ftl/cmd_new.go index 1afa32ad1..831364a03 100644 --- a/cmd/ftl/cmd_new.go +++ b/cmd/ftl/cmd_new.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" "github.com/TBD54566975/scaffolder" @@ -31,10 +30,9 @@ type newCmd struct { } type newGoCmd struct { - Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` - Dir string `arg:"" help:"Directory to initialize the module in."` - Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` - GoVersion string + Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` + Dir string `arg:"" help:"Directory to initialize the module in."` + Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` } type newKotlinCmd struct { @@ -60,8 +58,6 @@ func (i newGoCmd) Run(ctx context.Context) error { logger := log.FromContext(ctx) logger.Debugf("Creating FTL Go module %q in %s", name, path) - - i.GoVersion = runtime.Version()[2:] if err := scaffold(ctx, config.Hermit, goruntime.Files(), i.Dir, i, scaffolder.Exclude("^go.mod$")); err != nil { return err } diff --git a/common/moduleconfig/moduleconfig.go b/common/moduleconfig/moduleconfig.go index 990e96b73..dddad5134 100644 --- a/common/moduleconfig/moduleconfig.go +++ b/common/moduleconfig/moduleconfig.go @@ -133,7 +133,7 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error { case "go": if config.DeployDir == "" { - config.DeployDir = ".ftl" + config.DeployDir = "_ftl" } if len(config.Deploy) == 0 { config.Deploy = []string{"main"} diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index a539cf6ee..dbc72146e 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "ftl", - "version": "0.0.0", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ftl", - "version": "0.0.0", + "version": "0.1.2", "dependencies": { "lookpath": "^1.2.2", "semver": "^7.6.0", diff --git a/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl similarity index 94% rename from go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl rename to go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl index df0b6eca7..56b07af1f 100644 --- a/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl +++ b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl b/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go similarity index 100% rename from go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl rename to go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go diff --git a/go-runtime/compile/build-template/go.work.tmpl b/go-runtime/compile/build-template/go.work.tmpl index cbd6143cf..683cd1d34 100644 --- a/go-runtime/compile/build-template/go.work.tmpl +++ b/go-runtime/compile/build-template/go.work.tmpl @@ -2,8 +2,6 @@ go {{ .GoVersion }} use ( . -{{- range .SharedModulesPaths }} - {{ . }} -{{- end }} - .ftl/go/main + _ftl/go/modules + _ftl/go/main ) diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 4b3774294..4a234a356 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -3,6 +3,7 @@ package compile import ( "context" "fmt" + "maps" "os" "path" "path/filepath" @@ -14,7 +15,7 @@ import ( "unicode" sets "github.com/deckarep/golang-set/v2" - "golang.org/x/exp/maps" + gomaps "golang.org/x/exp/maps" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" @@ -34,16 +35,12 @@ import ( "github.com/TBD54566975/ftl/internal/reflect" ) -type MainWorkContext struct { - GoVersion string - SharedModulesPaths []string -} - type ExternalModuleContext struct { + ModuleDir string *schema.Schema GoVersion string FTLVersion string - Module *schema.Module + Main string Replacements []*modfile.Replace } @@ -56,14 +53,13 @@ type goVerb struct { } type mainModuleContext struct { - GoVersion string - FTLVersion string - Name string - SharedModulesPaths []string - Verbs []goVerb - Replacements []*modfile.Replace - SumTypes []goSumType - LocalSumTypes []goSumType + GoVersion string + FTLVersion string + Name string + Verbs []goVerb + Replacements []*modfile.Replace + SumTypes []goSumType + LocalSumTypes []goSumType } type goSumType struct { @@ -83,14 +79,25 @@ type ModifyFilesTransaction interface { End() error } -const buildDirName = ".ftl" +func (b ExternalModuleContext) NonMainModules() []*schema.Module { + modules := make([]*schema.Module, 0, len(b.Modules)) + for _, module := range b.Modules { + if module.Name == b.Main { + continue + } + modules = append(modules, module) + } + return modules +} + +const buildDirName = "_ftl" func buildDir(moduleDir string) string { return filepath.Join(moduleDir, buildDirName) } // Build the given module. -func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { +func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { if err := filesTransaction.Begin(); err != nil { return err } @@ -123,27 +130,19 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc funcs := maps.Clone(scaffoldFuncs) - buildDir := buildDir(moduleDir) - err = os.MkdirAll(buildDir, 0750) - if err != nil { - return fmt.Errorf("failed to create build directory: %w", err) - } - - var sharedModulesPaths []string - for _, mod := range sch.Modules { - if mod.Name == config.Module { - continue - } - sharedModulesPaths = append(sharedModulesPaths, filepath.Join(projectRootDir, buildDirName, "go", "modules", mod.Name)) - } - - if err := internal.ScaffoldZip(mainWorkTemplateFiles(), moduleDir, MainWorkContext{ - GoVersion: goModVersion, - SharedModulesPaths: sharedModulesPaths, - }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { - return fmt.Errorf("failed to scaffold zip: %w", err) + logger.Debugf("Generating external modules") + if err := generateExternalModules(ExternalModuleContext{ + ModuleDir: moduleDir, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Schema: sch, + Main: config.Module, + Replacements: replacements, + }); err != nil { + return fmt.Errorf("failed to generate external modules: %w", err) } + buildDir := buildDir(moduleDir) logger.Debugf("Extracting schema") result, err := ExtractModuleSchema(config.Dir, sch) if err != nil { @@ -187,14 +186,13 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc goVerbs = append(goVerbs, goverb) } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Name: result.Module.Name, - SharedModulesPaths: sharedModulesPaths, - Verbs: goVerbs, - Replacements: replacements, - SumTypes: getSumTypes(result.Module, sch, result.NativeNames), - LocalSumTypes: getLocalSumTypes(result.Module), + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Name: result.Module.Name, + Verbs: goVerbs, + Replacements: replacements, + SumTypes: getSumTypes(result.Module, sch, result.NativeNames), + LocalSumTypes: getLocalSumTypes(result.Module), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -202,7 +200,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc logger.Debugf("Tidying go.mod files") wg, wgctx := errgroup.WithContext(ctx) wg.Go(func() error { - if err := exec.Command(wgctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { + if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err) } return filesTransaction.ModifiedFiles(filepath.Join(moduleDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) @@ -214,6 +212,13 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc } return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) + modulesDir := filepath.Join(buildDir, "go", "modules") + wg.Go(func() error { + if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { + return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err) + } + return filesTransaction.ModifiedFiles(filepath.Join(modulesDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) + }) if err := wg.Wait(); err != nil { return err } @@ -222,16 +227,10 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } -func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConfigs []moduleconfig.ModuleConfig, sch *schema.Schema) error { - logger := log.FromContext(ctx) - logger.Debugf("Generating stubs for modules") - - sharedFtlDir := filepath.Join(projectRoot, buildDirName) - - // Wipe the modules directory to ensure we don't have any stale modules. - err := os.RemoveAll(sharedFtlDir) +func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { + goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) if err != nil { - return fmt.Errorf("failed to remove %s: %w", sharedFtlDir, err) + return fmt.Errorf("failed to propagate replacements for library %q: %w", dir, err) } ftlVersion := "" @@ -239,64 +238,27 @@ func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConf ftlVersion = ftl.Version } - for _, module := range sch.Modules { - var moduleConfig *moduleconfig.ModuleConfig - for _, mc := range moduleConfigs { - mcCopy := mc - if mc.Module == module.Name { - moduleConfig = &mcCopy - break - } - } - - var goModVersion string - var replacements []*modfile.Replace - - // If there's no module config, use the go.mod file for the first config we find. - if moduleConfig == nil { - if len(moduleConfigs) > 0 { - _, goModVersion, err = updateGoModule(filepath.Join(moduleConfigs[0].Dir, "go.mod")) - if err != nil { - return err - } - } else { - // The best we can do here if we don't have a module to read from is to use the current Go version. - goModVersion = runtime.Version()[2:] - } - - replacements = []*modfile.Replace{} - } else { - replacements, goModVersion, err = updateGoModule(filepath.Join(moduleConfig.Dir, "go.mod")) - if err != nil { - return err - } - } - - goVersion := runtime.Version()[2:] - if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 { - return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) - } - - context := ExternalModuleContext{ - Schema: sch, - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Module: module, - Replacements: replacements, - } - - funcs := maps.Clone(scaffoldFuncs) - err = internal.ScaffoldZip(externalModuleTemplateFiles(), projectRoot, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) - if err != nil { - return fmt.Errorf("failed to scaffold zip: %w", err) - } + return generateExternalModules(ExternalModuleContext{ + ModuleDir: dir, + GoVersion: goModFile.Go.Version, + FTLVersion: ftlVersion, + Schema: schema, + Replacements: replacements, + }) +} - modulesDir := filepath.Join(sharedFtlDir, "go", "modules", module.Name) - if err := exec.Command(ctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { - return fmt.Errorf("failed to tidy go.mod: %w", err) - } +func generateExternalModules(context ExternalModuleContext) error { + // Wipe the modules directory to ensure we don't have any stale modules. + err := os.RemoveAll(filepath.Join(buildDir(context.ModuleDir), "go", "modules")) + if err != nil { + return fmt.Errorf("could not remove old external modules: %w", err) } + funcs := maps.Clone(scaffoldFuncs) + err = internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) + if err != nil { + return fmt.Errorf("failed to scaffold external modules: %w", err) + } return nil } @@ -613,7 +575,7 @@ func getLocalSumTypes(module *schema.Module) []goSumType { } } } - out := maps.Values(sumTypes) + out := gomaps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) @@ -656,7 +618,7 @@ func getSumTypes(module *schema.Module, sch *schema.Schema, nativeNames NativeNa Variants: variants, } } - out := maps.Values(sumTypes) + out := gomaps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) diff --git a/go-runtime/compile/devel.go b/go-runtime/compile/devel.go index 345309da6..c6f2f6270 100644 --- a/go-runtime/compile/devel.go +++ b/go-runtime/compile/devel.go @@ -8,14 +8,9 @@ import ( "github.com/TBD54566975/ftl/internal" ) -func mainWorkTemplateFiles() *zip.Reader { - return internal.ZipRelativeToCaller("main-work-template") -} - func externalModuleTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("external-module-template") } - func buildTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("build-template") } diff --git a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl similarity index 80% rename from go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl rename to go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl index be3d76079..2f1d05e47 100644 --- a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl @@ -1,4 +1,4 @@ -module ftl/{{ .Module.Name }} +module ftl go {{ .GoVersion }} @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl similarity index 66% rename from go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl rename to go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl index 657e8f20a..a7c7c1b2d 100644 --- a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl @@ -1,13 +1,13 @@ // Code generated by FTL. DO NOT EDIT. -package {{.Module.Name}} +package {{.Name}} import ( "context" -{{- range $import, $alias := (.Module | imports)}} +{{- range $import, $alias := (.|imports)}} {{if $alias}}{{$alias}} {{end}}"{{$import}}" {{- end}} -{{- $sumTypes := (.Module | sumTypes)}} +{{- $sumTypes := $ | sumTypes}} {{- if $sumTypes}} "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" @@ -16,18 +16,18 @@ import ( var _ = context.Background -{{- range .Module.Decls }} +{{- range .Decls }} {{- if .IsExported}} {{if .Comments}} {{.Comments|comment -}} // {{- end}} {{- if is "Topic" .}} -var {{.Name|title}} = ftl.Topic[{{type $.Module .Event}}]("{{.Name}}") +var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}") {{- else if and (is "Enum" .) .IsValueEnum}} {{- $enumName := .Name}} //ftl:enum -type {{.Name|title}} {{type $.Module .Type}} +type {{.Name|title}} {{type $ .Type}} const ( {{- range .Variants }} {{.Name|title}} {{$enumName}} = {{.Value|value}} @@ -38,14 +38,14 @@ const ( {{$enumInterfaceFuncName := enumInterfaceFunc . -}} type {{.Name|title}} interface { {{$enumInterfaceFuncName}}() } {{- range .Variants }} -{{if (or (basicType $.Module .) (isStandaloneEnumVariant .))}} -type {{.Name|title}} {{type $.Module .Value.Value}} +{{if (or (basicType $ .) (isStandaloneEnumVariant .))}} +type {{.Name|title}} {{type $ .Value.Value}} {{end}} func ({{.Name|title}}) {{$enumInterfaceFuncName}}() {} {{- end}} {{- else if is "TypeAlias" .}} //ftl:typealias -type {{.Name|title}} {{type $.Module .Type}} +type {{.Name|title}} {{type $ .Type}} {{- else if is "Data" .}} type {{.Name|title}} {{- if .TypeParameters}}[ @@ -54,25 +54,25 @@ type {{.Name|title}} {{- end -}} ]{{- end}} struct { {{- range .Fields}} - {{.Name|title}} {{type $.Module .Type}} `json:"{{.Name}}"` + {{.Name|title}} {{type $ .Type}} `json:"{{.Name}}"` {{- end}} } {{- else if is "Verb" .}} //ftl:verb -{{- if and (eq (type $.Module .Request) "ftl.Unit") (eq (type $.Module .Response) "ftl.Unit")}} +{{- if and (eq (type $ .Request) "ftl.Unit") (eq (type $ .Response) "ftl.Unit")}} func {{.Name|title}}(context.Context) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") } -{{- else if eq (type $.Module .Request) "ftl.Unit"}} -func {{.Name|title}}(context.Context) ({{type $.Module .Response}}, error) { +{{- else if eq (type $ .Request) "ftl.Unit"}} +func {{.Name|title}}(context.Context) ({{type $ .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") } -{{- else if eq (type $.Module .Response) "ftl.Unit"}} -func {{.Name|title}}(context.Context, {{type $.Module .Request}}) error { +{{- else if eq (type $ .Response) "ftl.Unit"}} +func {{.Name|title}}(context.Context, {{type $ .Request}}) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") } {{- else}} -func {{.Name|title}}(context.Context, {{type $.Module .Request}}) ({{type $.Module .Response}}, error) { +func {{.Name|title}}(context.Context, {{type $ .Request}}) ({{type $ .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") } {{- end}} diff --git a/go-runtime/compile/external-module-template/go.mod b/go-runtime/compile/external-module-template/go.mod new file mode 100644 index 000000000..8da40da65 --- /dev/null +++ b/go-runtime/compile/external-module-template/go.mod @@ -0,0 +1,3 @@ +module exclude + +go 1.22.2 diff --git a/go-runtime/compile/external-module-template/go.work.tmpl b/go-runtime/compile/external-module-template/go.work.tmpl new file mode 100644 index 000000000..204b39b25 --- /dev/null +++ b/go-runtime/compile/external-module-template/go.work.tmpl @@ -0,0 +1,6 @@ +go {{ .GoVersion }} + +use ( + . + _ftl/go/modules +) diff --git a/go-runtime/compile/main-work-template/go.work.tmpl b/go-runtime/compile/main-work-template/go.work.tmpl deleted file mode 100644 index 1dbed607a..000000000 --- a/go-runtime/compile/main-work-template/go.work.tmpl +++ /dev/null @@ -1,8 +0,0 @@ -go {{ .GoVersion }} - -use ( - . -{{- range .SharedModulesPaths }} - {{ . }} -{{- end }} -) diff --git a/go-runtime/compile/release.go b/go-runtime/compile/release.go index a258dd217..8b909688c 100644 --- a/go-runtime/compile/release.go +++ b/go-runtime/compile/release.go @@ -8,23 +8,12 @@ import ( _ "embed" ) -//go:embed main-work-template.zip -var mainWorkTemplateBytes []byte - //go:embed external-module-template.zip var externalModuleTemplateBytes []byte //go:embed build-template.zip var buildTemplateBytes []byte -func mainWorkTemplateFiles() *zip.Reader { - zr, err := zip.NewReader(bytes.NewReader(mainWorkTemplateBytes), int64(len(mainWorkTemplateBytes))) - if err != nil { - panic(err) - } - return zr -} - func externalModuleTemplateFiles() *zip.Reader { zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplateBytes), int64(len(externalModuleTemplateBytes))) if err != nil { diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go index abbc0d83d..8c974259b 100644 --- a/go-runtime/compile/schema_test.go +++ b/go-runtime/compile/schema_test.go @@ -56,7 +56,7 @@ func TestExtractModuleSchema(t *testing.T) { assert.NoError(t, err) actual := schema.Normalise(r.Module) expected := `module one { - config configValue one.Config + config configValue one.Config secret secretValue String database postgres testDb @@ -184,9 +184,6 @@ func TestExtractModuleSchemaTwo(t *testing.T) { if testing.Short() { t.SkipNow() } - - assert.NoError(t, prebuildTestModule(t, "testdata/two")) - r, err := ExtractModuleSchema("testdata/two", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, r.Errors, nil) @@ -368,9 +365,6 @@ func TestExtractModulePubSub(t *testing.T) { if testing.Short() { t.SkipNow() } - - assert.NoError(t, prebuildTestModule(t, "testdata/pubsub")) - r, err := ExtractModuleSchema("testdata/pubsub", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, nil, r.Errors, "expected no schema errors") diff --git a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl index 7d6b63f1d..372814bd3 100644 --- a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl +++ b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl @@ -1,9 +1,9 @@ module ftl/{{ .Name }} -go {{ .GoVersion }} +go 1.21 require github.com/TBD54566975/ftl latest {{- range $old, $new := .Replace }} replace {{ $old }} => {{ $new }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/integration/harness.go b/integration/harness.go index 6cb0af336..d4c55fc95 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -17,7 +17,6 @@ import ( "connectrpc.com/connect" "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" - "github.com/otiai10/copy" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" @@ -74,23 +73,11 @@ func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Ac assert.True(t, ok) if ftlConfigPath != "" { - ftlConfigPath = filepath.Join(cwd, "testdata", "go", ftlConfigPath) - projectPath := filepath.Join(tmpDir, "ftl-project.toml") - - // Copy the specified FTL config to the temporary directory. - err = copy.Copy(ftlConfigPath, projectPath) - if err == nil { - t.Setenv("FTL_CONFIG", projectPath) - } else { - // Use a path into the testdata directory instead of one relative to - // tmpDir. Otherwise we have a chicken and egg situation where the config - // can't be loaded until the module is copied over, and the config itself - // is used by FTL during startup. - // Some tests still rely on this behavior, so we can't remove it entirely. - t.Logf("Failed to copy %s to %s: %s", ftlConfigPath, projectPath, err) - t.Setenv("FTL_CONFIG", ftlConfigPath) - } - + // Use a path into the testdata directory instead of one relative to + // tmpDir. Otherwise we have a chicken and egg situation where the config + // can't be loaded until the module is copied over, and the config itself + // is used by FTL during startup. + t.Setenv("FTL_CONFIG", filepath.Join(cwd, "testdata", "go", ftlConfigPath)) } else { err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644) assert.NoError(t, err)