From 1a461c60a2ddcc9412df5192592c06444fb693df Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 07:36:44 +1100
Subject: [PATCH 01/10] fix: run go mod tidy on _ftl/go/modules too (#1108)

This isn't necessary to build, but VSCode doesn't like it.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
---
 go-runtime/compile/build.go      | 34 ++++++++++++++++++++++++--------
 internal/model/deployment_key.go |  2 +-
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go
index d59a61be59..42d10aa43a 100644
--- a/go-runtime/compile/build.go
+++ b/go-runtime/compile/build.go
@@ -14,6 +14,7 @@ import (
 	"github.com/TBD54566975/scaffolder"
 	"golang.design/x/reflect"
 	"golang.org/x/mod/modfile"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/TBD54566975/ftl"
@@ -97,11 +98,6 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error {
 		return err
 	}
 
-	logger.Debugf("Tidying go.mod")
-	if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil {
-		return fmt.Errorf("failed to tidy go.mod: %w", err)
-	}
-
 	logger.Debugf("Extracting schema")
 	nativeNames, main, err := ExtractModuleSchema(moduleDir)
 	if err != nil {
@@ -145,12 +141,34 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error {
 		return err
 	}
 
-	logger.Debugf("Compiling")
+	wg, wgctx := errgroup.WithContext(ctx)
+
+	logger.Debugf("Tidying go.mod files")
+	wg.Go(func() error {
+		if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil {
+			return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err)
+		}
+		return nil
+	})
 	mainDir := filepath.Join(buildDir, "go", "main")
-	if err := exec.Command(ctx, log.Debug, mainDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil {
-		return fmt.Errorf("failed to tidy go.mod: %w", err)
+	wg.Go(func() error {
+		if err := exec.Command(wgctx, log.Debug, mainDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil {
+			return fmt.Errorf("%s: failed to tidy go.mod: %w", mainDir, err)
+		}
+		return nil
+	})
+	wg.Go(func() error {
+		modulesDir := filepath.Join(buildDir, "go", "modules")
+		if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil {
+			return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err)
+		}
+		return nil
+	})
+	if err := wg.Wait(); err != nil {
+		return err
 	}
 
+	logger.Debugf("Compiling")
 	return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx)
 }
 
diff --git a/internal/model/deployment_key.go b/internal/model/deployment_key.go
index 370b6fb70b..b06412f1c1 100644
--- a/internal/model/deployment_key.go
+++ b/internal/model/deployment_key.go
@@ -52,7 +52,7 @@ func ParseDeploymentKey(input string) (DeploymentKey, error) {
 	}, nil
 }
 
-func (d *DeploymentKey) String() string {
+func (d DeploymentKey) String() string {
 	return fmt.Sprintf("%s-%s", d.module, d.hash)
 }
 

From ac7953bfccc6bdc0f6524bab264bc07d3b118150 Mon Sep 17 00:00:00 2001
From: worstell <worstell@squareup.com>
Date: Mon, 18 Mar 2024 16:46:26 -0400
Subject: [PATCH 02/10] feat: handle enums in jsonschema (#1107)

fixes #1099
---
 backend/schema/intvalue.go        |  2 +
 backend/schema/jsonschema.go      | 79 +++++++++++++++----------------
 backend/schema/jsonschema_test.go | 79 ++++++++++++++++++++++++++++++-
 backend/schema/parser.go          |  1 +
 backend/schema/schema.go          | 12 -----
 backend/schema/stringvalue.go     |  2 +
 6 files changed, 121 insertions(+), 54 deletions(-)

diff --git a/backend/schema/intvalue.go b/backend/schema/intvalue.go
index 9632360385..5a82ea09a9 100644
--- a/backend/schema/intvalue.go
+++ b/backend/schema/intvalue.go
@@ -31,4 +31,6 @@ func (i *IntValue) String() string {
 	return fmt.Sprintf("%d", i.Value)
 }
 
+func (i *IntValue) GetValue() any { return i.Value }
+
 func (*IntValue) schemaValueType() Type { return &Int{} }
diff --git a/backend/schema/jsonschema.go b/backend/schema/jsonschema.go
index f31ed66b4c..3df4ebedc0 100644
--- a/backend/schema/jsonschema.go
+++ b/backend/schema/jsonschema.go
@@ -19,42 +19,40 @@ func DataToJSONSchema(sch *Schema, ref Ref) (*jsonschema.Schema, error) {
 		return nil, fmt.Errorf("unknown data type %s", ref)
 	}
 
-	// Collect all data types.
-	dataTypes := sch.DataMap()
-
 	// Encode root, and collect all data types reachable from the root.
 	refs := map[RefKey]*Ref{}
-	root := nodeToJSSchema(sch, data, refs)
+	root := nodeToJSSchema(data, refs)
 	if len(refs) == 0 {
 		return root, nil
 	}
 
-	// Resolve and encode all data types reachable from the root.
+	// Resolve and encode all types reachable from the root.
 	root.Definitions = map[string]jsonschema.SchemaOrBool{}
-	for key, r := range refs {
-		data, ok := dataTypes[RefKey{Module: key.Module, Name: key.Name}]
-		if !ok {
-			return nil, fmt.Errorf("unknown data type %s", key)
-		}
-
-		if len(r.TypeParameters) > 0 {
-			monomorphisedData, err := data.Monomorphise(r)
-			if err != nil {
-				return nil, err
+	for _, r := range refs {
+		decl := sch.ResolveRef(r)
+		switch n := decl.(type) {
+		case *Data:
+			if len(r.TypeParameters) > 0 {
+				monomorphisedData, err := n.Monomorphise(r)
+				if err != nil {
+					return nil, err
+				}
+
+				ref := fmt.Sprintf("%s.%s", r.Module, refName(r))
+				root.Definitions[ref] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(monomorphisedData, refs)}
+			} else {
+				root.Definitions[r.String()] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(n, refs)}
 			}
-			data = monomorphisedData
-
-			ref := fmt.Sprintf("%s.%s", r.Module, refName(r))
-			root.Definitions[ref] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(sch, data, refs)}
-		} else {
-			root.Definitions[r.String()] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(sch, data, refs)}
+		case *Enum:
+			root.Definitions[r.String()] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(n, refs)}
+		case *Config, *Database, *Secret, *Verb:
+			return nil, fmt.Errorf("reference to unsupported node type %T", decl)
 		}
-
 	}
 	return root, nil
 }
 
-func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschema.Schema {
+func nodeToJSSchema(node Node, refs map[RefKey]*Ref) *jsonschema.Schema {
 	switch node := node.(type) {
 	case *Any:
 		return &jsonschema.Schema{}
@@ -72,7 +70,7 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 			AdditionalProperties: jsBool(false),
 		}
 		for _, field := range node.Fields {
-			jsField := nodeToJSSchema(sch, field.Type, dataRefs)
+			jsField := nodeToJSSchema(field.Type, refs)
 			jsField.Description = jsComments(field.Comments)
 			if _, ok := field.Type.(*Optional); !ok {
 				schema.Required = append(schema.Required, field.Name)
@@ -81,6 +79,16 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 		}
 		return schema
 
+	case *Enum:
+		schema := &jsonschema.Schema{
+			Description: jsComments(node.Comments),
+		}
+		values := make([]any, len(node.Variants))
+		for i, v := range node.Variants {
+			values[i] = v.Value.GetValue()
+		}
+		return schema.WithEnum(values...)
+
 	case *Int:
 		st := jsonschema.Integer
 		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
@@ -118,7 +126,7 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 			Type: &jsonschema.Type{SimpleTypes: &st},
 			Items: &jsonschema.Items{
 				SchemaOrBool: &jsonschema.SchemaOrBool{
-					TypeObject: nodeToJSSchema(sch, node.Element, dataRefs),
+					TypeObject: nodeToJSSchema(node.Element, refs),
 				},
 			},
 		}
@@ -128,8 +136,8 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 		// JSON schema generic map of key type to value type
 		return &jsonschema.Schema{
 			Type:                 &jsonschema.Type{SimpleTypes: &st},
-			PropertyNames:        &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(sch, node.Key, dataRefs)},
-			AdditionalProperties: &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(sch, node.Value, dataRefs)},
+			PropertyNames:        &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(node.Key, refs)},
+			AdditionalProperties: &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(node.Value, refs)},
 		}
 
 	case *Ref:
@@ -140,22 +148,13 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 		} else {
 			ref = fmt.Sprintf("#/definitions/%s", node.String())
 		}
-
-		decl := sch.ResolveRef(node)
-		if decl != nil {
-			if _, ok := decl.(*Data); ok {
-				dataRefs[node.ToRefKey()] = node
-			}
-		}
-
-		schema := &jsonschema.Schema{Ref: &ref}
-
-		return schema
+		refs[node.ToRefKey()] = node
+		return &jsonschema.Schema{Ref: &ref}
 
 	case *Optional:
 		null := jsonschema.Null
 		return &jsonschema.Schema{AnyOf: []jsonschema.SchemaOrBool{
-			{TypeObject: nodeToJSSchema(sch, node.Type, dataRefs)},
+			{TypeObject: nodeToJSSchema(node.Type, refs)},
 			{TypeObject: &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &null}}},
 		}}
 
@@ -164,7 +163,7 @@ func nodeToJSSchema(sch *Schema, node Node, dataRefs map[RefKey]*Ref) *jsonschem
 
 	case Decl, *Field, Metadata, *MetadataCalls, *MetadataDatabases, *MetadataIngress,
 		*MetadataAlias, IngressPathComponent, *IngressPathLiteral, *IngressPathParameter, *Module,
-		*Schema, Type, *Database, *Verb, *Enum, *EnumVariant,
+		*Schema, Type, *Database, *Verb, *EnumVariant,
 		Value, *StringValue, *IntValue, *Config, *Secret, Symbol:
 		panic(fmt.Sprintf("unsupported node type %T", node))
 
diff --git a/backend/schema/jsonschema_test.go b/backend/schema/jsonschema_test.go
index 95791fe895..ec7fa6a570 100644
--- a/backend/schema/jsonschema_test.go
+++ b/backend/schema/jsonschema_test.go
@@ -30,6 +30,8 @@ var jsonSchemaSample = &Schema{
 					{Name: "ref", Type: &Ref{Module: "bar", Name: "Bar"}},
 					{Name: "any", Type: &Any{}},
 					{Name: "keyValue", Type: &Ref{Module: "foo", Name: "Generic", TypeParameters: []Type{&String{}, &Int{}}}},
+					{Name: "stringEnumRef", Type: &Ref{Module: "foo", Name: "StringEnum"}},
+					{Name: "intEnumRef", Type: &Ref{Module: "foo", Name: "IntEnum"}},
 				},
 			},
 			&Data{
@@ -43,6 +45,20 @@ var jsonSchemaSample = &Schema{
 					{Name: "value", Type: &Ref{Name: "V"}},
 				},
 			},
+			&Enum{
+				Name: "StringEnum",
+				Variants: []*EnumVariant{
+					{Name: "A", Value: &StringValue{Value: "A"}},
+					{Name: "B", Value: &StringValue{Value: "B"}},
+				},
+			},
+			&Enum{
+				Name: "IntEnum",
+				Variants: []*EnumVariant{
+					{Name: "Zero", Value: &IntValue{Value: 0}},
+					{Name: "One", Value: &IntValue{Value: 1}},
+				},
+			},
 		}},
 		{Name: "bar", Decls: []Decl{
 			&Data{Name: "Bar", Fields: []*Field{{Name: "bar", Type: &String{}}}},
@@ -71,7 +87,9 @@ func TestDataToJSONSchema(t *testing.T) {
     "optionalMap",
     "ref",
     "any",
-    "keyValue"
+    "keyValue",
+    "stringEnumRef",
+    "intEnumRef"
   ],
   "additionalProperties": false,
   "definitions": {
@@ -103,6 +121,12 @@ func TestDataToJSONSchema(t *testing.T) {
       },
       "type": "object"
     },
+    "foo.IntEnum": {
+      "enum": [
+        0,
+        1
+      ]
+    },
     "foo.Item": {
       "required": [
         "name"
@@ -114,6 +138,12 @@ func TestDataToJSONSchema(t *testing.T) {
         }
       },
       "type": "object"
+    },
+    "foo.StringEnum": {
+      "enum": [
+        "A",
+        "B"
+      ]
     }
   },
   "properties": {
@@ -148,6 +178,9 @@ func TestDataToJSONSchema(t *testing.T) {
     "int": {
       "type": "integer"
     },
+    "intEnumRef": {
+      "$ref": "#/definitions/foo.IntEnum"
+    },
     "keyValue": {
       "$ref": "#/definitions/foo.Generic[String, Int]"
     },
@@ -206,6 +239,9 @@ func TestDataToJSONSchema(t *testing.T) {
       "description": "Field comment",
       "type": "string"
     },
+    "stringEnumRef": {
+      "$ref": "#/definitions/foo.StringEnum"
+    },
     "time": {
       "type": "string",
       "format": "date-time"
@@ -232,7 +268,9 @@ func TestJSONSchemaValidation(t *testing.T) {
     "optionalMap": {"one": 2, "two": null},
     "ref": {"bar": "Name"},
     "any": [{"name": "Name"}, "string", 1, 1.23, true, "2018-11-13T20:20:39+00:00", ["one"], {"one": 2}, null],
-    "keyValue": {"key": "string", "value": 1}
+    "keyValue": {"key": "string", "value": 1},
+	"stringEnumRef": "A",
+	"intEnumRef": 0
   }
    `
 
@@ -250,3 +288,40 @@ func TestJSONSchemaValidation(t *testing.T) {
 	err = jsonschema.Validate(v)
 	assert.NoError(t, err)
 }
+
+func TestInvalidEnumValidation(t *testing.T) {
+	input := `
+   {
+    "string": "string",
+    "int": 1,
+    "float": 1.23,
+    "bool": true,
+    "time": "2018-11-13T20:20:39+00:00",
+    "array": ["one"],
+    "arrayOfRefs": [{"name": "Name"}],
+    "arrayOfArray": [[]],
+    "optionalArray": [null, "foo"],
+    "map": {"one": 2},
+    "optionalMap": {"one": 2, "two": null},
+    "ref": {"bar": "Name"},
+    "any": [{"name": "Name"}, "string", 1, 1.23, true, "2018-11-13T20:20:39+00:00", ["one"], {"one": 2}, null],
+    "keyValue": {"key": "string", "value": 1},
+	"stringEnumRef": "B",
+	"intEnumRef": 3
+  }
+   `
+
+	schema, err := DataToJSONSchema(jsonSchemaSample, Ref{Module: "foo", Name: "Foo"})
+	assert.NoError(t, err)
+	schemaJSON, err := json.MarshalIndent(schema, "", "  ")
+	assert.NoError(t, err)
+	jsonschema, err := jsonschema.CompileString("http://ftl.block.xyz/schema.json", string(schemaJSON))
+	assert.NoError(t, err)
+
+	var v interface{}
+	err = json.Unmarshal([]byte(input), &v)
+	assert.NoError(t, err)
+
+	err = jsonschema.Validate(v)
+	assert.Contains(t, err.Error(), "value must be one of \"0\", \"1\"")
+}
diff --git a/backend/schema/parser.go b/backend/schema/parser.go
index 5a83c23717..ebbbfec901 100644
--- a/backend/schema/parser.go
+++ b/backend/schema/parser.go
@@ -115,6 +115,7 @@ type Metadata interface {
 //sumtype:decl
 type Value interface {
 	Node
+	GetValue() any
 	schemaValueType() Type
 }
 
diff --git a/backend/schema/schema.go b/backend/schema/schema.go
index f08c630920..57a33d4d90 100644
--- a/backend/schema/schema.go
+++ b/backend/schema/schema.go
@@ -103,18 +103,6 @@ func (s *Schema) Module(name string) optional.Option[*Module] {
 	return optional.None[*Module]()
 }
 
-func (s *Schema) DataMap() map[RefKey]*Data {
-	dataTypes := map[RefKey]*Data{}
-	for _, module := range s.Modules {
-		for _, decl := range module.Decls {
-			if data, ok := decl.(*Data); ok {
-				dataTypes[RefKey{Module: module.Name, Name: data.Name}] = data
-			}
-		}
-	}
-	return dataTypes
-}
-
 // Upsert inserts or replaces a module.
 func (s *Schema) Upsert(module *Module) {
 	for i, m := range s.Modules {
diff --git a/backend/schema/stringvalue.go b/backend/schema/stringvalue.go
index 0092b85f36..ecbfb1f8b3 100644
--- a/backend/schema/stringvalue.go
+++ b/backend/schema/stringvalue.go
@@ -31,4 +31,6 @@ func (s *StringValue) String() string {
 	return fmt.Sprintf("\"%s\"", s.Value)
 }
 
+func (s *StringValue) GetValue() any { return s.Value }
+
 func (*StringValue) schemaValueType() Type { return &String{} }

From cc18bf32bed75ff2dd30ee404b1d0cb6d12330bc Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 15:13:09 +1100
Subject: [PATCH 03/10] chore: replace bit with Justfile (#1109)

This was way more work and way more irritating than I hoped it would be.
I was hoping to switch back to make, but make doesn't support spaces in
filenames. Then I tried ninja, which doesn't support "|" in filenames.
Finally, I tried just using Just, but the lack of a dependency system
meant that really slow targets like the kt-runtime and npm made it
unreasonably painful.

So in the end I wrote a [little
tool](https://github.com/alecthomas/mktg) which acts like a single
"make" target entry, executing the provided command if any input is
newer than all outputs. This gives us fairly fast incremental builds
with little complexity.
---
 .github/workflows/integration.yml             |   2 +
 .github/workflows/rebuild.yml                 |   4 +-
 .github/workflows/release.yml                 |   4 +-
 .github/workflows/writecache.yml              |   4 +-
 Bitfile                                       | 139 ------------------
 Dockerfile.controller                         |   6 +-
 Dockerfile.runner                             |   6 +-
 Justfile                                      |  76 +++++++---
 Procfile                                      |  12 --
 .../controller/scaling/localscaling/devel.go  |   2 +-
 bin/{.bit-0.5.2.pkg => .dbmate-2.13.0.pkg}    |   0
 bin/.mktg-0.3.0.pkg                           |   1 +
 bin/bit                                       |   1 -
 bin/dbmate                                    |   1 +
 bin/mktg                                      |   1 +
 frontend/local.go                             |   2 +-
 go-runtime/release.go                         |   3 -
 integration/integration_test.go               |   2 +-
 kotlin-runtime/release.go                     |   6 -
 scripts/ftl-run                               |   4 +-
 20 files changed, 77 insertions(+), 199 deletions(-)
 delete mode 100644 Bitfile
 delete mode 100644 Procfile
 rename bin/{.bit-0.5.2.pkg => .dbmate-2.13.0.pkg} (100%)
 create mode 120000 bin/.mktg-0.3.0.pkg
 delete mode 120000 bin/bit
 create mode 120000 bin/dbmate
 create mode 120000 bin/mktg

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index b4632ebf33..425ea0d824 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -37,6 +37,8 @@ jobs:
         uses: ./.github/actions/build-cache
       - name: Docker Compose
         run: docker compose up -d --wait
+      - name: Create DB
+        run: just init-db
       - name: Download Go Modules
         run: go mod download
       - name: Run ${{ matrix.test }}
diff --git a/.github/workflows/rebuild.yml b/.github/workflows/rebuild.yml
index f086d78922..49809371c0 100644
--- a/.github/workflows/rebuild.yml
+++ b/.github/workflows/rebuild.yml
@@ -23,7 +23,5 @@ jobs:
         run: docker compose up -d --wait
       - name: Init DB
         run: ftl-initdb
-      - name: Clean All
-        run: bit -c
       - name: Rebuild All
-        run: bit
+        run: just build-all
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index a898cafddf..4d1bc36e59 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -134,11 +134,11 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Build Console
-        run: bit frontend/dist/index.html
+        run: just build-frontend
       - name: Publish Go Binaries
         run: |
           just errtrace
-          bit build/release/ftl
+          just build ftl # Ensure all the prerequisites are built before we use goreleaser
           goreleaser release --skip=validate
         env:
           GITHUB_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/writecache.yml b/.github/workflows/writecache.yml
index 6c3b881e79..a04c10dea0 100644
--- a/.github/workflows/writecache.yml
+++ b/.github/workflows/writecache.yml
@@ -19,10 +19,8 @@ jobs:
         run: docker compose up -d --wait
       - name: Init DB
         run: ftl-initdb
-      - name: Clean All
-        run: bit -c
       - name: Rebuild All
-        run: bit
+        run: bit build-all
       - name: Download Maven Dependencies
         run: mvn -f kotlin-runtime/ftl-runtime dependency:resolve --batch-mode
       - name: Download Go Dependencies
diff --git a/Bitfile b/Bitfile
deleted file mode 100644
index c17c661ad0..0000000000
--- a/Bitfile
+++ /dev/null
@@ -1,139 +0,0 @@
-VERSION = %(git describe --tags --always --dirty | sed -e 's/^v//')%
-DEST = build
-RELEASE = %{DEST}/release
-
-GO_SOURCES = **/*.go
-
-SCHEMA_IN = backend/schema/**/*.go cmd/ftl-schema/**/*.go
-SCHEMA_OUT = backend/protos/xyz/block/ftl/v1/schema/schema.proto
-
-NODE_PROTOC_PLUGINS = frontend/node_modules/.bin/protoc-gen-connect-es \
-  frontend/node_modules/.bin/protoc-gen-es
-
-PROTO_IN = **/*.proto **/buf.* bin/protoc* bin/buf %{NODE_PROTOC_PLUGINS}
-# There's no real way to mechanically generate the list of outputs, so we just
-# explicitly list them here.
-PROTO_OUT = backend/protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go \
-    backend/protos/xyz/block/ftl/v1/schema/schema.pb.go \
-    backend/protos/xyz/block/ftl/v1/console/console.pb.go \
-    backend/protos/xyz/block/ftl/v1/schema/runtime.pb.go \
-    backend/protos/xyz/block/ftl/v1/ftl.pb.go \
-    frontend/src/protos/xyz/block/ftl/v1/ftl_connect.ts \
-    frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts \
-    frontend/src/protos/xyz/block/ftl/v1/schema/runtime_pb.ts \
-    frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts \
-    frontend/src/protos/xyz/block/ftl/v1/console/console_pb.ts
-
-COMMON_LOG_IN = internal/log/api.go
-COMMON_LOG_OUT = internal/log/log_level_string.go
-
-KT_RUNTIME_IN = kotlin-runtime/ftl-runtime/**/*.{kt,kts} kotlin-runtime/ftl-runtime/**/pom.xml
-KT_RUNTIME_OUT = kotlin-runtime/ftl-runtime/target/ftl-runtime-1.0-SNAPSHOT.jar
-KT_RUNTIME_RUNNER_TEMPLATE_OUT = build/template/ftl/jars/ftl-runtime.jar
-
-CLIENT_IN = frontend/src/**/*
-CLIENT_OUT = frontend/dist/index.html
-NPM_IN = frontend/package{,-lock}.json
-NPM_OUT = frontend/node_modules %{NODE_PROTOC_PLUGINS}
-
-SQLC_IN = sqlc.yaml \
-  backend/controller/sql/schema/*.sql \
-  backend/controller/sql/queries.sql
-SQLC_OUT = backend/controller/sql/db.go \
-  %(grep -q copyfrom backend/controller/sql/queries.sql && echo backend/controller/sql/copyfrom.go || true)% \
-  backend/controller/sql/models.go \
-  backend/controller/sql/queries.sql.go
-
-RUNNER_TEMPLATE_ZIP = backend/controller/scaling/localscaling/template.zip
-
-#virtual release:
-#  inputs: %{RELEASE}/ftl %{RELEASE}/ftl-controller %{RELEASE}/ftl-runner
-
-%{SQLC_OUT}: %{SQLC_IN}
-  build:
-    sqlc generate --experimental
-    # sqlc 1.18.0 generates a file with a missing import
-    gosimports -w backend/controller/sql/querier.go
-
-%{RELEASE}:
-  build: install -m 0700 -d %{OUT}
-
-# Build all binaries
-# implicit %{RELEASE}/%{1}: cmd/*
-#   inputs: %{RELEASE} %{GO_SOURCES} %{CLIENT_OUT}
-#   build: go build -o %{OUT} -tags release -ldflags "-X github.com/TBD54566975/ftl.Version=%{VERSION} -X github.com/TBD54566975/ftl.Timestamp=$(date +%s)" ./cmd/%{1}
-
-%{RELEASE}/ftl-controller: %{RELEASE} %{GO_SOURCES} %{CLIENT_OUT} cmd/ftl-controller/**/*.go
-  build: go build -o %{OUT} -tags release -ldflags "-X github.com/TBD54566975/ftl.Version=%{VERSION} -X github.com/TBD54566975/ftl.Timestamp=$(date +%s)" ./cmd/ftl-controller
-
-%{RELEASE}/ftl-runner: %{RELEASE} %{GO_SOURCES} cmd/ftl-runner/**/*.go
-  build: go build -o %{OUT} -tags release -ldflags "-X github.com/TBD54566975/ftl.Version=%{VERSION} -X github.com/TBD54566975/ftl.Timestamp=$(date +%s)" ./cmd/ftl-runner
-
-%{RELEASE}/ftl: %{RELEASE} %{GO_SOURCES} cmd/ftl/**/*.go %{CLIENT_OUT} **/*.zip
-  build: go build -o %{OUT} -tags release -ldflags "-X github.com/TBD54566975/ftl.Version=%{VERSION} -X github.com/TBD54566975/ftl.Timestamp=$(date +%s)" ./cmd/ftl
-
-%{RELEASE}/ftl-initdb: %{RELEASE} %{GO_SOURCES} cmd/ftl-initdb/**/*.go
-  build: go build -o %{OUT} -tags release -ldflags "-X github.com/TBD54566975/ftl.Version=%{VERSION} -X github.com/TBD54566975/ftl.Timestamp=$(date +%s)" ./cmd/ftl-initdb
-
-# Release builds include zipped scaffolding becaused raw go:embed doesn't
-# preserve permissions or symlinks. Irritating.
-go-runtime/scaffolding.zip: go-runtime/scaffolding/**/*
-  cd go-runtime/scaffolding
-  build: zip -q --symlinks -r ../scaffolding.zip .
-
-go-runtime/compile/build-template.zip: go-runtime/compile/build-template/**/*
-  cd go-runtime/compile/build-template
-  build: zip -q --symlinks -r ../build-template.zip .
-
-go-runtime/compile/external-module-template.zip: go-runtime/compile/external-module-template/**/*
-  cd go-runtime/compile/external-module-template
-  build: zip -q --symlinks -r ../external-module-template.zip .
-
-kotlin-runtime/scaffolding.zip: kotlin-runtime/scaffolding/**/*
-  cd kotlin-runtime/scaffolding
-  build: zip -q --symlinks -r ../scaffolding.zip .
-
-kotlin-runtime/external-module-template.zip: kotlin-runtime/external-module-template/**/*
-  cd kotlin-runtime/external-module-template
-  build: zip -q --symlinks -r ../external-module-template.zip .
-
-%{SCHEMA_OUT}: %{SCHEMA_IN}
-  build:
-    ftl-schema > %{OUT}
-    buf format -w %{OUT}
-
-%{PROTO_OUT}: %{PROTO_IN}
-  build:
-    buf format -w
-    buf lint
-    (cd backend/protos && buf generate)
-  # There's a build cycle dependency here, so we can't clean: ftl-schema depends on generated .pb.go files
-  -clean
-
-%{KT_RUNTIME_OUT}: %{KT_RUNTIME_IN} %{PROTO_IN}
-  # TODO: Figure out how to make Maven build completely offline. Bizarrely "-o" does not do this.
-  build:
-    mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
-  +clean: mvn -f kotlin-runtime/ftl-runtime -B clean
-
-%(dirname %{KT_RUNTIME_RUNNER_TEMPLATE_OUT})%:
-  build: install -m 0700 -d %{OUT}
-
-%{KT_RUNTIME_RUNNER_TEMPLATE_OUT}: %{KT_RUNTIME_OUT} %(dirname %{KT_RUNTIME_RUNNER_TEMPLATE_OUT})%
-  build: install -m 0600 %{KT_RUNTIME_OUT} %{OUT}
-
-%{RUNNER_TEMPLATE_ZIP}: %{KT_RUNTIME_RUNNER_TEMPLATE_OUT}
-  build: cd build/template && zip -q --symlinks -r ../../%{OUT} .
-
-%{COMMON_LOG_OUT}: %{COMMON_LOG_IN}
-  build: go generate %{IN}
-
-%{NPM_OUT}: %{NPM_IN}
-  cd frontend
-  build: npm install
-  -clean # Don't clean node_modules
-
-%{CLIENT_OUT}: %{CLIENT_IN} %{NPM_OUT}
-  cd frontend
-  build: npm run build
-  clean: rm -rf dist .parcel-cache
\ No newline at end of file
diff --git a/Dockerfile.controller b/Dockerfile.controller
index 7510ec7f03..73851f049c 100644
--- a/Dockerfile.controller
+++ b/Dockerfile.controller
@@ -20,9 +20,9 @@ RUN go mod download -x
 # Build
 COPY . /src/
 RUN just errtrace
-RUN bit build/release/ftl-controller
-RUN bit build/release/ftl-initdb
-RUN bit build/release/ftl
+RUN just build ftl-controller
+RUN just build ftl-initdb
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04
diff --git a/Dockerfile.runner b/Dockerfile.runner
index 603bfabd5f..4220107d1a 100644
--- a/Dockerfile.runner
+++ b/Dockerfile.runner
@@ -23,12 +23,12 @@ RUN go mod download -x
 COPY . /src/
 
 # Build runner template
-RUN bit build/template/ftl/jars/ftl-runtime.jar
+RUN just build-kt-runtime
 
 # Build runner
 RUN just errtrace
-RUN bit build/release/ftl-runner
-RUN bit build/release/ftl
+RUN just build ftl-runner
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04
diff --git a/Justfile b/Justfile
index 2b8e07f06a..d94cfad2d8 100644
--- a/Justfile
+++ b/Justfile
@@ -1,25 +1,65 @@
 set positional-arguments
 
-# Start a hot-reloading dev cluster
-dev: install-jars
-  goreman -logtime=false start
+RELEASE := "build/release"
+VERSION := `git describe --tags --always --dirty | sed -e 's/^v//'`
+KT_RUNTIME_OUT := "kotlin-runtime/ftl-runtime/target/ftl-runtime-1.0-SNAPSHOT.jar"
+KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar"
+RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip"
+TIMESTAMP := `date +%s`
+SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto"
+ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template"
+FRONTEND_OUT := "frontend/dist/index.html"
 
-# Install all JARs to local Maven repository and local build directory
-install-jars:
-  bit 'kotlin-runtime/**/*.jar'
+_help:
+  @just -l
 
-# Deploy the Go time module
-deploy-time:
-  ftl deploy examples/time
+# Run errtrace on Go files to add stacks
+errtrace:
+  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
 
-# Deploy the Kotlin echo module
-deploy-echo-kotlin:
-  ftl deploy examples/echo-kotlin
+# Build everything
+build-all: build-frontend build-generate build-kt-runtime build-protos build-sqlc build-zips
+  just build ftl
 
-regen-schema:
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.proto
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.pb.go
+# Run "go generate" on all packages
+build-generate:
+  go generate -x ./...
 
-# Run errtrace on Go files to add stacks
-errtrace:
-  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
\ No newline at end of file
+# Build command-line tools
+build +tools: build-protos build-sqlc build-zips build-frontend
+  for tool in $@; do go build -o "{{RELEASE}}/$tool" -tags release -ldflags "-X github.com/TBD54566975/ftl.Version={{VERSION}} -X github.com/TBD54566975/ftl.Timestamp={{TIMESTAMP}}" "./cmd/$tool"; done
+
+export DATABASE_URL := "postgres://postgres:secret@localhost:54320/ftl?sslmode=disable"
+
+# Explicitly initialise the database
+init-db:
+  dbmate drop || true
+  dbmate create
+  dbmate --migrations-dir backend/controller/sql/schema up
+
+# Regenerate SQLC code
+build-sqlc:
+  sqlc generate --experimental
+
+# Build the ZIP files that are embedded in the FTL release binaries
+build-zips: build-kt-runtime
+  for dir in {{ZIP_DIRS}}; do (cd $dir && rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip .); done
+
+# Rebuild frontend
+build-frontend: npm-install
+  mktg {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
+
+# Build the Kotlin runtime (if necessary)
+build-kt-runtime:
+  mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}})
+  mktg {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
+  install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}
+  mktg {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
+
+# Install Node dependencies
+npm-install:
+  mktg frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
+
+# Regenerate protos
+build-protos: npm-install
+  mktg {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
diff --git a/Procfile b/Procfile
deleted file mode 100644
index 8c09d30839..0000000000
--- a/Procfile
+++ /dev/null
@@ -1,12 +0,0 @@
-controller: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-controller --key C01H5BRT09Y07547SETZ4HWRA09 --bind http://localhost:8892
-regenerate: watchexec -e yaml -e sql -e proto -i "frontend" -i "examples/**" --debounce 1s -- bit backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go backend/controller/sql/db.go
-runner0: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTS6ABP1EHGZSAGJMBV50A --language go --language kotlin --bind http://localhost:8894 --template-dir build/template
-runner1: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTSGKQ8AZ9S22N9N1SM9HV --language go --language kotlin --bind http://localhost:8895 --template-dir build/template
-runner2: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD0H13WX636B70WV7D216G --language go --language kotlin --bind http://localhost:8896 --template-dir build/template
-runner3: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD0V6EFZWKV2G2XBBHEDP9 --language go --language kotlin --bind http://localhost:8897 --template-dir build/template
-runner4: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD0Z5GRT3QP1MBARC4TW08 --language go --language kotlin --bind http://localhost:8898 --template-dir build/template
-runner5: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD12R0RZZS8AGYBGVP5KQ8 --language go --language kotlin --bind http://localhost:8899 --template-dir build/template
-runner6: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD15Y9SPGDQMXNJ0CF3C5M --language go --language kotlin --bind http://localhost:8900 --template-dir build/template
-runner7: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD18Z6CTY301G8GE8N52CP --language go --language kotlin --bind http://localhost:8901 --template-dir build/template
-runner8: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD1D1W3KJG7NY63DYCCJMY --language go --language kotlin --bind http://localhost:8902 --template-dir build/template
-runner9: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8J2BMH8VTNKAT8MB66Q2SHG --language go --language kotlin --bind http://localhost:8903 --template-dir build/template
diff --git a/backend/controller/scaling/localscaling/devel.go b/backend/controller/scaling/localscaling/devel.go
index 5e197cdee1..8a0f4d5ace 100644
--- a/backend/controller/scaling/localscaling/devel.go
+++ b/backend/controller/scaling/localscaling/devel.go
@@ -17,7 +17,7 @@ var templateDirOnce sync.Once
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
 		// TODO: Figure out how to make maven build offline
-		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
+		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "just", "build-kt-runtime").RunBuffered(ctx)
 		if err != nil {
 			panic(err)
 		}
diff --git a/bin/.bit-0.5.2.pkg b/bin/.dbmate-2.13.0.pkg
similarity index 100%
rename from bin/.bit-0.5.2.pkg
rename to bin/.dbmate-2.13.0.pkg
diff --git a/bin/.mktg-0.3.0.pkg b/bin/.mktg-0.3.0.pkg
new file mode 120000
index 0000000000..383f4511d4
--- /dev/null
+++ b/bin/.mktg-0.3.0.pkg
@@ -0,0 +1 @@
+hermit
\ No newline at end of file
diff --git a/bin/bit b/bin/bit
deleted file mode 120000
index e4de0ee468..0000000000
--- a/bin/bit
+++ /dev/null
@@ -1 +0,0 @@
-.bit-0.5.2.pkg
\ No newline at end of file
diff --git a/bin/dbmate b/bin/dbmate
new file mode 120000
index 0000000000..0f5b1ef9b0
--- /dev/null
+++ b/bin/dbmate
@@ -0,0 +1 @@
+.dbmate-2.13.0.pkg
\ No newline at end of file
diff --git a/bin/mktg b/bin/mktg
new file mode 120000
index 0000000000..30b31f6b0f
--- /dev/null
+++ b/bin/mktg
@@ -0,0 +1 @@
+.mktg-0.3.0.pkg
\ No newline at end of file
diff --git a/frontend/local.go b/frontend/local.go
index 8fff07a9d4..52dc8c673a 100644
--- a/frontend/local.go
+++ b/frontend/local.go
@@ -22,7 +22,7 @@ func Server(ctx context.Context, timestamp time.Time, publicURL *url.URL, allowO
 	logger := log.FromContext(ctx)
 	logger.Debugf("Building console...")
 
-	err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "frontend/**/*").RunBuffered(ctx)
+	err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "just", "build-frontend").RunBuffered(ctx)
 	if err != nil {
 		return nil, err
 	}
diff --git a/go-runtime/release.go b/go-runtime/release.go
index c27d878ee1..4857c28c5d 100644
--- a/go-runtime/release.go
+++ b/go-runtime/release.go
@@ -12,9 +12,6 @@ import (
 var archive []byte
 
 // Files is the FTL Go runtime scaffolding files.
-//
-// scaffolding.zip can be generated by running `bit go-runtime/scaffolding.zip`
-// or indirectly via `bit build/release/ftl`.
 func Files() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
 	if err != nil {
diff --git a/integration/integration_test.go b/integration/integration_test.go
index 248f03f400..2c431eb4ae 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -272,7 +272,7 @@ func runTests(t *testing.T, tmpDir string, tests []test) {
 	ctx := log.ContextWithLogger(context.Background(), logger)
 	logger.Debugf("Building ftl")
 	binDir := filepath.Join(rootDir, "build", "release")
-	err = exec.Command(ctx, log.Debug, rootDir, filepath.Join(rootDir, "bin", "bit"), "build/release/ftl", "**/*.jar").RunBuffered(ctx)
+	err = exec.Command(ctx, log.Debug, rootDir, "just", "build", "ftl").RunBuffered(ctx)
 	assert.NoError(t, err)
 
 	controller := rpc.Dial(ftlv1connect.NewControllerServiceClient, "http://localhost:8892", log.Debug)
diff --git a/kotlin-runtime/release.go b/kotlin-runtime/release.go
index 7b98612768..1f79325792 100644
--- a/kotlin-runtime/release.go
+++ b/kotlin-runtime/release.go
@@ -15,9 +15,6 @@ var archive []byte
 var externalModuleTemplate []byte
 
 // Files is the FTL Kotlin runtime scaffolding files.
-//
-// scaffolding.zip can be generated by running `bit kotlin-runtime/scaffolding.zip`
-// or indirectly via `bit build/release/ftl`.
 func Files() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
 	if err != nil {
@@ -27,9 +24,6 @@ func Files() *zip.Reader {
 }
 
 // ExternalModuleTemplates are templates for scaffolding external modules in the FTL Kotlin runtime.
-//
-// external-module-template.zip can be generated by running `bit kotlin-runtime/external-module-template.zip`
-// or indirectly via `bit build/release/ftl`.
 func ExternalModuleTemplates() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplate), int64(len(externalModuleTemplate)))
 	if err != nil {
diff --git a/scripts/ftl-run b/scripts/ftl-run
index aadd93db1e..c1e1cb4d07 100755
--- a/scripts/ftl-run
+++ b/scripts/ftl-run
@@ -6,9 +6,7 @@ cd "${top}"
 
 
 build() {
-  bit build/template/ftl/jars/ftl-runtime.jar \
-    build/release/ftl-controller \
-    build/release/ftl-runner 
+  just build ftl-controller ftl-runner
 }
 
 build

From f57b0448e6dde4a953d3008048d5ccc69ffaebb6 Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 16:10:08 +1100
Subject: [PATCH 04/10] chore: some missing CI fixes

---
 .github/workflows/ci.yml         | 2 +-
 .github/workflows/rebuild.yml    | 2 +-
 .github/workflows/writecache.yml | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e834e33229..b277a25957 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,7 +47,7 @@ jobs:
       - name: Docker Compose
         run: docker compose up -d --wait
       - name: Initialise database
-        run: ftl-initdb --recreate
+        run: just init-db
       - name: Vet SQL
         run: sqlc vet
   lint:
diff --git a/.github/workflows/rebuild.yml b/.github/workflows/rebuild.yml
index 49809371c0..a300bcb232 100644
--- a/.github/workflows/rebuild.yml
+++ b/.github/workflows/rebuild.yml
@@ -22,6 +22,6 @@ jobs:
         # New version of "sqlc generate" needs the DB up for some unknown reason
         run: docker compose up -d --wait
       - name: Init DB
-        run: ftl-initdb
+        run: just init-db
       - name: Rebuild All
         run: just build-all
diff --git a/.github/workflows/writecache.yml b/.github/workflows/writecache.yml
index a04c10dea0..4f67f8be56 100644
--- a/.github/workflows/writecache.yml
+++ b/.github/workflows/writecache.yml
@@ -18,9 +18,9 @@ jobs:
       - name: Docker Compose
         run: docker compose up -d --wait
       - name: Init DB
-        run: ftl-initdb
+        run: just init-db
       - name: Rebuild All
-        run: bit build-all
+        run: just build-all
       - name: Download Maven Dependencies
         run: mvn -f kotlin-runtime/ftl-runtime dependency:resolve --batch-mode
       - name: Download Go Dependencies

From fbef33f1b0846e88550bc8903c167c6704d79100 Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 17:13:55 +1100
Subject: [PATCH 05/10] fix: don't blindly run sqlc during builds as it
 requires PG

---
 Justfile                               | 15 ++++++++-------
 bin/{.mktg-0.3.0.pkg => .mk-0.5.0.pkg} |  0
 bin/mk                                 |  1 +
 bin/mktg                               |  1 -
 4 files changed, 9 insertions(+), 8 deletions(-)
 rename bin/{.mktg-0.3.0.pkg => .mk-0.5.0.pkg} (100%)
 create mode 120000 bin/mk
 delete mode 120000 bin/mktg

diff --git a/Justfile b/Justfile
index d94cfad2d8..2f856837d4 100644
--- a/Justfile
+++ b/Justfile
@@ -23,7 +23,8 @@ build-all: build-frontend build-generate build-kt-runtime build-protos build-sql
 
 # Run "go generate" on all packages
 build-generate:
-  go generate -x ./...
+  mk backend/schema/aliaskind_enumer.go : backend/schema/metadataalias.go -- go generate -x backend/schema
+  mk internal/log/log_level_string.go : internal/log/api.go -- go generate -x internal/log
 
 # Build command-line tools
 build +tools: build-protos build-sqlc build-zips build-frontend
@@ -39,7 +40,7 @@ init-db:
 
 # Regenerate SQLC code
 build-sqlc:
-  sqlc generate --experimental
+  mk backend/controller/sql/{db.go,models.go,querier.go,queries.sql.go} : backend/controller/sql/queries.sql backend/controller/sql/schema -- sqlc generate --experimental
 
 # Build the ZIP files that are embedded in the FTL release binaries
 build-zips: build-kt-runtime
@@ -47,19 +48,19 @@ build-zips: build-kt-runtime
 
 # Rebuild frontend
 build-frontend: npm-install
-  mktg {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
+  mk {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
 
 # Build the Kotlin runtime (if necessary)
 build-kt-runtime:
   mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}})
-  mktg {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
+  mk {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
   install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}
-  mktg {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
+  mk {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
 
 # Install Node dependencies
 npm-install:
-  mktg frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
+  mk frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
 
 # Regenerate protos
 build-protos: npm-install
-  mktg {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
+  mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
diff --git a/bin/.mktg-0.3.0.pkg b/bin/.mk-0.5.0.pkg
similarity index 100%
rename from bin/.mktg-0.3.0.pkg
rename to bin/.mk-0.5.0.pkg
diff --git a/bin/mk b/bin/mk
new file mode 120000
index 0000000000..b7eb922c34
--- /dev/null
+++ b/bin/mk
@@ -0,0 +1 @@
+.mk-0.5.0.pkg
\ No newline at end of file
diff --git a/bin/mktg b/bin/mktg
deleted file mode 120000
index 30b31f6b0f..0000000000
--- a/bin/mktg
+++ /dev/null
@@ -1 +0,0 @@
-.mktg-0.3.0.pkg
\ No newline at end of file

From 6dc4e4f226f1c890e18b6b65d6577b3aeac868c2 Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 17:48:36 +1100
Subject: [PATCH 06/10] chore: bump "mk" to fix some bugs

---
 bin/{.mk-0.5.0.pkg => .mk-0.6.0.pkg} | 0
 bin/mk                               | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename bin/{.mk-0.5.0.pkg => .mk-0.6.0.pkg} (100%)

diff --git a/bin/.mk-0.5.0.pkg b/bin/.mk-0.6.0.pkg
similarity index 100%
rename from bin/.mk-0.5.0.pkg
rename to bin/.mk-0.6.0.pkg
diff --git a/bin/mk b/bin/mk
index b7eb922c34..0201887dcd 120000
--- a/bin/mk
+++ b/bin/mk
@@ -1 +1 @@
-.mk-0.5.0.pkg
\ No newline at end of file
+.mk-0.6.0.pkg
\ No newline at end of file

From 5bc9f44f9f28932abeb648a5ae2e669d666dd9e5 Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Tue, 19 Mar 2024 19:54:51 +1100
Subject: [PATCH 07/10] chore: be silent when checking build dependencies

---
 Justfile                             | 27 ++++++++++++++-------------
 bin/{.mk-0.6.0.pkg => .mk-0.6.1.pkg} |  0
 bin/mk                               |  2 +-
 3 files changed, 15 insertions(+), 14 deletions(-)
 rename bin/{.mk-0.6.0.pkg => .mk-0.6.1.pkg} (100%)

diff --git a/Justfile b/Justfile
index 2f856837d4..8d7ea301fc 100644
--- a/Justfile
+++ b/Justfile
@@ -19,16 +19,18 @@ errtrace:
 
 # Build everything
 build-all: build-frontend build-generate build-kt-runtime build-protos build-sqlc build-zips
-  just build ftl
+  @just build ftl ftl-controller ftl-runner ftl-initdb
 
 # Run "go generate" on all packages
 build-generate:
-  mk backend/schema/aliaskind_enumer.go : backend/schema/metadataalias.go -- go generate -x backend/schema
-  mk internal/log/log_level_string.go : internal/log/api.go -- go generate -x internal/log
+  @mk backend/schema/aliaskind_enumer.go : backend/schema/metadataalias.go -- go generate -x backend/schema
+  @mk internal/log/log_level_string.go : internal/log/api.go -- go generate -x internal/log
 
 # Build command-line tools
 build +tools: build-protos build-sqlc build-zips build-frontend
-  for tool in $@; do go build -o "{{RELEASE}}/$tool" -tags release -ldflags "-X github.com/TBD54566975/ftl.Version={{VERSION}} -X github.com/TBD54566975/ftl.Timestamp={{TIMESTAMP}}" "./cmd/$tool"; done
+  #!/bin/bash
+  shopt -s extglob
+  for tool in $@; do mk "{{RELEASE}}/$tool" : !(build) -- go build -o "{{RELEASE}}/$tool" -tags release -ldflags "-X github.com/TBD54566975/ftl.Version={{VERSION}} -X github.com/TBD54566975/ftl.Timestamp={{TIMESTAMP}}" "./cmd/$tool"; done
 
 export DATABASE_URL := "postgres://postgres:secret@localhost:54320/ftl?sslmode=disable"
 
@@ -40,27 +42,26 @@ init-db:
 
 # Regenerate SQLC code
 build-sqlc:
-  mk backend/controller/sql/{db.go,models.go,querier.go,queries.sql.go} : backend/controller/sql/queries.sql backend/controller/sql/schema -- sqlc generate --experimental
+  @mk backend/controller/sql/{db.go,models.go,querier.go,queries.sql.go} : backend/controller/sql/queries.sql backend/controller/sql/schema -- sqlc generate --experimental
 
 # Build the ZIP files that are embedded in the FTL release binaries
 build-zips: build-kt-runtime
-  for dir in {{ZIP_DIRS}}; do (cd $dir && rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip .); done
+  @for dir in {{ZIP_DIRS}}; do (cd $dir && mk ../$(basename ${dir}).zip : . -- "rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip ."); done
 
 # Rebuild frontend
 build-frontend: npm-install
-  mk {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
+  @mk {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
 
 # Build the Kotlin runtime (if necessary)
 build-kt-runtime:
-  mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}})
-  mk {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
-  install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}
-  mk {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
+  @mk {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
+  @mk {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} : {{KT_RUNTIME_OUT}} -- "mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}) && install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}"
+  @mk {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
 
 # Install Node dependencies
 npm-install:
-  mk frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
+  @mk frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
 
 # Regenerate protos
 build-protos: npm-install
-  mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
+  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
diff --git a/bin/.mk-0.6.0.pkg b/bin/.mk-0.6.1.pkg
similarity index 100%
rename from bin/.mk-0.6.0.pkg
rename to bin/.mk-0.6.1.pkg
diff --git a/bin/mk b/bin/mk
index 0201887dcd..07886b2300 120000
--- a/bin/mk
+++ b/bin/mk
@@ -1 +1 @@
-.mk-0.6.0.pkg
\ No newline at end of file
+.mk-0.6.1.pkg
\ No newline at end of file

From aaf27d5f4ca4b55612b18519ca5a64c113b0889c Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Wed, 20 Mar 2024 00:58:06 +1100
Subject: [PATCH 08/10] chore: bump mk to fix a bug

---
 bin/{.mk-0.6.1.pkg => .mk-0.6.2.pkg} | 0
 bin/mk                               | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename bin/{.mk-0.6.1.pkg => .mk-0.6.2.pkg} (100%)

diff --git a/bin/.mk-0.6.1.pkg b/bin/.mk-0.6.2.pkg
similarity index 100%
rename from bin/.mk-0.6.1.pkg
rename to bin/.mk-0.6.2.pkg
diff --git a/bin/mk b/bin/mk
index 07886b2300..87009134b8 120000
--- a/bin/mk
+++ b/bin/mk
@@ -1 +1 @@
-.mk-0.6.1.pkg
\ No newline at end of file
+.mk-0.6.2.pkg
\ No newline at end of file

From 9245419dad8aa1b8b7b96b76145a43a06ebad95b Mon Sep 17 00:00:00 2001
From: Alec Thomas <aat@tbd.email>
Date: Wed, 20 Mar 2024 01:04:56 +1100
Subject: [PATCH 09/10] chore: add "just clean"

---
 Justfile | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Justfile b/Justfile
index 8d7ea301fc..174467fd0f 100644
--- a/Justfile
+++ b/Justfile
@@ -17,6 +17,13 @@ _help:
 errtrace:
   git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
 
+# Clean the build directory
+clean:
+  rm -rf build
+  rm -rf frontend/node_modules
+  find . -name '*.zip' -exec rm {} \;
+  mvn -f kotlin-runtime/ftl-runtime clean
+
 # Build everything
 build-all: build-frontend build-generate build-kt-runtime build-protos build-sqlc build-zips
   @just build ftl ftl-controller ftl-runner ftl-initdb
@@ -64,4 +71,4 @@ npm-install:
 
 # Regenerate protos
 build-protos: npm-install
-  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
\ No newline at end of file
+  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"

From a07113e1c5e827c0c0fb88ebe4f27bc14f19f82a Mon Sep 17 00:00:00 2001
From: worstell <worstell@squareup.com>
Date: Tue, 19 Mar 2024 15:00:06 -0400
Subject: [PATCH 10/10] fix: generated external modules don't import from
 metadata refs (#1110)

when we generate an external module, we stub the verb bodies. we
shouldn't add imports which are only dependencies of the verb expression
body because this import may be unused once the body is stubbed, causing
build errors.
---
 backend/schema/visit.go      | 16 ++++++++++++
 buildengine/build_go_test.go | 50 ++++++++++++++++++++++++++++++++++++
 buildengine/build_kotlin.go  |  2 +-
 go-runtime/compile/build.go  |  2 +-
 4 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/backend/schema/visit.go b/backend/schema/visit.go
index 2f65d50706..c3430329ca 100644
--- a/backend/schema/visit.go
+++ b/backend/schema/visit.go
@@ -11,3 +11,19 @@ func Visit(n Node, visit func(n Node, next func() error) error) error {
 		return nil
 	})
 }
+
+// VisitExcludingMetadataChildren visits all nodes in the schema except the children of metadata nodes.
+// This is used when generating external modules to avoid adding imports only referenced in the bodies of
+// stubbed verbs.
+func VisitExcludingMetadataChildren(n Node, visit func(n Node, next func() error) error) error {
+	return visit(n, func() error {
+		if _, ok := n.(Metadata); !ok {
+			for _, child := range n.schemaChildren() {
+				if err := VisitExcludingMetadataChildren(child, visit); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+}
diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go
index 1ee5237b50..c0ebf26e26 100644
--- a/buildengine/build_go_test.go
+++ b/buildengine/build_go_test.go
@@ -126,3 +126,53 @@ func Nothing(context.Context) error {
 		assertGeneratedModule("go/modules/other/external_module.go", expected),
 	})
 }
+
+func TestMetadataImportsExcluded(t *testing.T) {
+	sch := &schema.Schema{
+		Modules: []*schema.Module{
+			schema.Builtins(),
+			{Name: "test", Decls: []schema.Decl{
+				&schema.Data{Name: "Req"},
+				&schema.Data{Name: "Resp"},
+				&schema.Verb{
+					Name:     "call",
+					Request:  &schema.Ref{Name: "Req"},
+					Response: &schema.Ref{Name: "Resp"},
+					Metadata: []schema.Metadata{
+						&schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}},
+					},
+				},
+			}},
+		},
+	}
+	expected := `// Code generated by FTL. DO NOT EDIT.
+
+//ftl:module test
+package test
+
+import (
+  "context"
+)
+
+var _ = context.Background
+
+type Req struct {
+}
+
+type Resp struct {
+}
+
+//ftl:verb
+func Call(context.Context, Req) (Resp, error) {
+  panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()")
+}
+`
+	bctx := buildContext{
+		moduleDir: "testdata/modules/another",
+		buildDir:  "_ftl",
+		sch:       sch,
+	}
+	testBuild(t, bctx, []assertion{
+		assertGeneratedModule("go/modules/test/external_module.go", expected),
+	})
+}
diff --git a/buildengine/build_kotlin.go b/buildengine/build_kotlin.go
index d23fa2c546..80d9541809 100644
--- a/buildengine/build_kotlin.go
+++ b/buildengine/build_kotlin.go
@@ -157,7 +157,7 @@ var scaffoldFuncs = scaffolder.FuncMap{
 	},
 	"imports": func(m *schema.Module) []string {
 		imports := sets.NewSet[string]()
-		_ = schema.Visit(m, func(n schema.Node, next func() error) error {
+		_ = schema.VisitExcludingMetadataChildren(m, func(n schema.Node, next func() error) error {
 			switch n.(type) {
 			case *schema.Verb:
 				imports.Append("xyz.block.ftl.Context", "xyz.block.ftl.Ignore", "xyz.block.ftl.Verb")
diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go
index 42d10aa43a..597a8639dc 100644
--- a/go-runtime/compile/build.go
+++ b/go-runtime/compile/build.go
@@ -190,7 +190,7 @@ var scaffoldFuncs = scaffolder.FuncMap{
 	},
 	"imports": func(m *schema.Module) map[string]string {
 		imports := map[string]string{}
-		_ = schema.Visit(m, func(n schema.Node, next func() error) error {
+		_ = schema.VisitExcludingMetadataChildren(m, func(n schema.Node, next func() error) error {
 			switch n := n.(type) {
 			case *schema.Ref:
 				if n.Module == "" || n.Module == m.Name {