Merge branch 'main' into worstell/20240319-extract-secrets-configs-in…

…-kotlin

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
       - name: Docker Compose
         run: docker compose up -d --wait
       - name: Initialise database
-        run: ftl-initdb --recreate
+        run: just init-db
       - name: Vet SQL
         run: sqlc vet
   lint:

.github/workflows/integration.yml

@@ -37,6 +37,8 @@ jobs:
         uses: ./.github/actions/build-cache
       - name: Docker Compose
         run: docker compose up -d --wait
+      - name: Create DB
+        run: just init-db
       - name: Download Go Modules
         run: go mod download
       - name: Run ${{ matrix.test }}

.github/workflows/rebuild.yml

@@ -22,8 +22,6 @@ jobs:
         # New version of "sqlc generate" needs the DB up for some unknown reason
         run: docker compose up -d --wait
       - name: Init DB
-        run: ftl-initdb
-      - name: Clean All
-        run: bit -c
+        run: just init-db
       - name: Rebuild All
-        run: bit
+        run: just build-all

.github/workflows/release.yml

@@ -134,11 +134,11 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Build Console
-        run: bit frontend/dist/index.html
+        run: just build-frontend
       - name: Publish Go Binaries
         run: |
           just errtrace
-          bit build/release/ftl
+          just build ftl # Ensure all the prerequisites are built before we use goreleaser
           goreleaser release --skip=validate
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/writecache.yml

@@ -18,11 +18,9 @@ jobs:
       - name: Docker Compose
         run: docker compose up -d --wait
       - name: Init DB
-        run: ftl-initdb
-      - name: Clean All
-        run: bit -c
+        run: just init-db
       - name: Rebuild All
-        run: bit
+        run: just build-all
       - name: Download Maven Dependencies
         run: mvn -f kotlin-runtime/ftl-runtime dependency:resolve --batch-mode
       - name: Download Go Dependencies

Dockerfile.controller

@@ -20,9 +20,9 @@ RUN go mod download -x
 # Build
 COPY . /src/
 RUN just errtrace
-RUN bit build/release/ftl-controller
-RUN bit build/release/ftl-initdb
-RUN bit build/release/ftl
+RUN just build ftl-controller
+RUN just build ftl-initdb
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04

Dockerfile.runner

@@ -23,12 +23,12 @@ RUN go mod download -x
 COPY . /src/
 
 # Build runner template
-RUN bit build/template/ftl/jars/ftl-runtime.jar
+RUN just build-kt-runtime
 
 # Build runner
 RUN just errtrace
-RUN bit build/release/ftl-runner
-RUN bit build/release/ftl
+RUN just build ftl-runner
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04

Justfile

@@ -1,25 +1,74 @@
 set positional-arguments
 
-# Start a hot-reloading dev cluster
-dev: install-jars
-  goreman -logtime=false start
+RELEASE := "build/release"
+VERSION := `git describe --tags --always --dirty | sed -e 's/^v//'`
+KT_RUNTIME_OUT := "kotlin-runtime/ftl-runtime/target/ftl-runtime-1.0-SNAPSHOT.jar"
+KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar"
+RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip"
+TIMESTAMP := `date +%s`
+SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto"
+ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template"
+FRONTEND_OUT := "frontend/dist/index.html"
 
-# Install all JARs to local Maven repository and local build directory
-install-jars:
-  bit 'kotlin-runtime/**/*.jar'
+_help:
+  @just -l
 
-# Deploy the Go time module
-deploy-time:
-  ftl deploy examples/time
+# Run errtrace on Go files to add stacks
+errtrace:
+  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
 
-# Deploy the Kotlin echo module
-deploy-echo-kotlin:
-  ftl deploy examples/echo-kotlin
+# Clean the build directory
+clean:
+  rm -rf build
+  rm -rf frontend/node_modules
+  find . -name '*.zip' -exec rm {} \;
+  mvn -f kotlin-runtime/ftl-runtime clean
 
-regen-schema:
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.proto
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.pb.go
+# Build everything
+build-all: build-frontend build-generate build-kt-runtime build-protos build-sqlc build-zips
+  @just build ftl ftl-controller ftl-runner ftl-initdb
 
-# Run errtrace on Go files to add stacks
-errtrace:
-  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
+# Run "go generate" on all packages
+build-generate:
+  @mk backend/schema/aliaskind_enumer.go : backend/schema/metadataalias.go -- go generate -x backend/schema
+  @mk internal/log/log_level_string.go : internal/log/api.go -- go generate -x internal/log
+
+# Build command-line tools
+build +tools: build-protos build-sqlc build-zips build-frontend
+  #!/bin/bash
+  shopt -s extglob
+  for tool in $@; do mk "{{RELEASE}}/$tool" : !(build) -- go build -o "{{RELEASE}}/$tool" -tags release -ldflags "-X github.com/TBD54566975/ftl.Version={{VERSION}} -X github.com/TBD54566975/ftl.Timestamp={{TIMESTAMP}}" "./cmd/$tool"; done
+
+export DATABASE_URL := "postgres://postgres:secret@localhost:54320/ftl?sslmode=disable"
+
+# Explicitly initialise the database
+init-db:
+  dbmate drop || true
+  dbmate create
+  dbmate --migrations-dir backend/controller/sql/schema up
+
+# Regenerate SQLC code
+build-sqlc:
+  @mk backend/controller/sql/{db.go,models.go,querier.go,queries.sql.go} : backend/controller/sql/queries.sql backend/controller/sql/schema -- sqlc generate --experimental
+
+# Build the ZIP files that are embedded in the FTL release binaries
+build-zips: build-kt-runtime
+  @for dir in {{ZIP_DIRS}}; do (cd $dir && mk ../$(basename ${dir}).zip : . -- "rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip ."); done
+
+# Rebuild frontend
+build-frontend: npm-install
+  @mk {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
+
+# Build the Kotlin runtime (if necessary)
+build-kt-runtime:
+  @mk {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
+  @mk {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} : {{KT_RUNTIME_OUT}} -- "mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}) && install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}"
+  @mk {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
+
+# Install Node dependencies
+npm-install:
+  @mk frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
+
+# Regenerate protos
+build-protos: npm-install
+  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"

backend/controller/scaling/localscaling/devel.go

@@ -17,7 +17,7 @@ var templateDirOnce sync.Once
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
 		// TODO: Figure out how to make maven build offline
-		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
+		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "just", "build-kt-runtime").RunBuffered(ctx)
 		if err != nil {
 			panic(err)
 		}

backend/schema/intvalue.go

@@ -31,4 +31,6 @@ func (i *IntValue) String() string {
 	return fmt.Sprintf("%d", i.Value)
 }
 
+func (i *IntValue) GetValue() any { return i.Value }
+
 func (*IntValue) schemaValueType() Type { return &Int{} }