diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml
index bc33e6e58..2d0d46abf 100644
--- a/.github/workflows/smoketest.yml
+++ b/.github/workflows/smoketest.yml
@@ -1,4 +1,5 @@
 on:
+  pull_request:
   push:
     branches:
       - main
@@ -40,6 +41,8 @@ jobs:
           set -euo pipefail
           echo "Deploying the tagged release to the cluster"
           cd deployment && just deploy-version ${{ env.LATEST_VERSION }} && cd ..
+      - name: Build Language Plugins
+        run: just build-language-plugins
       - name: Smoke test the tagged release images
         run: |
           set -euo pipefail
@@ -52,6 +55,8 @@ jobs:
           fetch-depth: 1
       - name: Init Hermit
         uses: cashapp/activate-hermit@v1
+      - name: Build Language Plugins
+        run: just build-language-plugins
       - name: Smoke test HEAD with a full deploy to test upgrade path
         run: |
           set -euo pipefail
diff --git a/deployment/values-release.yaml b/deployment/values-release.yaml
index b23fc2061..8a96cdf1c 100644
--- a/deployment/values-release.yaml
+++ b/deployment/values-release.yaml
@@ -15,12 +15,10 @@ controller:
         port: 8891
 provisioner:
   enabled: true
-  image:
-    repository: "ftl:5000/ftl-provisioner"
-    tag: "latest"
   service:
     ports:
       - name: "http-8893"
         port: 8893
+
 istio:
   enabled: true