ci(shared): bump vendored ci files

ci/vendir.lock.yml

@@ -2,14 +2,14 @@ apiVersion: vendir.k14s.io/v1alpha1
 directories:
 - contents:
   - git:
-      commitTitle: 'chore: have version for prepping docker build'
-      sha: ffb5caaacf1331b1e74f7c7f4893bfb7b3656eef
+      commitTitle: 'chore: retry test-on-docker'
+      sha: 95c4fd73e3a8aa92e4b52b97e4066a1f6bdb19ab
     path: .
   path: ../.github/workflows/vendor
 - contents:
   - git:
-      commitTitle: 'chore: have version for prepping docker build'
-      sha: ffb5caaacf1331b1e74f7c7f4893bfb7b3656eef
+      commitTitle: 'chore: retry test-on-docker'
+      sha: 95c4fd73e3a8aa92e4b52b97e4066a1f6bdb19ab
     path: .
   path: ./vendor
 kind: LockConfig

ci/vendir.yml

@@ -8,7 +8,7 @@ directories:
   - path: . # Copy this folder out to ..
     git:
       url: https://github.com/GaloyMoney/concourse-shared.git
-      ref: ffb5caaacf1331b1e74f7c7f4893bfb7b3656eef
+      ref: 95c4fd73e3a8aa92e4b52b97e4066a1f6bdb19ab
     includePaths:
     - shared/actions/*
     excludePaths:
@@ -20,7 +20,7 @@ directories:
   - path: .
     git:
       url: https://github.com/GaloyMoney/concourse-shared.git
-      ref: ffb5caaacf1331b1e74f7c7f4893bfb7b3656eef
+      ref: 95c4fd73e3a8aa92e4b52b97e4066a1f6bdb19ab
     includePaths:
     - shared/ci/**/*
     excludePaths:

ci/vendor/pipeline-fragments.lib.yml

@@ -118,6 +118,46 @@ plan:
 on_failure: #@ slack_failure_notification()
 #@ end
 
+#@ def test_on_docker_host(container, additional_params={}):
+#@ params = dict(additional_params)
+#@ params["REPO_PATH"] = data.values.gh_repository + "-" + container
+#@ params["GOOGLE_CREDENTIALS"] = "((staging-gcp-creds.creds_json))"
+#@ params["SSH_PRIVATE_KEY"] = "((staging-ssh.ssh_private_key))"
+#@ params["SSH_PUB_KEY"] = "((staging-ssh.ssh_public_key))"
+#@ params["TEST_CONTAINER"] = container
+#@ params["JEST_TIMEOUT"] = 90000
+name: #@ container
+serial: true
+plan:
+- { put: docker-host, params: { acquire: true } }
+- in_parallel:
+  - { get: repo, trigger: true }
+  - { get: pipeline-tasks }
+- task: #@ container
+  attempts: 2
+  timeout: 12m
+  tags: ["galoy-staging"]
+  config:
+    platform: linux
+    image_resource: #@ rust_task_image_config()
+    inputs:
+    - name: pipeline-tasks
+    - name: docker-host
+    - name: repo
+      path: #@ data.values.gh_repository + "-" + container
+    caches:
+    - path: cargo-home
+    - path: cargo-target-dir
+    params: #@ params
+    run:
+      path: pipeline-tasks/ci/vendor/tasks/test-on-docker-host.sh
+  ensure:
+    put: docker-host
+    params:
+      release: docker-host
+on_failure: #@ slack_failure_notification()
+#@ end
+
 #@ def nodejs_integration_test():
 name: test-integration
 serial: true
@@ -338,7 +378,7 @@ plan:
         path: pipeline-tasks/ci/vendor/tasks/chart-open-charts-pr.sh
 #@ end
 
-#@ def repo_resource():
+#@ def repo_resource(webhook = False):
 name: repo
 type: git
 source:
@@ -347,6 +387,9 @@ source:
   uri: #@ data.values.git_uri
   branch: #@ data.values.git_branch
   private_key: #@ data.values.github_private_key
+#@ if webhook:
+webhook_token: ((webhook.secret))
+#@ end
 #@ end
 
 #@ def pipeline_tasks_resource():
@@ -369,14 +412,17 @@ source:
   repository: #@ public_docker_registry() + "/" + data.values.gh_repository
 #@ end
 
-#@ def nodejs_deps_resource():
+#@ def nodejs_deps_resource(webhook = False):
 name: deps
 type: git
 source:
   paths: [yarn.lock]
   uri: #@ data.values.git_uri
   branch: #@ data.values.git_branch
   private_key: #@ data.values.github_private_key
+#@ if webhook:
+webhook_token: ((webhook.secret))
+#@ end
 #@ end
 
 #@ def deps_version_resource():
@@ -454,6 +500,16 @@ source:
   private_key: #@ data.values.github_private_key
 #@ end
 
+#@ def docker_host_pool():
+name: docker-host
+type: pool
+source:
+  uri: [email protected]:GaloyMoney/concourse-locks.git
+  branch: main
+  pool: docker-hosts
+  private_key: #@ data.values.github_private_key
+#@ end
+
 #@ def gcr_resource_type():
 name: gcs-resource
 type: docker-image

ci/vendor/tasks/test-on-docker-host.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -eu
+
+export CI_ROOT=$(pwd)
+
+host_name=$(cat docker-host/metadata | jq -r '.docker_host_name')
+echo "Running on host: ${host_name}"
+host_zone=$(cat docker-host/metadata | jq -r '.docker_host_zone')
+gcp_project=$(cat docker-host/metadata | jq -r '.docker_host_project')
+
+gcloud_ssh() {
+  gcloud compute ssh ${host_name} \
+    --zone=${host_zone} \
+    --project=${gcp_project} \
+    --ssh-key-file=${CI_ROOT}/login.ssh \
+    --tunnel-through-iap \
+    --command "$@" 2> /dev/null
+}
+
+cat <<EOF > ${CI_ROOT}/gcloud-creds.json
+${GOOGLE_CREDENTIALS}
+EOF
+cat <<EOF > ${CI_ROOT}/login.ssh
+${SSH_PRIVATE_KEY}
+EOF
+chmod 600 ${CI_ROOT}/login.ssh
+cat <<EOF > ${CI_ROOT}/login.ssh.pub
+${SSH_PUB_KEY}
+EOF
+gcloud auth activate-service-account --key-file ${CI_ROOT}/gcloud-creds.json 2> /dev/null
+
+gcloud_ssh "docker ps -qa | xargs docker rm -fv || true; sudo rm -rf ${REPO_PATH}"
+
+pushd ${REPO_PATH}
+
+make create-tmp-env-ci || true
+
+gcloud compute scp --ssh-key-file=${CI_ROOT}/login.ssh \
+  --recurse $(pwd) ${host_name}:${REPO_PATH} \
+  --tunnel-through-iap \
+  --zone=${host_zone} \
+  --project=${gcp_project} > /dev/null
+
+gcloud_ssh "cd ${REPO_PATH}; export TMP_ENV_CI=tmp.env.ci; export COMPOSE_PROJECT_NAME=${REPO_PATH}; docker compose pull; docker compose -f docker-compose.yml up ${TEST_CONTAINER}"
+
+container_id=$(gcloud_ssh "docker ps -q -f status=exited -f name=${PWD##*/}-${TEST_CONTAINER}-")
+test_status=$(gcloud_ssh "docker inspect $container_id --format='{{.State.ExitCode}}'")
+
+gcloud_ssh "cd ${REPO_PATH}; docker compose down --remove-orphans --timeout 1"
+
+exit $test_status