From 72ca26103dfec8ae9c66444e49e4cfe66aab468e Mon Sep 17 00:00:00 2001 From: Alex Todorov Date: Thu, 25 Jul 2024 12:31:35 +0300 Subject: [PATCH 1/4] Split Linode VM provisioning into a script which will be used later for provisioning other jobs --- .github/provision-linode-vm.sh | 46 ++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 48 +++------------------------------- 2 files changed, 49 insertions(+), 45 deletions(-) create mode 100755 .github/provision-linode-vm.sh diff --git a/.github/provision-linode-vm.sh b/.github/provision-linode-vm.sh new file mode 100755 index 000000000..7af2211d5 --- /dev/null +++ b/.github/provision-linode-vm.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -x + +# Install linode-cli +python3 --version +pipx install linode-cli +linode-cli --version + +# Authorize hosted-runner +mkdir -p ~/.ssh/ +ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> .github/authorized_keys + + +# Provision VM +echo "INFO: From ENVs: RUNNER_VM_NAME=$LC_RUNNER_VM_NAME" + +# inject authorized keys into cloud-init for the `ubuntu@` user +while read -r LINE; do + echo " - $LINE" >> .github/linode-cloud-init.template +done < .github/authorized_keys + +# WARNING: we do not specify --authorized_keys for root b/c +# linode-cli expects each key as a separate argument and iteratively constructing +# the argument list hits issues with quoting the jey values b/c of white-space. +# All SSH logins should be via the `ubuntu@` user. For more info see: +# https://www.linode.com/community/questions/21290/how-to-pass-multiple-ssh-public-keys-with-linode-cli-linodes-create +linode-cli linodes create --json \ + --image 'linode/ubuntu24.04' --region "$LINODE_REGION" \ + --type "$LINODE_VM_SIZE" --label "$LC_RUNNER_VM_NAME" \ + --root_pass "$(uuidgen)" --backups_enabled false --booted true --private_ip false \ + --metadata.user_data "$(base64 --wrap 0 < .github/linode-cloud-init.template)" > output.json + +# provision the GitHub Runner binary on the VM +# passing additional ENV values +IP_ADDRESS=$(jq -r '.[0].ipv4[0]' < output.json) +SSH_USER_AT_HOSTNAME="ubuntu@$IP_ADDRESS" +echo "INFO: $SSH_USER_AT_HOSTNAME" + +until ssh -i ~/.ssh/id_rsa \ + -o SendEnv=LC_GITHUB_REPO_ADMIN_TOKEN,LC_RUNNER_VM_NAME,LC_WORKFLOW_ID,LC_PROXY_ENABLED,LC_PROXY_SECRET_VARIANT,LC_PROXY_TYPE \ + -o StrictHostKeyChecking=no "$SSH_USER_AT_HOSTNAME" < .github/provision-github-runner.sh; do + echo "DEBUG: retrying ssh connection ..." + sleep 30 +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23d38e7bb..3c75c446a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -582,18 +582,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install linode-cli - run: | - python3 --version - pipx install linode-cli - linode-cli --version - - - name: Authorize hosted-runner - run: | - mkdir -p ~/.ssh/ - ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa - cat ~/.ssh/id_rsa.pub >> .github/authorized_keys - # See https://github.com/actions/runner/issues/1879#issuecomment-1123196869 - name: Create matrix.txt run: | @@ -612,45 +600,15 @@ jobs: - name: Provision VM if: env.LC_GITHUB_REPO_ADMIN_TOKEN run: | - echo "INFO: From ENVs: RUNNER_VM_NAME=${{ env.RUNNER_VM_NAME }}" - echo "INFO: From Step: RUNNER_VM_NAME=${{ steps.get-env.outputs.runner_vm_name }}" - - # inject authorized keys into cloud-init for the `ubuntu@` user - while read -r LINE; do - echo " - $LINE" >> .github/linode-cloud-init.template - done < .github/authorized_keys - - # WARNING: we do not specify --authorized_keys for root b/c - # linode-cli expects each key as a separate argument and iteratively constructing - # the argument list hits issues with quoting the jey values b/c of white-space. - # All SSH logins should be via the `ubuntu@` user. For more info see: - # https://www.linode.com/community/questions/21290/how-to-pass-multiple-ssh-public-keys-with-linode-cli-linodes-create - linode-cli linodes create --json \ - --image 'linode/ubuntu24.04' --region ${{ env.LINODE_REGION }} \ - --type ${{ env.LINODE_VM_SIZE }} --label "${{ steps.get-env.outputs.runner_vm_name }}" \ - --root_pass "$(uuidgen)" --backups_enabled false --booted true --private_ip false \ - --metadata.user_data "$(base64 --wrap 0 < .github/linode-cloud-init.template)" > output.json - - # provision the GitHub Runner binary on the VM - # passing additional ENV values - IP_ADDRESS=$(jq -r '.[0].ipv4[0]' < output.json) - SSH_USER_AT_HOSTNAME="ubuntu@$IP_ADDRESS" - echo "INFO: $SSH_USER_AT_HOSTNAME" - - export LC_RUNNER_VM_NAME="${{ steps.get-env.outputs.runner_vm_name }}" - export LC_WORKFLOW_ID="$GITHUB_RUN_ID" - until ssh -i ~/.ssh/id_rsa \ - -o SendEnv=LC_GITHUB_REPO_ADMIN_TOKEN,LC_RUNNER_VM_NAME,LC_WORKFLOW_ID,LC_PROXY_ENABLED,LC_PROXY_SECRET_VARIANT,LC_PROXY_TYPE \ - -o StrictHostKeyChecking=no "$SSH_USER_AT_HOSTNAME" < .github/provision-github-runner.sh; do - echo "DEBUG: retrying ssh connection ..." - sleep 30 - done + .github/provision-linode-vm.sh env: LC_GITHUB_REPO_ADMIN_TOKEN: ${{ secrets.GH_REPO_ADMIN_TOKEN }} LC_RUNNER_EPHEMERAL: false + LC_RUNNER_VM_NAME: ${{ steps.get-env.outputs.runner_vm_name }} LC_PROXY_ENABLED: ${{ matrix.proxy }} LC_PROXY_SECRET_VARIANT: ${{ matrix.secret }} LC_PROXY_TYPE: ${{ matrix.proxy_type }} + LC_WORKFLOW_ID: ${{ github.run_id }} LINODE_CLI_TOKEN: ${{ secrets.LINODE_CLI_TOKEN }} remove-github-runner: From 6b5a16edfcc1e14a90db0a2650c69c635f8cce0f Mon Sep 17 00:00:00 2001 From: Alex Todorov Date: Thu, 25 Jul 2024 12:34:53 +0300 Subject: [PATCH 2/4] Split Linode VM removal into a script which will be used later for provisioning other jobs --- .github/remove-linode-vm.sh | 11 +++++++++++ .github/workflows/ci.yml | 10 ++-------- 2 files changed, 13 insertions(+), 8 deletions(-) create mode 100755 .github/remove-linode-vm.sh diff --git a/.github/remove-linode-vm.sh b/.github/remove-linode-vm.sh new file mode 100755 index 000000000..7be695886 --- /dev/null +++ b/.github/remove-linode-vm.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +# Install linode-cli +python3 --version +pipx install linode-cli +linode-cli --version + +VM_ID=$(linode-cli linodes list --json --label "$LC_RUNNER_VM_NAME" | jq -r '.[0].id') +linode-cli linodes delete "$VM_ID" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c75c446a..680ad3a76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -660,18 +660,12 @@ jobs: echo "runner_vm_name=${{ env.RUNNER_VM_NAME }}-$HASH_VALUE" >> "$GITHUB_OUTPUT" - - name: Install linode-cli - run: | - python3 --version - pipx install linode-cli - linode-cli --version - - name: Remove VM run: | - VM_ID=$(linode-cli linodes list --json --label "${{ steps.get-env.outputs.runner_vm_name }}" | jq -r '.[0].id') - linode-cli linodes delete "$VM_ID" + .github/remove-linode-vm.sh env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_CLI_TOKEN }} + LC_RUNNER_VM_NAME: ${{ steps.get-env.outputs.runner_vm_name }} integration-test-cli: strategy: From ac25e2f29376de3f4d09b2e699175d83c095b373 Mon Sep 17 00:00:00 2001 From: Alex Todorov Date: Mon, 29 Jul 2024 15:46:51 +0300 Subject: [PATCH 3/4] Install dev dependencies inside the self-hosted runner --- .github/provision-github-runner.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/provision-github-runner.sh b/.github/provision-github-runner.sh index 4d182bf4f..fc962b43c 100755 --- a/.github/provision-github-runner.sh +++ b/.github/provision-github-runner.sh @@ -7,7 +7,8 @@ curl -L https://github.com/actions/runner/releases/download/v2.317.0/actions-run tar xzf ./runner.tar.gz sudo ./bin/installdependencies.sh -sudo apt install -y jq +# for 3rd party dependencies and building the code +sudo apt install -y build-essential clang curl gcc jq libssl-dev pkg-config protobuf-compiler unzip OWNER_REPO_SLUG="gluwa/creditcoin3" REPOSITORY_URL="https://github.com/$OWNER_REPO_SLUG" From 663b94f67ddf6ec5b8b9d32174d1d7f2bdab8053 Mon Sep 17 00:00:00 2001 From: Alex Todorov Date: Tue, 30 Jul 2024 14:36:34 +0300 Subject: [PATCH 4/4] Introduce random dalay and retries before provisioning Linode VMs --- .github/provision-linode-vm.sh | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/provision-linode-vm.sh b/.github/provision-linode-vm.sh index 7af2211d5..0d1af6fc4 100755 --- a/.github/provision-linode-vm.sh +++ b/.github/provision-linode-vm.sh @@ -21,20 +21,30 @@ while read -r LINE; do echo " - $LINE" >> .github/linode-cloud-init.template done < .github/authorized_keys -# WARNING: we do not specify --authorized_keys for root b/c -# linode-cli expects each key as a separate argument and iteratively constructing -# the argument list hits issues with quoting the jey values b/c of white-space. -# All SSH logins should be via the `ubuntu@` user. For more info see: -# https://www.linode.com/community/questions/21290/how-to-pass-multiple-ssh-public-keys-with-linode-cli-linodes-create -linode-cli linodes create --json \ - --image 'linode/ubuntu24.04' --region "$LINODE_REGION" \ - --type "$LINODE_VM_SIZE" --label "$LC_RUNNER_VM_NAME" \ - --root_pass "$(uuidgen)" --backups_enabled false --booted true --private_ip false \ - --metadata.user_data "$(base64 --wrap 0 < .github/linode-cloud-init.template)" > output.json +# retry until we get a VM +IP_ADDRESS="" +while [ -z "$IP_ADDRESS" ]; do + # if all jobs retry rate-limited queries at the same time they still hit the limit + # and subsequently fail. Max number of retries is hard-coded to 3 in linodecli + # use up to 60 sec random delay to avoid everything being scheduled at once! + sleep $((RANDOM % 60)) + + # WARNING: we do not specify --authorized_keys for root b/c + # linode-cli expects each key as a separate argument and iteratively constructing + # the argument list hits issues with quoting the jey values b/c of white-space. + # All SSH logins should be via the `ubuntu@` user. For more info see: + # https://www.linode.com/community/questions/21290/how-to-pass-multiple-ssh-public-keys-with-linode-cli-linodes-create + linode-cli linodes create --json \ + --image 'linode/ubuntu24.04' --region "$LINODE_REGION" \ + --type "$LINODE_VM_SIZE" --label "$LC_RUNNER_VM_NAME" \ + --root_pass "$(uuidgen)" --backups_enabled false --booted true --private_ip false \ + --metadata.user_data "$(base64 --wrap 0 < .github/linode-cloud-init.template)" > output.json + + IP_ADDRESS=$(jq -r '.[0].ipv4[0]' < output.json) +done # provision the GitHub Runner binary on the VM # passing additional ENV values -IP_ADDRESS=$(jq -r '.[0].ipv4[0]' < output.json) SSH_USER_AT_HOSTNAME="ubuntu@$IP_ADDRESS" echo "INFO: $SSH_USER_AT_HOSTNAME"