Skip to content

Commit

Permalink
Merge pull request #560 from cpanel/RE-991
Browse files Browse the repository at this point in the history 
Update reboot_watch to allow for one retry in case elevate-cpanel fails
  • Loading branch information
@toddr @davelcpanelnet
toddr authored and davelcpanelnet committed Nov 26, 2024
2 parents 42320ba + ab07383 commit e25de96
Show file tree
Hide file tree
Showing 19 changed files with 983 additions and 125 deletions.
113 changes: 62 additions & 51 deletions .github/workflows/openstack.yml → .github/workflows/openstack-centos-7.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Openstack Test Run
name: "Test Run: centos-7"

on:
push:
Expand All @@ -11,7 +11,7 @@ on:
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-openstack
group: ${{ github.workflow }}-${{ github.ref }}-openstack-centos-7
cancel-in-progress: true

env:
Expand All @@ -29,7 +29,7 @@ env:
TF_VAR_os_project_name: ${{ secrets.OS_PROJECT_NAME }}
TF_VAR_os_region_name: ${{ secrets.OS_REGION_NAME }}
TF_VAR_os_username: ${{ secrets.OS_USERNAME }}
tf_working_directory: "./.github/workflows/openstack/terraform"
tf_working_directory: "${{ github.workspace }}/.github/workflows/openstack/centos-7"

jobs:
testsuite:
Expand All @@ -39,7 +39,7 @@ jobs:
runs-on: self-hosted
defaults:
run:
working-directory: "./.github/workflows/openstack/terraform"
working-directory: "${{ github.workspace }}/.github/workflows/openstack/centos-7"
steps:
- uses: actions/checkout@v4
- name: Terraform fmt
Expand All @@ -48,11 +48,6 @@ jobs:
terraform fmt -check
continue-on-error: true

- name: DEBUGVAR
id: debugvar
run: |
env | awk 'tolower($0)~/auth|project_name/'
- name: Terraform Init
id: init
run: |
Expand All @@ -66,7 +61,16 @@ jobs:
- name: Terraform Apply
id: apply
run: |
pwd
terraform apply -no-color -auto-approve
mv -v .terraform.lock.hcl terraform.lock.hcl
find $PWD
- name: Upload Terraform State File as Artifact
uses: actions/[email protected]
with:
name: ${{ github.run_id }}-terraform.tfstate
path: "${{ github.workspace }}/.github/workflows/openstack/centos-7"

- name: Terraform Output File Create
run: |
Expand All @@ -77,18 +81,11 @@ jobs:
run: |
echo "$(jq -r '.address.value' ${{ github.run_id }}-tf.out.json)" > ${{ github.workspace }}/${{ github.run_id }}-vm_ip
- name: Upload JSON Output
uses: actions/upload-artifact@v4
with:
name: ${{ github.run_id }}-tf.out.json
path: ${{ github.workspace }}/
overwrite: true

- name: Upload VM Output
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v4.4.3
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/${{ github.run_id }}-vm_ip
overwrite: true

clone_elevate_repo:
Expand All @@ -98,7 +95,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand Down Expand Up @@ -151,7 +148,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand Down Expand Up @@ -186,7 +183,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand Down Expand Up @@ -217,8 +214,9 @@ jobs:
cp -pv /opt/${REPODIR}/.github/workflows/openstack/status_marker /scripts/status_marker
cp -pv /opt/${REPODIR}/.github/workflows/openstack/reboot_watch /scripts/reboot_watch
chmod -v +x /scripts/elevate-cpanel
/usr/local/cpanel/cpkeyclt
until /usr/local/cpanel/cpkeyclt; do echo "Retrying /usr/local/cpanel/cpkeyct until it passes..."; sleep 5; done
/scripts/elevate-cpanel --non-interactive --skip-cpanel-version-check --start &
exit 0
wait_for_stage_1_reboot:
runs-on: self-hosted
Expand All @@ -227,7 +225,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -254,7 +252,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -281,8 +279,8 @@ jobs:
debug: true
script: |
/scripts/status_marker 1
/scripts/elevate-cpanel --log &
sleep 1.5
tail -n40 -F /var/log/elevate-cpanel.log&
sleep .5
REBOOT_STRING="Rebooting into stage 2 of 5" RETVAL=1 /scripts/reboot_watch
wait_for_stage_2_reboot:
Expand All @@ -292,7 +290,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -319,7 +317,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -346,8 +344,8 @@ jobs:
debug: true
script: |
/scripts/status_marker 2
/scripts/elevate-cpanel --log &
sleep 1.5
tail -n40 -F /var/log/elevate-cpanel.log&
sleep .5
REBOOT_STRING="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch
wait_for_stage_3_reboot:
Expand All @@ -357,7 +355,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -384,7 +382,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -410,8 +408,8 @@ jobs:
command_timeout: 30m
script: |
/scripts/status_marker 3
/scripts/elevate-cpanel --log &
sleep 1.5
tail -n40 -F /var/log/elevate-cpanel.log&
sleep .5
REBOOT_STRING="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch
wait_for_stage_4_reboot:
Expand All @@ -421,7 +419,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -448,7 +446,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -474,8 +472,8 @@ jobs:
command_timeout: 35m
script: |
/scripts/status_marker 4
/scripts/elevate-cpanel --log &
sleep 1.5
tail -n40 -F /var/log/elevate-cpanel.log&
sleep .5
REBOOT_STRING="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch
wait_for_stage_5_reboot:
Expand All @@ -485,7 +483,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -512,7 +510,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -538,8 +536,8 @@ jobs:
command_timeout: 35m
script: |
/scripts/status_marker 5
/scripts/elevate-cpanel --log &
sleep 1.5
tail -n40 -F /var/log/elevate-cpanel.log&
sleep 2.5
SKIP_PID_CHECK=1 REBOOT_STRING="Doing final reboot" RETVAL=1 /scripts/reboot_watch
wait_for_final_reboot:
Expand All @@ -549,7 +547,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -576,7 +574,7 @@ jobs:
VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
steps:
- name: Download VM IP
uses: actions/download-artifact@v4
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-vm_ip
path: ${{ github.workspace }}/
Expand All @@ -603,18 +601,31 @@ jobs:
script: |
REPODIR=$(echo ${{ github.repository }} | cut -d / -f2)
/usr/local/cpanel/3rdparty/bin/prove -lvm /opt/${REPODIR}/t/integration/*.t
/usr/local/cpanel/3rdparty/bin/prove -lvm /opt/${REPODIR}/t/integration/centos7-to-almalinux8/*.t
terraform_openstack_destroy:
runs-on: self-hosted
needs: verify_upgraded_os
defaults:
run:
working-directory: "./.github/workflows/openstack/terraform"
working-directory: "${{ github.workspace }}/.github/workflows/openstack/centos-7"
steps:
- name: Download Terraform Output JSON
uses: actions/download-artifact@v4
- name: Download Terraform State
uses: actions/download-artifact@v4.1.8
with:
name: ${{ github.run_id }}-tf.out.json
path: ${{ github.workspace }}/tf.out.json
- name: Destroy OpenStack VM
run: terraform destroy -no-color -auto-approve
name: ${{ github.run_id }}-terraform.tfstate
path: "${{ github.workspace }}/.github/workflows/openstack/centos-7/terraform"


- name: Stage Files & Show Artifact Paths
run: |
cd ${{ github.workspace }}/.github/workflows/openstack/centos-7/terraform/
mv -v terraform.lock.hcl .terraform.lock.hcl
find ${{ github.workspace }}/.github/workflows/openstack/centos-7/
- name: Initialize Terraform & Destroy
run: |
cd ${{ github.workspace }}/.github/workflows/openstack/centos-7/terraform/
rm -Rfv .terraform || true
terraform init
terraform destroy -no-color -auto-approve
Loading

0 comments on commit e25de96

Please sign in to comment.