Skip to content

[gh] self-hosted runners monitor #28637

[gh] self-hosted runners monitor

[gh] self-hosted runners monitor #28637

name: '[gh] self-hosted runners monitor'
on:
workflow_dispatch:
schedule:
# Run every 30th minute.
- cron: '*/30 * * * *'
jobs:
runners_monitor:
if: github.repository == 'Tencent/Hippy'
runs-on: ubuntu-latest
env:
cache_file: /tmp/gh_runners_monitor.data
cache_key: gh_runners_monitor
steps:
- name: Token
uses: navikt/github-app-token-generator@v1
id: get-token
with:
private-key: ${{ secrets.BOT_APP_KEY }}
app-id: ${{ secrets.BOT_APP_ID }}
- name: Check
id: check
uses: actions/[email protected]
with:
github-token: ${{ steps.get-token.outputs.token }}
script: |
const { actions } = github.rest;
const os = require('os');
const fs = require('fs');
const runners = await github.paginate(actions.listSelfHostedRunnersForRepo, {
per_page: 100,
...context.repo
});
const offline_runners = runners.filter(runner => runner.status === 'offline');
if (offline_runners.length > 0) {
fs.appendFileSync(process.env.GITHUB_OUTPUT, `total_runners=${runners.length}${os.EOL}`, { encoding: 'utf8' });
fs.appendFileSync(process.env.GITHUB_OUTPUT, `offline_runners=${JSON.stringify(offline_runners)}${os.EOL}`, { encoding: 'utf8' });
}
- name: Cache
if: steps.check.outputs.offline_runners
uses: actions/cache@v3
env:
prefix: gh_runners_monitor
with:
path: ${{ env.cache_file }}
key: ${{ env.prefix }}-${{ github.run_id }}
restore-keys: ${{ env.prefix }}
- name: Notification
if: steps.check.outputs.offline_runners
uses: actions/[email protected]
env:
offline_runners: ${{ steps.check.outputs.offline_runners }}
with:
script: |
const fs = require('fs');
const os = require('os');
const offline_runners = JSON.parse(process.env.offline_runners);
if (offline_runners.length === 0) {
throw new Error('offline_runners is empty');
}
const message = [
`<font color="warning">${offline_runners.length}/${{ steps.check.outputs.total_runners }}</font> self-hosted runners are offline.`,
...offline_runners.map(runner => `> ${runner.name}`)
].join(os.EOL);
try {
const stored_data = JSON.parse(fs.readFileSync(process.env.cache_file, { encoding: 'utf8'}));
if (stored_data.message === message && Date.now() - stored_data.timestamp < 6_60_60_000 /* every 6th hour */) {
return;
}
} catch(e) {}
await github.request("POST ${{ secrets.WECHAT_WORK_BOT_WEBHOOK }}", {
headers: {
"content-type": "application/json"
},
data: {
chatid: "${{ secrets.WECHAT_WORK_ADMIN_CHAT_ID }}",
msgtype: "markdown",
markdown: {
content: message
}
}
});
fs.writeFileSync(process.env.cache_file, JSON.stringify({
message,
timestamp: Date.now()
}), { encoding: 'utf8' });