-
Notifications
You must be signed in to change notification settings - Fork 53
357 lines (322 loc) · 16.1 KB
/
integration-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
name: integration tests
on:
pull_request:
schedule:
- cron: "0 11 * * *" #once a day at 11 UTC
concurrency:
group: integration-tests-${{ github.ref }}
cancel-in-progress: true
jobs:
setup-matrix:
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
pluginDirs: ${{ steps.set-plugin-dirs.outputs.pluginDirs }}
canaryVersion: ${{ steps.npm-canary-version.outputs.version }}
canaryDockerTag: ${{ steps.docker-canary-tag.outputs.result }}
latestVersion: ${{ steps.npm-latest-version.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup plugin dir variable
id: set-plugin-dirs
run: echo "pluginDirs=$(find ./examples -type d -name "src" -not -path "*/node_modules*" -maxdepth 3 -exec test -e "{}/plugin.json" \; -print | sed "s/\/src$//" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
- name: Setup NPM canary version variable
id: npm-canary-version
run: echo "version=$(npm view @grafana/ui dist-tags.canary)" >> $GITHUB_OUTPUT
- name: Setup docker canary tag variable
id: docker-canary-tag
uses: actions/github-script@v6
env:
INPUT_NPM-TAG: ${{ steps.npm-canary-version.outputs.version }}
with:
result-encoding: string
script: |
const script = require('./.github/workflows/scripts/npm-to-docker-image.js');
return await script({ core });
- name: Setup NPM latest version variable
id: npm-latest-version
run: echo "version=$(npm view @grafana/ui dist-tags.latest)" >> $GITHUB_OUTPUT
run-integration-tests:
needs: setup-matrix
runs-on: ubuntu-latest
strategy:
matrix:
pluginDir: ${{ fromJson(needs.setup-matrix.outputs.pluginDirs) }}
fail-fast: false
env:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
GF_AUTH_BASIC_ENABLED: "false"
GF_DEFAULT_APP_MODE: "development"
GF_INSTALL_PLUGINS: "marcusolsson-static-datasource"
steps:
- uses: actions/checkout@v3
- name: Set the name of the plugin-example to be tested
id: example-name
run: |
echo "PLUGIN_NAME=$(basename ${{ matrix.pluginDir }})" >> $GITHUB_OUTPUT
- name: Setup node version
uses: actions/setup-node@v3
with:
node-version: "16"
registry-url: "https://registry.npmjs.org"
cache: "npm"
- name: Install dependencies
run: |
npm ci
working-directory: ${{ matrix.pluginDir }}
- name: Run frontend tests
run: |
npm run test:ci
working-directory: ${{ matrix.pluginDir }}
- name: Build plugin frontend
run: |
npm run build
working-directory: ${{ matrix.pluginDir }}
- name: Check for backend Mage file
id: backend-check
run: |
echo "MAGEFILE_EXISTS=$(test -f ./Magefile.go && echo true || echo false)" >> $GITHUB_OUTPUT
working-directory: ${{ matrix.pluginDir }}
- uses: actions/setup-go@v4
with:
go-version: "~1.20"
check-latest: true
cache-dependency-path: ${{ matrix.pluginDir }}/go.sum
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
- name: Test plugin backend
uses: magefile/mage-action@v2
with:
version: latest
args: -v test
workdir: ${{ matrix.pluginDir }}
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
- name: Build plugin backend
uses: magefile/mage-action@v2
with:
version: latest
args: -v build:linux
workdir: ${{ matrix.pluginDir }}
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
## CANARY_VERSION has to use NPM as we only publish the packages on changes
- name: Set environment vars for testing
run: |
echo "PLUGIN_ID=$(cat src/plugin.json | jq -r '.id')" >> $GITHUB_ENV
echo "EXPECTED_GRAFANA_VERSION=$(npx semver@latest $(cat src/plugin.json | jq -r '.dependencies.grafanaDependency') -c)" >> $GITHUB_ENV
echo "CANARY_VERSION=${{ needs.setup-matrix.outputs.canaryVersion }}" >> $GITHUB_ENV
echo "CANARY_DOCKER_TAG=${{ needs.setup-matrix.outputs.canaryDockerTag }}" >> $GITHUB_ENV
echo "LATEST_STABLE_VERSION=${{ needs.setup-matrix.outputs.latestVersion }}" >> $GITHUB_ENV
working-directory: ${{ matrix.pluginDir }}
- name: Has Integration Tests
id: has-integration-tests
continue-on-error: true
run: |
echo "DIR=$(test -d ./cypress && echo true || echo false)" >> $GITHUB_OUTPUT
working-directory: ${{ matrix.pluginDir }}
## If we're creating examples for new features that are only found in canary versions the grafanaDependency
## will be greater than any stable release of Grafana and so will not be available on docker.
- name: Should run tests against expected and latest
id: should-run-expected-latest-tests
continue-on-error: true
run: |
npm exec --package semver-compare-cli --call 'semver-compare $LATEST_STABLE_VERSION ge $EXPECTED_GRAFANA_VERSION'
working-directory: ${{ matrix.pluginDir }}
- name: Start Docker container using expected version of grafana
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
run: |
docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana:$EXPECTED_GRAFANA_VERSION
working-directory: ${{ matrix.pluginDir }}
## Expected Version Tests
## Runs the plugin tests against the minimum compatible version of Grafana defined in its plugin.json.
# Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests.
- name: Start Integration tests using expected versions
id: expected-version-tests
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
uses: nick-fields/retry@v2
continue-on-error: true
with:
timeout_minutes: 30
max_attempts: 3
retry_on: error
command: npm run e2e --prefix ${{ matrix.pluginDir }}
- name: Uploading artifacts for tests with expected versions
uses: actions/upload-artifact@v3
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.expected-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success'
with:
name: ${{steps.example-name.outputs.PLUGIN_NAME}}-expected-version-tests
path: |
${{ matrix.pluginDir }}/cypress/screenshots
${{ matrix.pluginDir }}/cypress/videos
${{ matrix.pluginDir }}/cypress/report.json
retention-days: 3
- name: Failing build due to test failures (expected versions)
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.expected-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success'
run: exit 1
- name: Stop and remove Docker container
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
run: |
docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
working-directory: ${{ matrix.pluginDir }}
# Latest Version Tests
# Runs the plugin tests against the latest stable version of Grafana.
# Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests.
- name: Install latest version of e2e libraries
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
run: |
npm install @grafana/e2e-selectors@latest @grafana/e2e@latest
working-directory: ${{ matrix.pluginDir }}
- name: Start Docker container using latest version of grafana
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
run: |
docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana:latest
working-directory: ${{ matrix.pluginDir }}
- name: Start Integration tests using latest version of Grafana
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
id: latest-version-tests
uses: nick-fields/retry@v2
continue-on-error: true
with:
timeout_minutes: 30
max_attempts: 3
retry_on: error
command: npm run e2e --prefix ${{ matrix.pluginDir }}
- name: Uploading artifacts for tests with latest version of Grafana
uses: actions/upload-artifact@v3
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.latest-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success'
with:
name: ${{steps.example-name.outputs.PLUGIN_NAME}}-latest-version-tests
path: |
${{ matrix.pluginDir }}/cypress/screenshots
${{ matrix.pluginDir }}/cypress/videos
${{ matrix.pluginDir }}/cypress/report.json
retention-days: 3
- name: Failing build due to test failures (latest version of Grafana)
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.latest-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success'
run: exit 1
- name: Stop and remove Docker container
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
run: |
docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
working-directory: ${{ matrix.pluginDir }}
## Canary Version Tests
## Runs the plugin tests against the latest build of Grafana main branch.
- name: Upgrade @grafana packages using canary_version
run: |
npm install --legacy-peer-deps $(echo $(cat package.json | jq -r --arg version $CANARY_VERSION '["@grafana/eslint-config", "@grafana/tsconfig", "@grafana/scenes", "@grafana/experimental"] as $blacklist | [(.devDependencies,.dependencies) | keys] | flatten | unique | map(select ( test("^@grafana") ) ) as $deps | $deps - $blacklist | map("\(.)@\($version)") | join(" ")') )
npm install --force --save-optional @swc/core @swc/core-linux-arm-gnueabihf @swc/core-linux-arm64-gnu @swc/core-linux-arm64-musl @swc/core-linux-x64-gnu @swc/core-linux-x64-musl
working-directory: ${{ matrix.pluginDir }}
- name: Run frontend tests
run: |
npm run test:ci
working-directory: ${{ matrix.pluginDir }}
- name: Build plugin with canary_version @grafana dependencies
run: |
npm run build
working-directory: ${{ matrix.pluginDir }}
- name: Build plugin backend
uses: magefile/mage-action@v2
with:
version: latest
args: -v build:linux
workdir: ${{ matrix.pluginDir }}
if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'
# Canary versions live at grafana/grafana-dev
- name: Start Docker container using canary version of grafana
if: steps.has-integration-tests.outputs.DIR == 'true'
run: |
docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana-dev:$CANARY_DOCKER_TAG
working-directory: ${{ matrix.pluginDir }}
- name: Start Integration tests using canary version of Grafana
id: canary-version-tests
if: steps.has-integration-tests.outputs.DIR == 'true'
continue-on-error: true
uses: nick-fields/retry@v2
with:
timeout_minutes: 30
max_attempts: 3
retry_on: error
command: npm run e2e --prefix ${{ matrix.pluginDir }}
- name: Uploading artifacts for tests with canary versions
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.canary-version-tests.outcome != 'success'
uses: actions/upload-artifact@v3
with:
name: ${{steps.example-name.outputs.PLUGIN_NAME}}-canary-version-tests
path: |
${{ matrix.pluginDir }}/cypress/screenshots
${{ matrix.pluginDir }}/cypress/videos
${{ matrix.pluginDir }}/cypress/report.json
retention-days: 3
- name: Failing build due to test failures (canary versions)
if: steps.has-integration-tests.outputs.DIR == 'true' && steps.canary-version-tests.outcome != 'success'
run: exit 1
- name: Stop and remove Docker container
if: steps.has-integration-tests.outputs.DIR == 'true'
run: |
docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
working-directory: ${{ matrix.pluginDir }}
notify:
if: ${{ (always() && github.event_name == 'schedule') }}
runs-on: ubuntu-latest
needs: [run-integration-tests]
steps:
- name: Send GitHub Action trigger data to Slack workflow
id: slack
uses: slackapi/[email protected]
if: contains(fromJson('["failure"]'), needs.run-integration-tests.result)
with:
channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":blob-sad: <${{ github.server_url }}/${{ github.repository }}|Plugin Examples> Integration Test `${{ needs.run-integration-tests.result }}`"
}
},
{
"type": "actions",
"elements": [
{
"type": "button",
"text": {
"type": "plain_text",
"text": ":octocat: Open Job",
"emoji": true
},
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}",
"action_id": "actionId-0"
}
]
},
{
"type": "divider"
},
{
"type": "context",
"elements": [
{
"type": "mrkdwn",
"text": "*Commit*: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
},
{
"type": "mrkdwn",
"text": "*Event*: ${{ github.event_name }}"
},
{
"type": "mrkdwn",
"text": "*Branch*: `${{ github.head_ref || github.ref_name }}`"
},
{
"type": "mrkdwn",
"text": "*PR Number*: `${{ github.event.pull_request.number || 'Not triggered by a PR' }}`"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK