integration tests #794
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: integration tests | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: "0 11 * * *" #once a day at 11 UTC | |
| concurrency: | |
| group: integration-tests-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| outputs: | |
| pluginDirs: ${{ steps.set-plugin-dirs.outputs.pluginDirs }} | |
| canaryVersion: ${{ steps.npm-canary-version.outputs.version }} | |
| canaryDockerTag: ${{ steps.docker-canary-tag.outputs.result }} | |
| latestVersion: ${{ steps.npm-latest-version.outputs.version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup plugin dir variable | |
| id: set-plugin-dirs | |
| run: echo "pluginDirs=$(find ./examples -type d -name "src" -not -path "*/node_modules*" -maxdepth 3 -exec test -e "{}/plugin.json" \; -print | sed "s/\/src$//" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| - name: Setup NPM canary version variable | |
| id: npm-canary-version | |
| run: echo "version=$(npm view @grafana/ui dist-tags.canary)" >> $GITHUB_OUTPUT | |
| - name: Setup docker canary tag variable | |
| id: docker-canary-tag | |
| uses: actions/github-script@v6 | |
| env: | |
| INPUT_NPM-TAG: ${{ steps.npm-canary-version.outputs.version }} | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require('./.github/workflows/scripts/npm-to-docker-image.js'); | |
| return await script({ core }); | |
| - name: Setup NPM latest version variable | |
| id: npm-latest-version | |
| run: echo "version=$(npm view @grafana/ui dist-tags.latest)" >> $GITHUB_OUTPUT | |
| run-integration-tests: | |
| needs: setup-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| pluginDir: ${{ fromJson(needs.setup-matrix.outputs.pluginDirs) }} | |
| fail-fast: false | |
| env: | |
| GF_AUTH_ANONYMOUS_ENABLED: "true" | |
| GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" | |
| GF_AUTH_BASIC_ENABLED: "false" | |
| GF_DEFAULT_APP_MODE: "development" | |
| GF_INSTALL_PLUGINS: "marcusolsson-static-datasource" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set the name of the plugin-example to be tested | |
| id: example-name | |
| run: | | |
| echo "PLUGIN_NAME=$(basename ${{ matrix.pluginDir }})" >> $GITHUB_OUTPUT | |
| - name: Setup node version | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: "16" | |
| registry-url: "https://registry.npmjs.org" | |
| cache: "npm" | |
| - name: Install dependencies | |
| run: | | |
| npm ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Run frontend tests | |
| run: | | |
| npm run test:ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Build plugin frontend | |
| run: | | |
| npm run build | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Check for backend Mage file | |
| id: backend-check | |
| run: | | |
| echo "MAGEFILE_EXISTS=$(test -f ./Magefile.go && echo true || echo false)" >> $GITHUB_OUTPUT | |
| working-directory: ${{ matrix.pluginDir }} | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "~1.20" | |
| check-latest: true | |
| cache-dependency-path: ${{ matrix.pluginDir }}/go.sum | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| - name: Test plugin backend | |
| uses: magefile/mage-action@v2 | |
| with: | |
| version: latest | |
| args: -v test | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| - name: Build plugin backend | |
| uses: magefile/mage-action@v2 | |
| with: | |
| version: latest | |
| args: -v build:linux | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| ## CANARY_VERSION has to use NPM as we only publish the packages on changes | |
| - name: Set environment vars for testing | |
| run: | | |
| echo "PLUGIN_ID=$(cat src/plugin.json | jq -r '.id')" >> $GITHUB_ENV | |
| echo "EXPECTED_GRAFANA_VERSION=$(npx semver@latest $(cat src/plugin.json | jq -r '.dependencies.grafanaDependency') -c)" >> $GITHUB_ENV | |
| echo "CANARY_VERSION=${{ needs.setup-matrix.outputs.canaryVersion }}" >> $GITHUB_ENV | |
| echo "CANARY_DOCKER_TAG=${{ needs.setup-matrix.outputs.canaryDockerTag }}" >> $GITHUB_ENV | |
| echo "LATEST_STABLE_VERSION=${{ needs.setup-matrix.outputs.latestVersion }}" >> $GITHUB_ENV | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Has Integration Tests | |
| id: has-integration-tests | |
| continue-on-error: true | |
| run: | | |
| echo "DIR=$(test -d ./cypress && echo true || echo false)" >> $GITHUB_OUTPUT | |
| working-directory: ${{ matrix.pluginDir }} | |
| ## If we're creating examples for new features that are only found in canary versions the grafanaDependency | |
| ## will be greater than any stable release of Grafana and so will not be available on docker. | |
| - name: Should run tests against expected and latest | |
| id: should-run-expected-latest-tests | |
| continue-on-error: true | |
| run: | | |
| npm exec --package semver-compare-cli --call 'semver-compare $LATEST_STABLE_VERSION ge $EXPECTED_GRAFANA_VERSION' | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Start Docker container using expected version of grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana:$EXPECTED_GRAFANA_VERSION | |
| working-directory: ${{ matrix.pluginDir }} | |
| ## Expected Version Tests | |
| ## Runs the plugin tests against the minimum compatible version of Grafana defined in its plugin.json. | |
| # Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests. | |
| - name: Start Integration tests using expected versions | |
| id: expected-version-tests | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| uses: nick-fields/retry@v2 | |
| continue-on-error: true | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: npm run e2e --prefix ${{ matrix.pluginDir }} | |
| - name: Uploading artifacts for tests with expected versions | |
| uses: actions/upload-artifact@v3 | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.expected-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| with: | |
| name: ${{steps.example-name.outputs.PLUGIN_NAME}}-expected-version-tests | |
| path: | | |
| ${{ matrix.pluginDir }}/cypress/screenshots | |
| ${{ matrix.pluginDir }}/cypress/videos | |
| ${{ matrix.pluginDir }}/cypress/report.json | |
| retention-days: 3 | |
| - name: Failing build due to test failures (expected versions) | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.expected-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: exit 1 | |
| - name: Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| # Latest Version Tests | |
| # Runs the plugin tests against the latest stable version of Grafana. | |
| # Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests. | |
| - name: Install latest version of e2e libraries | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| npm install @grafana/e2e-selectors@latest @grafana/e2e@latest | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Start Docker container using latest version of grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana:latest | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Start Integration tests using latest version of Grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| id: latest-version-tests | |
| uses: nick-fields/retry@v2 | |
| continue-on-error: true | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: npm run e2e --prefix ${{ matrix.pluginDir }} | |
| - name: Uploading artifacts for tests with latest version of Grafana | |
| uses: actions/upload-artifact@v3 | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.latest-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| with: | |
| name: ${{steps.example-name.outputs.PLUGIN_NAME}}-latest-version-tests | |
| path: | | |
| ${{ matrix.pluginDir }}/cypress/screenshots | |
| ${{ matrix.pluginDir }}/cypress/videos | |
| ${{ matrix.pluginDir }}/cypress/report.json | |
| retention-days: 3 | |
| - name: Failing build due to test failures (latest version of Grafana) | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.latest-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: exit 1 | |
| - name: Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| ## Canary Version Tests | |
| ## Runs the plugin tests against the latest build of Grafana main branch. | |
| - name: Upgrade @grafana packages using canary_version | |
| run: | | |
| npm install --legacy-peer-deps $(echo $(cat package.json | jq -r --arg version $CANARY_VERSION '["@grafana/eslint-config", "@grafana/tsconfig", "@grafana/scenes", "@grafana/experimental"] as $blacklist | [(.devDependencies,.dependencies) | keys] | flatten | unique | map(select ( test("^@grafana") ) ) as $deps | $deps - $blacklist | map("\(.)@\($version)") | join(" ")') ) | |
| npm install --force --save-optional @swc/core @swc/core-linux-arm-gnueabihf @swc/core-linux-arm64-gnu @swc/core-linux-arm64-musl @swc/core-linux-x64-gnu @swc/core-linux-x64-musl | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Run frontend tests | |
| run: | | |
| npm run test:ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Build plugin with canary_version @grafana dependencies | |
| run: | | |
| npm run build | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Build plugin backend | |
| uses: magefile/mage-action@v2 | |
| with: | |
| version: latest | |
| args: -v build:linux | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| # Canary versions live at grafana/grafana-dev | |
| - name: Start Docker container using canary version of grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| run: | | |
| docker run -d -p 3000:3000 --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED grafana/grafana-dev:$CANARY_DOCKER_TAG | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Start Integration tests using canary version of Grafana | |
| id: canary-version-tests | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| continue-on-error: true | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: npm run e2e --prefix ${{ matrix.pluginDir }} | |
| - name: Uploading artifacts for tests with canary versions | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.canary-version-tests.outcome != 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{steps.example-name.outputs.PLUGIN_NAME}}-canary-version-tests | |
| path: | | |
| ${{ matrix.pluginDir }}/cypress/screenshots | |
| ${{ matrix.pluginDir }}/cypress/videos | |
| ${{ matrix.pluginDir }}/cypress/report.json | |
| retention-days: 3 | |
| - name: Failing build due to test failures (canary versions) | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.canary-version-tests.outcome != 'success' | |
| run: exit 1 | |
| - name: Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| notify: | |
| if: ${{ (always() && github.event_name == 'schedule') }} | |
| runs-on: ubuntu-latest | |
| needs: [run-integration-tests] | |
| steps: | |
| - name: Send GitHub Action trigger data to Slack workflow | |
| id: slack | |
| uses: slackapi/[email protected] | |
| if: contains(fromJson('["failure"]'), needs.run-integration-tests.result) | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_ID }} | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":blob-sad: <${{ github.server_url }}/${{ github.repository }}|Plugin Examples> Integration Test `${{ needs.run-integration-tests.result }}`" | |
| } | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": ":octocat: Open Job", | |
| "emoji": true | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}", | |
| "action_id": "actionId-0" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "context", | |
| "elements": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Commit*: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Event*: ${{ github.event_name }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Branch*: `${{ github.head_ref || github.ref_name }}`" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*PR Number*: `${{ github.event.pull_request.number || 'Not triggered by a PR' }}`" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |