integration-tests.yml

name: integration tests
on:
  pull_request:
  schedule:
    - cron: "0 11 * * *" #once a day at 11 UTC
concurrency:
  group: integration-tests-${{ github.ref }}
  cancel-in-progress: true
permissions:
  contents: read
  id-token: write
jobs:
  setup-matrix:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      pluginDirs: ${{ steps.set-plugin-dirs.outputs.pluginDirs }}
      canaryVersion: ${{ steps.npm-canary-version.outputs.version }}
      canaryDockerTag: ${{ steps.docker-canary-tag.outputs.result }}
      latestVersion: ${{ steps.npm-latest-version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup plugin dir variable
        id: set-plugin-dirs
        run: echo "pluginDirs=$(find ./examples -type d -name "src" -not -path "*/node_modules*" -maxdepth 3 -exec test -e "{}/plugin.json" \; -print | sed "s/\/src$//" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
      - name: Setup NPM canary version variable
        id: npm-canary-version
        run: echo "version=$(npm view @grafana/ui dist-tags.canary)" >> $GITHUB_OUTPUT
      - name: Setup docker canary tag variable
        id: docker-canary-tag
        uses: actions/github-script@v7
        env:
          INPUT_NPM-TAG: ${{ steps.npm-canary-version.outputs.version }}
        with:
          result-encoding: string
          script: |
            const script = require('./.github/workflows/scripts/npm-to-docker-image.js');
            return await script({ core });
      - name: Setup NPM latest version variable
        id: npm-latest-version
        run: echo "version=$(npm view @grafana/ui dist-tags.latest)" >> $GITHUB_OUTPUT

  run-integration-tests:
    needs: setup-matrix
    runs-on: ubuntu-latest
    strategy:
      matrix:
        pluginDir: ${{ fromJson(needs.setup-matrix.outputs.pluginDirs) }}
      fail-fast: false
    env:
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
      GF_AUTH_BASIC_ENABLED: "false"
      GF_DEFAULT_APP_MODE: "development"
      GF_INSTALL_PLUGINS: "marcusolsson-static-datasource"
    steps:
      - uses: actions/checkout@v4

      - name: Set the name of the plugin-example to be tested
        id: example-name
        run: |
          echo "PLUGIN_NAME=$(basename ${{ matrix.pluginDir }})" >> $GITHUB_OUTPUT

      - name: Setup node version
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          registry-url: "https://registry.npmjs.org"
          cache: "npm"

      - name: Install dependencies
        run: |
          npm ci
        working-directory: ${{ matrix.pluginDir }}

      - name: Run frontend tests
        run: |
          npm run test:ci
        working-directory: ${{ matrix.pluginDir }}

      - name: Build plugin frontend
        run: |
          npm run build
        working-directory: ${{ matrix.pluginDir }}

      - name: Check for backend Mage file
        id: backend-check
        run: |
          echo "MAGEFILE_EXISTS=$(test -f ./Magefile.go && echo true || echo false)" >> $GITHUB_OUTPUT
        working-directory: ${{ matrix.pluginDir }}

      - uses: actions/setup-go@v5
        with:
          go-version: "~1.21"
          check-latest: true
          cache-dependency-path: ${{ matrix.pluginDir }}/go.sum
        if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'

      - name: Test plugin backend
        uses: magefile/mage-action@v3
        with:
          version: latest
          args: -v test
          workdir: ${{ matrix.pluginDir }}
        if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'

      - name: Build plugin backend
        uses: magefile/mage-action@v3
        with:
          version: latest
          args: -v build:linux
          workdir: ${{ matrix.pluginDir }}
        if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'

      ## CANARY_VERSION has to use NPM as we only publish the packages on changes
      - name: Set environment vars for testing
        run: |
          echo "PLUGIN_ID=$(cat src/plugin.json | jq -r '.id')" >> $GITHUB_ENV
          echo "EXPECTED_GRAFANA_VERSION=$(npx semver@latest $(cat src/plugin.json | jq -r '.dependencies.grafanaDependency') -c)" >> $GITHUB_ENV
          echo "CANARY_VERSION=${{ needs.setup-matrix.outputs.canaryVersion }}" >> $GITHUB_ENV
          echo "CANARY_DOCKER_TAG=${{ needs.setup-matrix.outputs.canaryDockerTag }}" >> $GITHUB_ENV
          echo "LATEST_STABLE_VERSION=${{ needs.setup-matrix.outputs.latestVersion }}" >> $GITHUB_ENV
          if [ -f "${PWD}/.env" ]; then
            echo "ENV_FILE_OPTION=--env-file ${PWD}/.env" >> $GITHUB_ENV
          fi
        working-directory: ${{ matrix.pluginDir }}

      - name: Has Integration Tests
        id: has-integration-tests
        continue-on-error: true
        run: |
          echo "DIR=$($(test -f ./playwright.config.ts)  && echo "true" || echo "false")" >> $GITHUB_OUTPUT
        working-directory: ${{ matrix.pluginDir }}

      ## If we're creating examples for new features that are only found in canary versions the grafanaDependency
      ## will be greater than any stable release of Grafana and so will not be available on docker.
      - name: Should run tests against expected and latest
        id: should-run-expected-latest-tests
        continue-on-error: true
        run: |
          npm exec --package semver-compare-cli --call 'semver-compare $LATEST_STABLE_VERSION ge $EXPECTED_GRAFANA_VERSION'
        working-directory: ${{ matrix.pluginDir }}

      - name: Check if mockserver is defined
        id: mockserver-check
        run: |
          echo $(docker compose config --services)
          if docker compose config --services | grep -wq mockserver; then
            echo "MOCKSERVER_DEFINED=true" >> $GITHUB_OUTPUT
          else
            echo "MOCKSERVER_DEFINED=false" >> $GITHUB_OUTPUT
          fi
        working-directory: ${{ matrix.pluginDir }}

      - name: Run mock server
        if: steps.mockserver-check.outputs.MOCKSERVER_DEFINED == 'true'
        run: docker compose up -d mockserver
        working-directory: ${{ matrix.pluginDir }}

      - name: Setup playwright browser
        if: steps.has-integration-tests.outputs.DIR == 'true'
        run: npx playwright install --with-deps chromium
        working-directory: ${{ matrix.pluginDir }}

      - name: Expected version - Start Grafana
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        run: |
          docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana:$EXPECTED_GRAFANA_VERSION
        working-directory: ${{ matrix.pluginDir }}

      - name: Expected - Wait for Grafana to start
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        uses: nev7n/wait_for_response@v1
        with:
          url: "http://localhost:3000/"
          responseCode: 200
          timeout: 60000
          interval: 500

      - name: Expected - Run Playwright tests
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        id: expected-version-tests
        continue-on-error: true
        run: npm run e2e --prefix ${{ matrix.pluginDir }}

      - name: Expected - Publish report to GCS
        if: ${{ always() && steps.expected-version-tests.outcome == 'failure' }}
        uses: grafana/plugin-actions/publish-report@main
        with:
          grafana-version: ${{ env.EXPECTED_GRAFANA_VERSION }}
          path: ${{ matrix.pluginDir }}/playwright-report/

      - name: Expected - Failing build due to test failures
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.expected-version-tests.outcome != 'success'  && steps.should-run-expected-latest-tests.outcome == 'success'
        run: exit 1

      - name: Expected - Stop and remove Docker container
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        run: |
          docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
        working-directory: ${{ matrix.pluginDir }}

      # Latest Version Tests
      # Runs the plugin tests against the latest stable version of Grafana.
      # Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests.
      - name: Latest - Start Grafana
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        run: |
          docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana:latest
        working-directory: ${{ matrix.pluginDir }}

      - name: Latest - Wait for Grafana to start
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        uses: nev7n/wait_for_response@v1
        with:
          url: "http://localhost:3000/"
          responseCode: 200
          timeout: 60000
          interval: 500

      - name: Latest - Run Playwright tests
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        id: latest-version-tests
        continue-on-error: true
        run: npm run e2e --prefix ${{ matrix.pluginDir }}

      - name: Latest - Publish report to GCS
        if: ${{ always() && steps.latest-version-tests.outcome == 'failure' }}
        uses: grafana/plugin-actions/publish-report@main
        with:
          grafana-version: ${{ env.LATEST_STABLE_VERSION }}
          path: ${{ matrix.pluginDir }}/playwright-report/

      - name: Latest - Failing build due to test failures (latest version of Grafana)
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.latest-version-tests.outcome != 'success' && steps.should-run-expected-latest-tests.outcome == 'success'
        run: exit 1

      - name: Latest - Stop and remove Docker container
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success'
        run: |
          docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
        working-directory: ${{ matrix.pluginDir }}

      ## Canary Version Tests
      ## Runs the plugin tests against the latest build of Grafana main branch.

      - name: Canary - Upgrade @grafana packages using canary_version
        run: |
          npm install --legacy-peer-deps $(echo $(cat package.json | jq -r --arg version $CANARY_VERSION '["@grafana/eslint-config", "@grafana/tsconfig", "@grafana/scenes", "@grafana/experimental", "@grafana/plugin-e2e", "@grafana/plugin-meta-extractor"] as $blacklist | [(.devDependencies,.dependencies) | keys] | flatten | unique | map(select ( test("^@grafana") ) ) as $deps | $deps - $blacklist | map("\(.)@\($version)") | join(" ")') )
          npm install --force --save-optional @swc/core  @swc/core-linux-arm-gnueabihf @swc/core-linux-arm64-gnu @swc/core-linux-arm64-musl @swc/core-linux-x64-gnu @swc/core-linux-x64-musl
        working-directory: ${{ matrix.pluginDir }}

      - name: Canary - Run frontend tests
        run: |
          npm run test:ci
        working-directory: ${{ matrix.pluginDir }}

      - name: Canary - Build plugin with grafana dependencies
        run: |
          npm run build
        working-directory: ${{ matrix.pluginDir }}

      - name: Canary - Build plugin backend
        uses: magefile/mage-action@v3
        with:
          version: latest
          args: -v build:linux
          workdir: ${{ matrix.pluginDir }}
        if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true'

      # Canary versions live at grafana/grafana-dev
      - name: Canary - Start Grafana dev image
        if: steps.has-integration-tests.outputs.DIR == 'true'
        run: |
          docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana-dev:$CANARY_DOCKER_TAG
        working-directory: ${{ matrix.pluginDir }}

      - name: Canary - Wait for Grafana to start
        if: steps.has-integration-tests.outputs.DIR == 'true'
        uses: nev7n/wait_for_response@v1
        with:
          url: "http://localhost:3000/"
          responseCode: 200
          timeout: 60000
          interval: 500

      - name: Canary - Run Playwright tests
        if: steps.has-integration-tests.outputs.DIR == 'true'
        id: canary-version-tests
        continue-on-error: true
        run: npm run e2e --prefix ${{ matrix.pluginDir }}

      - name: Canary - Publish report to GCS
        if: ${{ always() && steps.canary-version-tests.outcome == 'failure' }}
        uses: grafana/plugin-actions/publish-report@main
        with:
          grafana-version: ${{ env.CANARY_DOCKER_TAG }}
          path: ${{ matrix.pluginDir }}/playwright-report/

      - name: Canary - Failing build due to test failures (canary versions)
        if: steps.has-integration-tests.outputs.DIR == 'true' && steps.canary-version-tests.outcome != 'success'
        run: exit 1

      - name: Canary - Stop and remove Docker container
        if: steps.has-integration-tests.outputs.DIR == 'true'
        run: |
          docker stop $PLUGIN_ID && docker rm $PLUGIN_ID
        working-directory: ${{ matrix.pluginDir }}

      - name: Stop mockserver if running
        if: steps.mockserver-check.outputs.MOCKSERVER_DEFINED == 'true'
        run: docker compose down mockserver
        working-directory: ${{ matrix.pluginDir }}

  notify:
    if: ${{ (always() && github.event_name == 'schedule') }}
    runs-on: ubuntu-latest
    needs: [run-integration-tests]
    steps:
      - name: Send GitHub Action trigger data to Slack workflow
        id: slack
        uses: slackapi/slack-github-action@v1.27.0
        if: contains(fromJson('["failure"]'), needs.run-integration-tests.result)
        with:
          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
          payload: |
            {
              "blocks": [
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": ":blob-sad: <${{ github.server_url }}/${{ github.repository }}|Plugin Examples> Integration Test `${{ needs.run-integration-tests.result }}`"
                      }
                    },
                    {
                      "type": "actions",
                      "elements": [
                        {
                          "type": "button",
                          "text": {
                            "type": "plain_text",
                            "text": ":octocat: Open Job",
                            "emoji": true
                          },
                          "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}",
                          "action_id": "actionId-0"
                        }
                      ]
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "context",
                      "elements": [
                        {
                          "type": "mrkdwn",
                          "text": "*Commit*: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Event*: ${{ github.event_name }}"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Branch*: `${{ github.head_ref || github.ref_name }}`"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*PR Number*: `${{ github.event.pull_request.number || 'Not triggered by a PR' }}`"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github. run_id }}?pr=${{ github.event.number }}|See Playwright reports>"
                        }
                      ]
                    }
                ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK