Skip to content

E2E matrix

E2E matrix #228

Workflow file for this run

name: E2E matrix
on:
schedule:
- cron: "0 5 * * *"
workflow_dispatch:
inputs:
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
env:
CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress
permissions: { }
jobs:
preinstall:
if: ${{ github.repository_owner == 'nrwl' }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
node_version:
- 18
- 20
- 22
# - 23
exclude:
# run just node v20 on macos
- os: macos-latest
node_version: 18
- os: macos-latest
node_version: 22
# - os: macos-latest
# node_version: 23
name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
name: Install pnpm
with:
version: 9.8.0
run_install: false
- name: Set node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node_version }}
cache: 'pnpm'
- name: Cache node_modules
id: cache-modules
uses: actions/cache@v4
with:
lookup-only: true
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ hashFiles('pnpm-lock.yaml') }}
- name: Ensure Python setuptools Installed on Macos
if: ${{ matrix.os == 'macos-latest' }}
id: brew-install-python-setuptools
run: brew install python-setuptools
- name: Install packages
if: steps.cache-modules.outputs.cache-hit != 'true'
run: pnpm install --frozen-lockfile
- name: Homebrew cache directory path
if: ${{ matrix.os == 'macos-latest' }}
id: homebrew-cache-dir-path
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
- name: Cache Homebrew
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache@v4
with:
lookup-only: true
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
key: brew-${{ matrix.node_version }}
restore-keys: |
brew-
- name: Cache Cypress
id: cache-cypress
uses: actions/cache@v4
with:
lookup-only: true
path: '${{ github.workspace }}/.cypress'
key: ${{ runner.os }}-cypress
- name: Install Cypress
if: steps.cache-cypress.outputs.cache-hit != 'true'
run: npx cypress install
e2e:
if: ${{ github.repository_owner == 'nrwl' }}
needs: preinstall
permissions:
contents: read
runs-on: ${{ matrix.os }}
timeout-minutes: 90
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
node_version:
- 18
- 20
- 22
# - 23
package_manager:
- npm
- yarn
- pnpm
project:
- e2e-angular
- e2e-cypress
- e2e-detox
- e2e-esbuild
- e2e-eslint
- e2e-expo
- e2e-gradle
- e2e-jest
- e2e-js
- e2e-lerna-smoke-tests
- e2e-next
- e2e-node
- e2e-nuxt
- e2e-nx-init
- e2e-nx
- e2e-playwright
- e2e-plugin
- e2e-react
- e2e-react-native
- e2e-release
- e2e-remix
- e2e-rollup
- e2e-storybook
- e2e-vite
- e2e-vue
- e2e-web
- e2e-webpack
- e2e-workspace-create
include:
# os short names
- os: ubuntu-latest
os_name: 'Linux'
- os: macos-latest
os_name: 'MacOS'
# test timeouts
- os: ubuntu-latest
os_timeout: 60
- os: macos-latest
os_timeout: 90
# codeowner groups
- project: e2e-angular
codeowners: 'S04SS457V38'
- project: e2e-cypress
codeowners: 'S04T16BTJJY'
- project: e2e-detox
codeowners: 'S04TNCNJG5N'
- project: e2e-esbuild
codeowners: 'S04SJ6HHP0X'
- project: e2e-expo
codeowners: 'S04TNCNJG5N'
- project: e2e-gradle
codeowners: 'S04TNCNJG5N'
- project: e2e-jest
codeowners: 'S04T16BTJJY'
- project: e2e-js
codeowners: 'S04SJ6HHP0X'
- project: e2e-lerna-smoke-tests
codeowners: 'S04TNCVEETS'
- project: e2e-eslint
codeowners: 'S04SYJGKSCT'
- project: e2e-next
codeowners: 'S04TNCNJG5N'
- project: e2e-node
codeowners: 'S04SJ6HHP0X'
- project: e2e-nx-init
codeowners: 'S04SYHYKGNP'
- project: e2e-nx
codeowners: 'S04SYHYKGNP'
- project: e2e-plugin
codeowners: 'S04SYHYKGNP'
- project: e2e-release
codeowners: 'S04SYHYKGNP'
- project: e2e-react
codeowners: 'S04TNCNJG5N'
- project: e2e-react-native
codeowners: 'S04TNCNJG5N'
- project: e2e-web
codeowners: 'S04SJ6PL98X'
- project: e2e-rollup
codeowners: 'S04SJ6PL98X'
- project: e2e-storybook
codeowners: 'S04SVQ8H0G5'
- project: e2e-playwright
codeowners: 'S04SVQ8H0G5'
- project: e2e-remix
codeowners: 'S04SVQ8H0G5'
- project: e2e-vite
codeowners: 'S04SJ6PL98X'
- project: e2e-vue
codeowners: 'S04SJ6PL98X'
- project: e2e-nuxt
codeowners: 'S04SJ6PL98X'
- project: e2e-webpack
codeowners: 'S04SJ6PL98X'
- project: e2e-workspace-create
codeowners: 'S04SYHYKGNP'
exclude:
# exclude react-native tests from ubuntu
- os: ubuntu-latest
project: e2e-react-native
- os: ubuntu-latest
project: e2e-detox
- os: ubuntu-latest
project: e2e-expo
# exclude non-CNW/Lerna tests from non-LTS node versions
- node_version: 18
project: e2e-angular
- node_version: 18
project: e2e-cypress
- node_version: 18
project: e2e-detox
- node_version: 18
project: e2e-esbuild
- node_version: 18
project: e2e-expo
- node_version: 18
project: e2e-gradle
- node_version: 18
project: e2e-jest
- node_version: 18
project: e2e-js
- node_version: 18
project: e2e-eslint
- node_version: 18
project: e2e-next
- node_version: 18
project: e2e-node
- node_version: 18
project: e2e-nuxt
- node_version: 18
project: e2e-nx-init
- node_version: 18
project: e2e-nx
- node_version: 18
project: e2e-plugin
- node_version: 18
project: e2e-playwright
- node_version: 18
project: e2e-react
- node_version: 18
project: e2e-react-native
- node_version: 18
project: e2e-web
- node_version: 18
project: e2e-remix
- node_version: 18
project: e2e-rollup
- node_version: 18
project: e2e-storybook
- node_version: 18
project: e2e-vite
- node_version: 18
project: e2e-vue
- node_version: 18
project: e2e-webpack
- node_version: 22
project: e2e-angular
- node_version: 22
project: e2e-cypress
- node_version: 22
project: e2e-detox
- node_version: 22
project: e2e-esbuild
- node_version: 22
project: e2e-expo
- node_version: 22
project: e2e-gradle
- node_version: 22
project: e2e-jest
- node_version: 22
project: e2e-js
- node_version: 22
project: e2e-eslint
- node_version: 22
project: e2e-next
- node_version: 22
project: e2e-node
- node_version: 22
project: e2e-nuxt
- node_version: 22
project: e2e-nx-init
- node_version: 22
project: e2e-nx
- node_version: 22
project: e2e-plugin
- node_version: 22
project: e2e-playwright
- node_version: 22
project: e2e-react
- node_version: 22
project: e2e-react-native
- node_version: 22
project: e2e-web
- node_version: 22
project: e2e-remix
- node_version: 22
project: e2e-rollup
- node_version: 22
project: e2e-storybook
- node_version: 22
project: e2e-vite
- node_version: 22
project: e2e-vue
- node_version: 22
project: e2e-webpack
# - node_version: 23
# project: e2e-angular
# - node_version: 23
# project: e2e-cypress
# - node_version: 23
# project: e2e-detox
# - node_version: 23
# project: e2e-esbuild
# - node_version: 23
# project: e2e-expo
# - node_version: 23
# project: e2e-gradle
# - node_version: 23
# project: e2e-jest
# - node_version: 23
# project: e2e-js
# - node_version: 23
# project: e2e-eslint
# - node_version: 23
# project: e2e-next
# - node_version: 23
# project: e2e-node
# - node_version: 23
# project: e2e-nuxt
# - node_version: 23
# project: e2e-nx-init
# - node_version: 23
# project: e2e-nx
# - node_version: 23
# project: e2e-plugin
# - node_version: 23
# project: e2e-playwright
# - node_version: 23
# project: e2e-react
# - node_version: 23
# project: e2e-react-native
# - node_version: 23
# project: e2e-web
# - node_version: 23
# project: e2e-remix
# - node_version: 23
# project: e2e-rollup
# - node_version: 23
# project: e2e-storybook
# - node_version: 23
# project: e2e-vite
# - node_version: 23
# project: e2e-vue
# - node_version: 23
# project: e2e-webpack
# run just npm v20 on macos
- os: macos-latest
package_manager: yarn
- os: macos-latest
package_manager: pnpm
- os: macos-latest
node_version: 18
- os: macos-latest
node_version: 22
# - os: macos-latest
# node_version: 23
fail-fast: false
name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Prepare dir for output
run: mkdir -p outputs
- uses: pnpm/action-setup@v4
name: Install pnpm
with:
version: 9.8.0
run_install: false
- name: Use Node.js ${{ matrix.node_version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node_version }}
cache: 'pnpm'
- name: Cache node_modules
id: cache-modules
uses: actions/cache@v4
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ hashFiles('pnpm-lock.yaml') }}
- name: Install packages
run: pnpm install --frozen-lockfile
- name: Cleanup
if: ${{ matrix.os == 'ubuntu-latest' }}
run: |
# Workaround to provide additional free space for testing.
# https://github.com/actions/virtual-environments/issues/2840
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo apt-get install lsof
echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
- name: Homebrew cache directory path
if: ${{ matrix.os == 'macos-latest' }}
id: homebrew-cache-dir-path
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
- name: Cache Homebrew
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache@v4
with:
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
key: brew-${{ matrix.node_version }}
restore-keys: |
brew-
- name: Cache Cypress
id: cache-cypress
uses: actions/cache@v4
with:
path: '${{ github.workspace }}/.cypress'
key: ${{ runner.os }}-cypress
- name: Install Cypress
if: steps.cache-cypress.outputs.cache-hit != 'true'
run: npx cypress install
- name: Install applesimutils, reset ios simulators
if: ${{ matrix.os == 'macos-latest' }}
run: |
HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null
xcrun simctl shutdown all && xcrun simctl erase all
- name: Configure git metadata (needed for lerna smoke tests)
run: |
git config --global user.email [email protected]
git config --global user.name "Test Test"
- name: Set starting timestamp
id: before-e2e
run: |
echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
- name: Run e2e tests
id: e2e-run
run: pnpm nx run-many -t e2e-local -p ${{ matrix.project }}
timeout-minutes: ${{ matrix.os_timeout }}
env:
GIT_AUTHOR_EMAIL: [email protected]
GIT_AUTHOR_NAME: Test
GIT_COMMITTER_EMAIL: [email protected]
GIT_COMMITTER_NAME: Test
NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
NODE_OPTIONS: --max_old_space_size=8192
SELECTED_PM: ${{ matrix.package_manager }}
npm_config_registry: http://localhost:4872
YARN_REGISTRY: http://localhost:4872
NX_CACHE_DIRECTORY: 'tmp'
NX_E2E_SKIP_CLEANUP: 'true'
NX_E2E_RUN_E2E: 'true'
NX_E2E_VERBOSE_LOGGING: 'true'
NX_PERF_LOGGING: 'false'
NX_DAEMON: 'true'
NX_SKIP_LOG_GROUPING: 'true'
- name: Save matrix config in file
if: ${{ always() }}
id: save-matrix
shell: bash
run: |
before=${{ steps.before-e2e.outputs.timestamp }}
now=$(date +%s)
delta=$(($now - $before))
matrix=$((
echo '${{ toJSON(matrix) }}'
) | jq --argjson delta $delta -c '. + { "status": "${{ steps.e2e-run.outcome}}", "duration": $delta }')
echo "$matrix" > matrix
path=outputs/${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
echo "path=$path" >> $GITHUB_OUTPUT
echo "$matrix" > $path
- name: Upload matrix config
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: outputs
overwrite: true
if-no-files-found: 'ignore'
path: ${{ steps.save-matrix.outputs.path }}
- name: Setup tmate session
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
uses: mxschmitt/[email protected]
timeout-minutes: 15
process-result:
if: ${{ always() && github.repository_owner == 'nrwl' }}
runs-on: ubuntu-latest
needs: e2e
outputs:
message: ${{ steps.process-json.outputs.SLACK_MESSAGE }}
proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }}
pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }}
codeowners: ${{ steps.process-json.outputs.CODEOWNERS }}
steps:
- name: Load outputs
uses: actions/download-artifact@v4
with:
name: outputs
path: outputs
- name: Join and stringify matrix configs
id: combine-json
run: |
combined=$((jq -s . outputs/*) | jq tostring)
echo "combined=$combined" >> $GITHUB_OUTPUT
- name: Make slack outputs
id: process-json
uses: actions/github-script@v7
env:
GITHUB_TOKEN: ${{ github.token }}
with:
script: |
const combined = JSON.parse(${{ steps.combine-json.outputs.combined }});
const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project));
// codeowners
const codeowners = new Set();
failedProjects.forEach(c => {
codeowners.add(c.codeowners);
});
core.setOutput('CODEOWNERS', Array.from(codeowners).join(','));
function trimSpace(res) {
return res.split('\n').map((l) => l.trim()).join('\n');
}
// failed message
let lastProject;
let result = `
\`\`\`
| Failed project | PM | OS | Node |
|--------------------------------|------|-------|------|`;
failedProjects.forEach(matrix => {
const project = matrix.project !== lastProject ? matrix.project : '...';
result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version} |`
lastProject = matrix.project;
});
result += `\`\`\``;
core.setOutput('SLACK_MESSAGE', trimSpace(result));
function humanizeDuration(num) {
let res = '';
const hours = Math.floor(num / 3600);
if (hours) {
res += `${hours}h `;
}
const mins = Math.floor((num % 3600) / 60);
if (mins) {
res += `${mins}m `;
}
const sec = num % 60;
if (sec) {
res += `${sec}s`
}
return res;
}
// duration message
const timeReport = {};
const pmReport = {
npm: 0,
yarn: 0,
pnpm: 0
};
const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native'];
combined.forEach((matrix) => {
if (matrix.os_name === 'Linux' && matrix.node_version === 18) {
pmReport[matrix.package_manager] += matrix.duration;
}
if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) {
if (timeReport[matrix.project]) {
if (matrix.duration > timeReport[matrix.project].max) {
timeReport[matrix.project].max = matrix.duration;
timeReport[
matrix.project
].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`;
}
if (matrix.duration < timeReport[matrix.project].min) {
timeReport[matrix.project].min = matrix.duration;
timeReport[
matrix.project
].minEnv = `${matrix.os_name}, ${matrix.package_manager}`;
}
} else {
timeReport[matrix.project] = {
min: matrix.duration,
max: matrix.duration,
minEnv: `${matrix.os_name}, ${matrix.package_manager}`,
maxEnv: `${matrix.os_name}, ${matrix.package_manager}`,
};
}
}
});
// project time report
let resultPkg = `
\`\`\`
| Project | Time |
|--------------------------------|---------------------------|`;
function mapProjectTime(proj, section) {
let res = '';
res += `${humanizeDuration(timeReport[proj][section])}`;
res += ` (${timeReport[proj][section + 'Env']})`
return res;
}
function durationIcon(proj, section) {
if (timeReport[proj][section] < 12 * 60) {
return `${section} ✅`;
}
if (timeReport[proj][section] < 15 * 60) {
return `${section} ❗`;
}
return `${section} ❌`;
}
Object.keys(timeReport).forEach(proj => {
resultPkg += `\n| ${proj.padEnd(30)} | |`;
resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`;
resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`;
});
resultPkg += `\`\`\``;
core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg));
// Print project duration report inline to allow reviewing on manual runs (when no slack message will be sent)
console.log(trimSpace(resultPkg));
let resultPm = `
\`\`\`
| PM | Total time |
|------|-------------|`;
Object.keys(pmReport).forEach(pm => {
resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |`
});
resultPm += `\`\`\``;
core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm));
// Print package manager duration report inline to allow reviewing on manual runs (when no slack message will be sent)
console.log(trimSpace(resultPm));
report-failure:
if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
needs: process-result
runs-on: ubuntu-latest
name: Report failure
steps:
- name: Send notification
uses: ravsamhq/notify-slack-action@v2
with:
status: 'failure'
message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}'
notification_title: '{workflow}'
footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
mention_groups: ${{ needs.process-result.outputs.codeowners }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
report-success:
if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
needs: e2e
runs-on: ubuntu-latest
name: Report status
steps:
- name: Send notification
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ needs.e2e.result }}
message_format: '{emoji} Workflow has {status_message}'
notification_title: '{workflow}'
footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
report-pm-time:
if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
needs: process-result
runs-on: ubuntu-latest
name: Report duration per package manager
steps:
- name: Send notification
uses: ravsamhq/notify-slack-action@v2
with:
status: 'skipped'
message_format: '${{ needs.process-result.outputs.pm-duration }}'
notification_title: 'Total duration per package manager (ubuntu only)'
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
report-proj-time:
if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
needs: process-result
runs-on: ubuntu-latest
name: Report duration per package manager
steps:
- name: Send notification
uses: ravsamhq/notify-slack-action@v2
with:
status: 'skipped'
message_format: '${{ needs.process-result.outputs.proj-duration }}'
notification_title: 'E2E Project duration stats'
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}