From 0f484d714d7640aff2ed00755b45da27f63deddf Mon Sep 17 00:00:00 2001 From: Tiago Bento Date: Sun, 28 Apr 2024 03:03:41 -0400 Subject: [PATCH] Test --- .github/workflows/ci_build.yml | 76 ++---- .github/workflows/ci_build_swf_stack.yml | 168 ------------- package.json | 4 +- pnpm-lock.yaml | 148 +++++++++++ repo/graph.dot | 2 + repo/graph.json | 3 + scripts/build-partitioning/package.json | 20 ++ .../src/build_partitioning.ts | 237 ++++++++++++++++++ scripts/build-partitioning/src/pnpm_filter.ts | 55 ++++ scripts/build-partitioning/tsconfig.json | 21 ++ 10 files changed, 510 insertions(+), 224 deletions(-) delete mode 100644 .github/workflows/ci_build_swf_stack.yml create mode 100644 scripts/build-partitioning/package.json create mode 100644 scripts/build-partitioning/src/build_partitioning.ts create mode 100644 scripts/build-partitioning/src/pnpm_filter.ts create mode 100644 scripts/build-partitioning/tsconfig.json diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml index 01f346e2f19..3a19e4e737b 100644 --- a/.github/workflows/ci_build.yml +++ b/.github/workflows/ci_build.yml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-13, windows-latest] + partition: [0, 1] runs-on: ${{ matrix.os }} steps: - name: "Support longpaths" @@ -40,47 +41,31 @@ jobs: id: ci_patterns uses: ./.github/actions/setup-ci-patterns + - name: "Setup environment" + uses: ./.github/actions/setup-env + - name: "Setup build mode {none,full,partial}" id: setup_build_mode shell: bash run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export CHANGED_SOURCE_PATHS_IN_ROOT=($(printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" | grep -v -e "^packages" -e "^examples")) - echo "Changed source paths in root:" - echo ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_ROOT[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: Build` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} -eq 0 ]; then - echo 'No source files changed in root; `CI :: Build` (partial) will run.' - echo "mode=partial" >> $GITHUB_OUTPUT - else - echo 'Source files changed in root; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" - - - name: "Setup environment" + pnpm bootstrap:root + bun scripts/build-partitioning/src/build_partitioning.ts /tmp/partitions.json ${{ !github.event.pull_request }} + echo "mode=$(jq --raw-output '.[${{ matrix.partition }}].mode' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "bootstrapPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].bootstrapPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "fullBuildPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].fullBuildPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "upstreamPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].upstreamPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "selfAndDownstreamPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].selfAndDownstreamPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "Done." + + - name: "Bootstrap" if: steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/setup-env - - - name: "FULL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' env: PLAYWRIGHT_BASE__installDeps: "true" uses: ./.github/actions/bootstrap + with: + pnpm_filter_string: ${{ steps.setup_build_mode.outputs.bootstrapPnpmFilterString }} - - name: "FULL → Build (without SWF Stack)" + - name: "FULL → Build" if: steps.setup_build_mode.outputs.mode == 'full' env: WEBPACK__minimize: "false" @@ -96,35 +81,18 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: >- - pnpm - -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='!@kie-tools/kn-plugin-workflow...' - -F='!@kie-tools/serverless-logic-web-tools...' - -r --workspace-concurrency=1 build:prod - - - name: "PARTIAL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'partial' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F "...[${{ steps.checkout_pr.outputs.base_sha }}]..." -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' + eval "pnpm ${{ steps.setup_build_mode.outputs.fullBuildPnpmFilterString }} --workspace-concurrency=1 build:prod" - - name: "PARTIAL → Build dependencies" + - name: "PARTIAL → Build upstream" if: steps.setup_build_mode.outputs.mode == 'partial' shell: bash env: KIE_TOOLS_BUILD__buildContainerImages: ${{ runner.os != 'Windows' }} KIE_TOOLS_BUILD__buildExamples: "true" run: | - export ALL_DEPENDENCIES_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F=$(jq --raw-output .name package.json)^..."') - export CHANGED_PKGS_EXCLUSION_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F='"'"'!$(jq --raw-output .name package.json)'"'"'"') - echo $ALL_DEPENDENCIES_FILTER - echo $CHANGED_PKGS_EXCLUSION_FILTER - eval "pnpm $ALL_DEPENDENCIES_FILTER $CHANGED_PKGS_EXCLUSION_FILTER -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' build:dev" + eval "pnpm ${{ steps.setup_build_mode.outputs.upstreamPnpmFilterString }} build:dev" - - name: "PARTIAL → Build changed and dependents" + - name: "PARTIAL → Build changed and downstream" if: steps.setup_build_mode.outputs.mode == 'partial' env: WEBPACK__minimize: "false" @@ -137,7 +105,7 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: | - pnpm -F "...[${{ steps.checkout_pr.outputs.base_sha }}]" -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' --workspace-concurrency=1 build:prod + eval "pnpm ${{ steps.setup_build_mode.outputs.selfAndDownstreamPnpmFilterString }} --workspace-concurrency=1 build:prod" - name: "Check tests result (`main` only)" if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' diff --git a/.github/workflows/ci_build_swf_stack.yml b/.github/workflows/ci_build_swf_stack.yml deleted file mode 100644 index 07962dfd11a..00000000000 --- a/.github/workflows/ci_build_swf_stack.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: "CI :: Build (SWF Stack)" - -on: - push: - branches: [main] - pull_request: - branches: ["**"] - types: [opened, reopened, ready_for_review, synchronize] - -concurrency: - group: ${{ github.event.pull_request && format('ci-build-swf-stack-full-pr-{0}', github.event.pull_request.number) || format('ci-build-swf-stack-full-push-main-{0}', github.sha) }} - cancel-in-progress: true - -env: - TMPDIR: "/tmp" - -jobs: - run: - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - name: "Checkout @ GitHub default" - uses: actions/checkout@v3 - - - name: "Checkout @ Simulated squashed-merge if PR" - id: checkout_pr - uses: ./.github/actions/checkout-pr - with: - ref: ${{ github.base_ref }} - - - name: "Setup CI patterns" - id: ci_patterns - uses: ./.github/actions/setup-ci-patterns - - - name: "Setup build mode {none,full}" - id: setup_build_mode - shell: bash - run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export PKGS_IN_SWF_STACK=("@kie-tools/serverless-logic-web-tools-swf-builder-image" "@kie-tools/kn-plugin-workflow" "@kie-tools/serverless-logic-web-tools" "@kie-tools/serverless-logic-web-tools-base-builder-image") - - npm install -g graph-data-structure@2.0.0 - export GREP_RES=$(NODE_PATH=$(npm prefix -g)/lib/node_modules node scripts/sparse-checkout/list_packages_dependencies.js ./repo "${PKGS_IN_SWF_STACK[@]}" | tr -d '\n' | xargs -d ' ' -I{} echo -n " -e \"{}\"") - echo "GREP RES:" - echo $GREP_RES - npm uninstall -g graph-data-structure - - export CHANGED_SOURCE_PATHS_IN_SWF_STACK=($(eval "printf '%s\\n' "${CHANGED_SOURCE_PATHS[@]}" | grep $GREP_RES")) - echo "Changed source paths in "SWF Stack":" - echo ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} -eq 0 ]; then - echo 'No source files changed in "SWF Stack"; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - else - echo 'Source files changed in "SWF Stack"; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" - - - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/setup-env - - - name: "Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='@kie-tools/kn-plugin-workflow...' -F='@kie-tools/serverless-logic-web-tools...' - - - name: "Build (only SWF Stack)" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - WEBPACK__minimize: "false" - WEBPACK__tsLoaderTranspileOnly: "false" - KIE_TOOLS_BUILD__runLinters: "true" - KIE_TOOLS_BUILD__runTests: "true" - KIE_TOOLS_BUILD__runEndToEndTests: "true" - KIE_TOOLS_BUILD__buildContainerImages: "true" - KIE_TOOLS_BUILD__buildExamples: "true" - KIE_TOOLS_BUILD__ignoreTestFailures: ${{ !github.event.pull_request }} - KIE_TOOLS_BUILD__ignoreEndToEndTestFailures: ${{ !github.event.pull_request }} - DISPLAY: ":99.0" - START_SERVER_AND_TEST_INSECURE: "true" - NODE_OPTIONS: "--max_old_space_size=4096" - run: >- - pnpm - -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='@kie-tools/kn-plugin-workflow...' - -F='@kie-tools/serverless-logic-web-tools...' - --workspace-concurrency=1 build:prod - - - name: "Check tests result (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: actions/github-script@v6 - env: - KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS: |- - ${{ steps.ci_patterns.outputs.tests_reports_patterns }} - ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns }} - with: - result-encoding: string - script: | - const patterns = process.env["KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS"] - .split("\n") - .map(p => p.trim()) - .filter(p => p); - - const script = require("./scripts/check-junit-report-results/src/index.js"); - await script({ core, glob, patterns }); - - - name: "Check hanging uncommitted files (you should commit those!)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - shell: bash - run: | - git diff - [ "0" == "$(git diff | wc -l | tr -d ' ')" ] - - - name: "Upload reports and artifacts" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/upload-ci-reports-and-artifacts - - - name: "Upload end-to-end tests results to Buildkite (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' && !github.event.pull_request - shell: bash - env: - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} - BUILDKITE_BRANCH: ${{ github.ref_name }} - BUILDKITE_MESSAGE: ${{ github.event.commits[0].message }} - run: | - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" - echo "---------------------------- starting upload -----------------------" - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" | xargs -I{} curl -X POST \ - -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \ - -F "format=junit" \ - -F "data=@{}" \ - -F "run_env[CI]=github_actions" \ - -F "run_env[key]=$GITHUB_ACTION-$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" \ - -F "run_env[number]=$GITHUB_RUN_NUMBER" \ - -F "run_env[branch]=$BUILDKITE_BRANCH" \ - -F "run_env[commit_sha]=$GITHUB_SHA" \ - -F "run_env[message]=$BUILDKITE_MESSAGE" \ - -F "run_env[url]=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - https://analytics-api.buildkite.com/v1/uploads - - - name: "Print storage usage (after build)" - if: always() && !cancelled() - shell: bash - run: | - df -h . diff --git a/package.json b/package.json index bbf4be36980..047e538e181 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "devDependencies": { "@kie-tools-scripts/bootstrap": "workspace:*", "@kie-tools-scripts/build-env": "workspace:*", + "@kie-tools-scripts/build-partitioning": "workspace:*", "@kie-tools-scripts/check-junit-report-results": "workspace:*", "@kie-tools-scripts/run-script-if": "workspace:*", "@kie-tools-scripts/sparse-checkout": "workspace:*", @@ -32,8 +33,7 @@ "react-dropzone": "^11.4.2" }, "engines": { - "node": ">=18", - "pnpm": "8.7.0" + "node": ">=18" }, "packageManager": "pnpm@8.7.0", "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2a960fe165..66b49712401 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: "@kie-tools-scripts/build-env": specifier: workspace:* version: link:scripts/build-env + "@kie-tools-scripts/build-partitioning": + specifier: workspace:* + version: link:scripts/build-partitioning "@kie-tools-scripts/check-junit-report-results": specifier: workspace:* version: link:scripts/check-junit-report-results @@ -13266,6 +13269,27 @@ importers: specifier: ^4.6.2 version: 4.8.4 + scripts/build-partitioning: + devDependencies: + "@pnpm/filter-workspace-packages": + specifier: ^5.0.27 + version: 5.0.27(@pnpm/logger@4.0.0)(@yarnpkg/core@4.0.0-rc.50)(typanion@3.9.0) + "@pnpm/find-workspace-packages": + specifier: ^4.0.27 + version: 4.0.27(@pnpm/logger@4.0.0)(@yarnpkg/core@4.0.0-rc.50)(typanion@3.9.0) + "@types/bun": + specifier: ^1.1.0 + version: 1.1.0 + bun: + specifier: ^1.1.5 + version: 1.1.5 + graph-data-structure: + specifier: ^2.0.0 + version: 2.0.0 + typescript: + specifier: ^4.6.2 + version: 4.8.4 + scripts/check-junit-report-results: dependencies: fast-xml-parser: @@ -23727,6 +23751,78 @@ packages: engines: { node: ">=8.0" } dev: true + /@oven/bun-darwin-aarch64@1.1.5: + resolution: + { integrity: sha512-z0k3W2XEfa11OVUW0vp9pWLlpcPKY6TtpwPvREXCA0nFWj9LxbIRr5FZ1U6+M0gCRlx+5XOpSsWQ7+/HJcDgSg== } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64-baseline@1.1.5: + resolution: + { integrity: sha512-w3bHmbgrU0L8fCzW0uf0MYPo2DMekHG0w2SSQ+ZspkGQ11ppTu7FJZxZMAw8UQAAX3FgltFnaI1p6J9gkv4MMw== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64@1.1.5: + resolution: + { integrity: sha512-Pv1QDb69u6JMXfWiAE638aUWzsARGxWDQ1J1YLxOJVzbEe9f8qvMPZX7yP4OcfFMyc1ZcFrd7kDSkXQHZ/DeTg== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-aarch64@1.1.5: + resolution: + { integrity: sha512-iObapm21/EsuQN3Z3m/DwAyjeGQaYuwC+KocPT29n7Tv+C/bubIEtikrmSGfA2CiSA5n6dFsFGk/t/uFfUuVMA== } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64-baseline@1.1.5: + resolution: + { integrity: sha512-ENUbLzO/tKpzUp04e9u+jZi+RFIpIJ/RQy7WKTstzce3fZqAsfh65CJE7TqKvFpHEDNxaRqJRg7yUonSCO1BmA== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64@1.1.5: + resolution: + { integrity: sha512-KrvxD5sUf0WcdjVJm2LyGJqPzBldYWT4EQy+N5c5gzFnrEKSXOsj5lFE2vPNICS8Zvwzt8zLr0OwKq/WU93xTQ== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64-baseline@1.1.5: + resolution: + { integrity: sha512-OCkUFyhbLR0kzAzuZg7pAa48TQFukYlaHTPH7ZWJkuS4ewFsDiYE+ET1BFn0LoRPlK/IhR8F6S21cH1elrlWCw== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64@1.1.5: + resolution: + { integrity: sha512-kLRNRrQhCqXXoc0E6y9lBYbec6Sd7zIZtasoO6V7pYatMEaDB1ufIktmM5YVtCGYqNJ1+zNsENuU9xxPuprPDA== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@patternfly/patternfly@4.224.2: resolution: { integrity: sha512-HGNV26uyHSIECuhjPg/WGn0mXbAotcs6ODfhAOkfYjIgGylddgiwElxUe1rpEHV5mQJJ2rMn4OdeJIIpzRX61g== } @@ -28998,6 +29094,13 @@ packages: "@types/node": 18.17.18 dev: true + /@types/bun@1.1.0: + resolution: + { integrity: sha512-QGK0yU4jh0OK1A7DyhPkQuKjHQCC5jSJa3dpWIEhHv/rPfb6zLfdArc4/uUUZBMTcjilsafRXnPWO+1owb572Q== } + dependencies: + bun-types: 1.1.0 + dev: true + /@types/cacheable-request@6.0.1: resolution: { integrity: sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ== } @@ -29720,6 +29823,13 @@ packages: resolution: { integrity: sha512-/4QOuy3ZpV7Ya1GTRz5CYSz3DgkKpyUptXuQ5PPce7uuyJAOR7r9FhkmxJfvcNUXyklbC63a+YvB3jxy7s9ngw== } + /@types/node@20.11.30: + resolution: + { integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw== } + dependencies: + undici-types: 5.26.5 + dev: true + /@types/normalize-package-data@2.4.0: resolution: { integrity: sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== } @@ -30091,6 +30201,13 @@ packages: source-map: 0.6.1 dev: true + /@types/ws@8.5.10: + resolution: + { integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A== } + dependencies: + "@types/node": 18.17.18 + dev: true + /@types/ws@8.5.5: resolution: { integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg== } @@ -33340,6 +33457,32 @@ packages: semver: 7.5.4 dev: true + /bun-types@1.1.0: + resolution: + { integrity: sha512-GhMDD7TosdJzQPGUOcQD5PZshvXVxDfwGAZs2dq+eSaPsRn3iUCzvpFlsg7Q51bXVzLAUs+FWHlnmpgZ5UggIg== } + dependencies: + "@types/node": 20.11.30 + "@types/ws": 8.5.10 + dev: true + + /bun@1.1.5: + resolution: + { integrity: sha512-9gVjgPLGQqKEAcuTL1x/hn7PUYvWHlx/tm3//xIRkJk55IVWXcmFM6kQtV+Jn9k5x3wvtAd8a3RRyk2alyKPjA== } + cpu: [arm64, x64] + os: [darwin, linux, win32] + hasBin: true + requiresBuild: true + optionalDependencies: + "@oven/bun-darwin-aarch64": 1.1.5 + "@oven/bun-darwin-x64": 1.1.5 + "@oven/bun-darwin-x64-baseline": 1.1.5 + "@oven/bun-linux-aarch64": 1.1.5 + "@oven/bun-linux-x64": 1.1.5 + "@oven/bun-linux-x64-baseline": 1.1.5 + "@oven/bun-windows-x64": 1.1.5 + "@oven/bun-windows-x64-baseline": 1.1.5 + dev: true + /busboy@1.6.0: resolution: { integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== } @@ -51785,6 +51928,11 @@ packages: resolution: { integrity: sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g== } + /undici-types@5.26.5: + resolution: + { integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== } + dev: true + /undoo@0.5.0: resolution: { integrity: sha512-SPlDcde+AUHoFKeVlH2uBJxqVkw658I4WR2rPoygC1eRCzm3GeoP8S6xXZVJeBVOQQid8X2xUBW0N4tOvvHH3Q== } diff --git a/repo/graph.dot b/repo/graph.dot index 5d868fe0065..952fc74b963 100644 --- a/repo/graph.dot +++ b/repo/graph.dot @@ -4,6 +4,7 @@ digraph G { edge [ headport = "n", tailport = "s", arrowhead =dot, arrowsize =0.5 ]; "kie-tools-root" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie-tools-scripts/bootstrap" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; + "@kie-tools-scripts/build-partitioning" [ color = "blue", fontcolor = "blue", style = "rounded" ]; "@kie-tools-scripts/check-junit-report-results" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie-tools-scripts/run-script-if" [ color = "blue", fontcolor = "blue", style = "rounded" ]; "@kie-tools-scripts/sparse-checkout" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; @@ -184,6 +185,7 @@ digraph G { "yard-vscode-extension" [ color = "blue", fontcolor = "blue", style = "rounded" ]; "@kie-tools-scripts/build-env" [ color = "blue", fontcolor = "blue", style = "rounded" ]; "kie-tools-root" -> "@kie-tools-scripts/bootstrap" [ style = "dashed", color = "black" ]; + "kie-tools-root" -> "@kie-tools-scripts/build-partitioning" [ style = "dashed", color = "black" ]; "kie-tools-root" -> "@kie-tools-scripts/check-junit-report-results" [ style = "dashed", color = "black" ]; "kie-tools-root" -> "@kie-tools-scripts/run-script-if" [ style = "dashed", color = "black" ]; "kie-tools-root" -> "@kie-tools-scripts/sparse-checkout" [ style = "dashed", color = "black" ]; diff --git a/repo/graph.json b/repo/graph.json index 2d0b7a04720..86f337af90c 100644 --- a/repo/graph.json +++ b/repo/graph.json @@ -3,6 +3,7 @@ "nodes": [ { "id": "kie-tools-root" }, { "id": "@kie-tools-scripts/bootstrap" }, + { "id": "@kie-tools-scripts/build-partitioning" }, { "id": "@kie-tools-scripts/check-junit-report-results" }, { "id": "@kie-tools-scripts/run-script-if" }, { "id": "@kie-tools-scripts/sparse-checkout" }, @@ -185,6 +186,7 @@ ], "links": [ { "source": "kie-tools-root", "target": "@kie-tools-scripts/bootstrap", "weight": 1 }, + { "source": "kie-tools-root", "target": "@kie-tools-scripts/build-partitioning", "weight": 1 }, { "source": "kie-tools-root", "target": "@kie-tools-scripts/check-junit-report-results", "weight": 1 }, { "source": "kie-tools-root", "target": "@kie-tools-scripts/run-script-if", "weight": 1 }, { "source": "kie-tools-root", "target": "@kie-tools-scripts/sparse-checkout", "weight": 1 }, @@ -1375,6 +1377,7 @@ ["yard-vscode-extension", "packages/yard-vscode-extension"], ["@kie-tools-scripts/bootstrap", "scripts/bootstrap"], ["@kie-tools-scripts/build-env", "scripts/build-env"], + ["@kie-tools-scripts/build-partitioning", "scripts/build-partitioning"], ["@kie-tools-scripts/check-junit-report-results", "scripts/check-junit-report-results"], ["@kie-tools-scripts/run-script-if", "scripts/run-script-if"], ["@kie-tools-scripts/sparse-checkout", "scripts/sparse-checkout"], diff --git a/scripts/build-partitioning/package.json b/scripts/build-partitioning/package.json new file mode 100644 index 00000000000..406f63fad23 --- /dev/null +++ b/scripts/build-partitioning/package.json @@ -0,0 +1,20 @@ +{ + "name": "@kie-tools-scripts/build-partitioning", + "version": "0.0.0", + "license": "Apache-2.0", + "keywords": [], + "files": [ + "bin.js", + "dist", + "src" + ], + "scripts": {}, + "devDependencies": { + "@pnpm/filter-workspace-packages": "^5.0.27", + "@pnpm/find-workspace-packages": "^4.0.27", + "@types/bun": "^1.1.0", + "bun": "^1.1.5", + "graph-data-structure": "^2.0.0", + "typescript": "^4.6.2" + } +} \ No newline at end of file diff --git a/scripts/build-partitioning/src/build_partitioning.ts b/scripts/build-partitioning/src/build_partitioning.ts new file mode 100644 index 00000000000..dd7ff2d80d2 --- /dev/null +++ b/scripts/build-partitioning/src/build_partitioning.ts @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import DatavisTechGraph from "graph-data-structure"; +import { pnpmFilter } from "./pnpm_filter"; +import { $ } from "bun"; +import * as path from "path"; +import * as fs from "fs"; + +const ROOT_PKG_NAME = "kie-tools-root"; +const PKGS_DIRS = ["packages", "examples"]; + +type None = { + mode: "none"; + name: string; +}; + +type Full = { + mode: "full"; + name: string; + bootstrapPnpmFilterString: string; + fullBuildPnpmFilterString: string; +}; + +type Partial = { + mode: "partial"; + name: string; + bootstrapPnpmFilterString: string; + upstreamPnpmFilterString: string; + selfAndDownstreamPnpmFilterString: string; +}; + +const outputPath = process.argv[2]; +const forceFull = process.argv[3] === "true"; + +async function main(): Promise> { + if (!outputPath) { + console.error("[build-partitioning] Usage 'bun build_partitioning.js [output-path: str] [force-full: bool]'"); + process.exit(1); + } + + const graphJson = await import(path.resolve(path.join("repo", "graph.json"))); + const packagesLocationByName = new Map(graphJson.serializedPackagesLocationByName); + + const datavisGraph = DatavisTechGraph(); + datavisGraph.deserialize(graphJson.serializedDatavisGraph); + + const allLeafPackages = new Set( + datavisGraph + .nodes() + .filter((s) => datavisGraph.indegree(s) === 0) + .filter((s) => s !== ROOT_PKG_NAME) + ); + + console.log(`[build-partitioning] All leaf packages:`); + console.log(allLeafPackages); + + const p1 = new Set([ + "@kie-tools/serverless-logic-web-tools-swf-builder-image", + "@kie-tools/kn-plugin-workflow", + "@kie-tools/serverless-logic-web-tools", + "@kie-tools/serverless-logic-web-tools-base-builder-image", + "@kie-tools/serverless-logic-web-tools-swf-dev-mode-image", + "@kie-tools/dashbuilder-swf-monitoring-dashboard", + "@kie-tools/dashbuilder-viewer-image", + "chrome-extension-serverless-workflow-editor", + "vscode-extension-dashbuilder-editor", + "yard-vscode-extension", + "swf-vscode-extension", + ]); + console.log(`[build-partitioning] P1:`); + console.log(p1); + + const p2 = new Set([...allLeafPackages].filter((leaf) => !p1.has(leaf))); + console.log(`[build-partitioning] P2:`); + console.log(p2); + + const leafPackagesOverlap = [...allLeafPackages].filter((leaf) => p1.has(leaf) && p2.has(leaf)); + const hasPartitionOverlap = leafPackagesOverlap.length > 0; + + console.log(`[build-partitioning] Overlap check (${hasPartitionOverlap ? "❌" : "✅"}):`); + if (hasPartitionOverlap) { + console.error(`[build-partitioning] Partitions overlap. Aborting.`); + console.error(leafPackagesOverlap); + process.exit(1); + } + + const nonLeafPackagesInPartitions = new Set([...p1, ...p2].filter((l) => !allLeafPackages.has(l))); + console.log(`[build-partitioning] Leaf check (${nonLeafPackagesInPartitions.size > 0 ? "❌" : "✅"}):`); + if (nonLeafPackagesInPartitions.size > 0) { + console.error(`[build-partitioning] Non-leaf packages found in partition definitions. Aborting.`); + console.error(nonLeafPackagesInPartitions); + process.exit(1); + } + + const partitions = [ + { + name: "Partition 1", + leaves: p1, + dirs: new Set( + Object.keys(await pnpmFilter([...p1].map((pkg) => `-F ${pkg}...`).join(" "), { alwaysIncludeRoot: false })) + ), + }, + { + name: "Partition 2", + leaves: p2, + dirs: new Set( + Object.keys(await pnpmFilter([...p2].map((pkg) => `-F ${pkg}...`).join(" "), { alwaysIncludeRoot: false })) + ), + }, + ]; + + const redundancy = new Set( + [...packagesLocationByName.values()].filter( + (pkgDir) => partitions[0].dirs.has(pkgDir) && partitions[1].dirs.has(pkgDir) + ) + ); + console.log(`[build-partitioning] Redundancy:`); + console.log(redundancy); + + const allDeepDirs = new Set( + Object.keys(await pnpmFilter(`-F * -F !${ROOT_PKG_NAME}...`, { alwaysIncludeRoot: false })) + ); + + const sanityCheck = + partitions[0].dirs.size + partitions[1].dirs.size - (partitions.length - 1) * redundancy.size === allDeepDirs.size; + console.log(`[build-partitioning] Sanity check (${!sanityCheck ? "❌" : "✅"}):`); + if (!sanityCheck) { + console.error(`[build-partitioning] All packages count: ${allDeepDirs.size}`); + console.error(`[build-partitioning] P1 packages count: ${partitions[0].dirs.size}`); + console.error(`[build-partitioning] P2 packages count: ${partitions[1].dirs.size}`); + console.error(`[build-partitioning] Redundant packages count: ${redundancy.size}`); + process.exit(1); + } + + const base_sha = (await $`git rev-parse HEAD~1`.text()).trim(); // TODO: Extract parameter + const head_sha = (await $`git rev-parse HEAD~0`.text()).trim(); // TODO: Extract parameter + const non_source_files_patterns = ""; // TODO: Extract parameter + const changedSourcePaths = ( + await $`git diff --name-only ${base_sha} ${head_sha} -- ${non_source_files_patterns}`.text() + ) + .trim() + .split(/\s/); + + console.log("[build-partitioning] Changed source paths:"); + console.log(new Set(changedSourcePaths)); + + const changedSourcePathsInRoot = changedSourcePaths.filter((path) => + PKGS_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`)) + ); + + const relevantPackageDirsInAllPartitions = (await $`pnpm -F ...[${base_sha}]... exec pwd`.text()).trim().split(/\s/); + const affectedPackageDirsInAllPartitions = (await $`pnpm -F ...[${base_sha}] exec pwd`.text()).trim().split(/\s/); + + return await Promise.all( + partitions.map(async (partition) => { + if (forceFull || changedSourcePathsInRoot.length > 0) { + console.log(`[build-partitioning] 'Full' build of '${partition.name}'.`); + console.log( + `[build-partitioning] Building ${partition.dirs.size}/${partition.dirs.size}/${allDeepDirs.size} packages.` + ); + return { + mode: "full", + name: partition.name, + bootstrapPnpmFilterString: [...partition.leaves].map((l) => `-F '${l}...'`).join(" "), + fullBuildPnpmFilterString: [...partition.leaves].map((l) => `-F '${l}...'`).join(" "), + }; + } + + const changedSourcePathsInPartition = changedSourcePaths.filter((path) => + [...partition.dirs].some((partitionDir) => path.startsWith(partitionDir)) + ); + if (changedSourcePathsInPartition.length === 0) { + console.log(`[build-partitioning] 'None' build of '${partition.name}'.`); + console.log(`[build-partitioning] Building 0/${partition.dirs.size}/${allDeepDirs.size} packages.`); + return { + mode: "none", + name: partition.name, + }; + } + + const relevantPackageDirsInPartition = new Set( + relevantPackageDirsInAllPartitions.map((s) => path.relative(".", s)).filter((s) => partition.dirs.has(s)) + ); + const affectedPackageDirsInPartition = new Set( + affectedPackageDirsInAllPartitions.map((s) => path.relative(".", s)).filter((s) => partition.dirs.has(s)) + ); + + console.log(`[build-partitioning] 'Partial' build of '${partition.name}'`); + console.log( + `[build-partitioning] Building ${relevantPackageDirsInPartition.size}/${relevantPackageDirsInAllPartitions.length}/${allDeepDirs.size} packages.` + ); + console.log(relevantPackageDirsInPartition); + + return { + mode: "partial", + name: partition.name, + bootstrapPnpmFilterString: [...relevantPackageDirsInPartition].map((p) => `-F '${p}'`).join(" "), + upstreamPnpmFilterString: `${[...affectedPackageDirsInPartition].map((p) => `-F '${p}^...'`).join(" ")} ${[ + ...affectedPackageDirsInPartition, + ] + .map((p) => `-F '!${p}'`) + .join(" ")}`, + selfAndDownstreamPnpmFilterString: [...affectedPackageDirsInPartition].map((p) => `-F '...${p}'`).join(" "), + }; + }) + ); +} + +const partitions = await main(); +const partitionsJson = JSON.stringify(partitions, null, 2); +console.log(`[build-partitioning]`); +console.log(`[build-partitioning] --- PARTITIONS JSON ---`); +console.log(partitionsJson); + +const resolvedOutputPath = path.resolve(".", outputPath); +fs.writeFileSync(resolvedOutputPath, partitionsJson); +console.log(`[build-partitioning] --> Written to '${resolvedOutputPath}'`); +console.log(`[build-partitioning] Done.`); + +process.exit(0); diff --git a/scripts/build-partitioning/src/pnpm_filter.ts b/scripts/build-partitioning/src/pnpm_filter.ts new file mode 100644 index 00000000000..6e78e8dc829 --- /dev/null +++ b/scripts/build-partitioning/src/pnpm_filter.ts @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { filterPackages } from "@pnpm/filter-workspace-packages"; +import findWorkspacePackages from "@pnpm/find-workspace-packages"; + +const workspaceDir = "."; + +export async function pnpmFilter(pnpmFilterString: string, { alwaysIncludeRoot }: { alwaysIncludeRoot: boolean }) { + const packages = await findWorkspacePackages(workspaceDir); + + const parsedFilterString = !pnpmFilterString + ? [] + : pnpmFilterString + .split(" ") + .filter((f) => f !== "-F") + .filter((f) => f !== "--filter"); + + const filters = parsedFilterString.map((f) => ({ + filter: `${f}`, + followProdDepsOnly: false, + alwaysIncludeRoot, + })); + + const filteredPackages = (await filterPackages(packages, filters, { prefix: "", workspaceDir })) + .selectedProjectsGraph; + + if (!alwaysIncludeRoot) { + return filteredPackages; + } + + return { + [workspaceDir]: { + package: packages.filter((p) => p.dir === workspaceDir)[0], + dependencies: [], + }, + ...filteredPackages, + }; +} diff --git a/scripts/build-partitioning/tsconfig.json b/scripts/build-partitioning/tsconfig.json new file mode 100644 index 00000000000..c2ae4995d8c --- /dev/null +++ b/scripts/build-partitioning/tsconfig.json @@ -0,0 +1,21 @@ +{ + "exclude": ["node_modules"], + "compilerOptions": { + "outDir": "dist", + "target": "es5", + "lib": ["es6", "dom", "es2018.promise", "es2019", "esnext"], + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "sourceMap": true, + "noImplicitAny": true, + "noImplicitThis": true, + "removeComments": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "downlevelIteration": true, + "skipLibCheck": true, + "jsx": "react-jsx" + }, + "include": ["src"] +}