diff --git a/.github/actions/setup-ci-patterns/action.yml b/.github/actions/setup-ci-patterns/action.yml index 6261751e347..b3ddb660d2b 100644 --- a/.github/actions/setup-ci-patterns/action.yml +++ b/.github/actions/setup-ci-patterns/action.yml @@ -54,9 +54,9 @@ runs: const pnpmWorkspacePackagesRootPaths = ["packages", "examples"].map(p => path.join(cwd, p)).join("\n"); - const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/non-source-files-patterns.txt"), "utf-8"); + const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/non-source-files-patterns.txt"), "utf-8"); const nonSourceFilesPatternsForGitDiff = nonSourceFilesPatterns.split("\n").filter(p => p.trim() !== "").map(p => `':!${p}'`).join(" "); - const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/tests-source-files-patterns.txt"), "utf-8"); + const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/tests-source-files-patterns.txt"), "utf-8"); core.setOutput("pnpm_workspace_packages_root_paths", pnpmWorkspacePackagesRootPaths); core.setOutput("non_source_files_patterns", nonSourceFilesPatterns); @@ -78,14 +78,14 @@ runs: core.setOutput(outputName, patterns); } - await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/tests-reports-patterns.txt")); - await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt")); - await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt")); - await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/build-artifacts-patterns.txt")); + await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/tests-reports-patterns.txt")); + await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt")); + await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt")); + await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/build-artifacts-patterns.txt")); core.setOutput( "end_to_end_tests_reports_patterns_for_find", - prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt", "utf-8")) + prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt", "utf-8")) .split("\n") .map(p => `-path '${p}'`) .join(" -o ") diff --git a/.github/supporting-files/ci/build-partitioning/assertions.ts b/.github/supporting-files/ci/build-partitioning/assertions.ts new file mode 100644 index 00000000000..6cb40ceb22a --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/assertions.ts @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PartitionDefinition } from "./types"; + +export async function assertLeafPackagesInPartitionsExist({ + packageNames, + allLeafPackages, +}: { + packageNames: string[]; + allLeafPackages: Set; +}) { + const nonLeafPackagesInPartitions = new Set(packageNames.filter((l) => !allLeafPackages.has(l))); + console.log(`[build-partitioning] Leaf check (${nonLeafPackagesInPartitions.size > 0 ? "❌" : "✅"}):`); + if (nonLeafPackagesInPartitions.size > 0) { + console.error(`[build-partitioning] Non-leaf packages found in partition definitions. Aborting.`); + console.error(nonLeafPackagesInPartitions); + process.exit(1); + } +} + +export async function assertLeafPackagesInPartitionDefinitionsDontOverlap({ + allLeafPackages, + p0, + p1, +}: { + allLeafPackages: Set; + p0: Set; + p1: Set; +}) { + const leafPackagesOverlap = [...allLeafPackages].filter((leaf) => p0.has(leaf) && p1.has(leaf)); + const hasPartitionOverlap = leafPackagesOverlap.length > 0; + + console.log(`[build-partitioning] Overlap check (${hasPartitionOverlap ? "❌" : "✅"}):`); + if (hasPartitionOverlap) { + console.error(`[build-partitioning] Partitions overlap. Aborting.`); + console.error(leafPackagesOverlap); + process.exit(1); + } +} + +export async function assertCompleteness({ + packageDirsByName, + partitions, + allPackageDirs, +}: { + packageDirsByName: Map; + partitions: PartitionDefinition[]; + allPackageDirs: Set; +}) { + const redundantPackageNames = new Set( + [...packageDirsByName.entries()] + .filter(([pkgName, pkgDir]) => partitions[0].dirs.has(pkgDir) && partitions[1].dirs.has(pkgDir)) + .map(([pkgName, pkgDir]) => pkgName) + ); + console.log(`[build-partitioning] Redundancy:`); + console.log(redundantPackageNames); + + const completenessCheck = + partitions[0].dirs.size + partitions[1].dirs.size - (partitions.length - 1) * redundantPackageNames.size === + allPackageDirs.size; + console.log(`[build-partitioning] Completeness check (${!completenessCheck ? "❌" : "✅"}):`); + if (!completenessCheck) { + console.error(`[build-partitioning] All packages count: ${allPackageDirs.size}`); + console.error(`[build-partitioning] ${partitions[0].name} packages count: ${partitions[0].dirs.size}`); + console.error(`[build-partitioning] ${partitions[1].name} packages count: ${partitions[1].dirs.size}`); + console.error(`[build-partitioning] Redundant packages count: ${redundantPackageNames.size}`); + process.exit(1); + } + return allPackageDirs; +} + +export async function assertOptimalPartialBuild(args: { + partition: PartitionDefinition; + upstreamPackageNamesInPartition: Set; + affectedPackageNamesInPartition: Set; + relevantPackageNamesInPartition: Set; +}) { + const isOptimalPartialPartitionBuild = + args.upstreamPackageNamesInPartition.size + args.affectedPackageNamesInPartition.size === + args.relevantPackageNamesInPartition.size; + + console.log( + `[build-partitioning] 'Partial' build of '${args.partition.name}': Optimal build check ((${ + !isOptimalPartialPartitionBuild ? "❌" : "✅" + }))` + ); + if (!isOptimalPartialPartitionBuild) { + console.error(`[build-partitioning] Non-optimal 'Partial' build. Aborting.`); + process.exit(1); + } +} diff --git a/.github/supporting-files/ci/build-partitioning/build_partitioning.ts b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts new file mode 100644 index 00000000000..fc2452b4916 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts @@ -0,0 +1,229 @@ +#!/usr/bin/env bun + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Partial, None, Full, PartitionDefinition } from "./types"; +import { __ROOT_PKG_NAME, __P0, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_DIRS } from "./globals"; +import { + assertLeafPackagesInPartitionDefinitionsDontOverlap, + assertLeafPackagesInPartitionsExist, + assertCompleteness, + assertOptimalPartialBuild, +} from "./assertions"; + +import { $ } from "bun"; +import { parseArgs } from "util"; +import * as path from "path"; +import * as fs from "fs"; + +const { + values: { + outputPath: __ARG_outputPath, + forceFull: __ARG_forceFull, + baseSha: __ARG_baseSha, + headSha: __ARG_headSha, + graphJsonPath: __ARG_graphJsonPath, + }, +} = parseArgs({ + args: Bun.argv, + options: { + graphJsonPath: { type: "string" }, + baseSha: { type: "string" }, + headSha: { type: "string" }, + forceFull: { type: "string" }, + outputPath: { type: "string" }, + }, + strict: true, + allowPositionals: true, +}); + +async function getPartitions(): Promise> { + console.log(``); + console.log(`[build-partitioning] --- Summary ---`); + console.log(`[build-partitioning] graphJsonPath: ${__ARG_graphJsonPath}`); + console.log(`[build-partitioning] baseSha: ${__ARG_baseSha}`); + console.log(`[build-partitioning] headSha: ${__ARG_headSha}`); + console.log(`[build-partitioning] forceFull: ${__ARG_forceFull}`); + console.log(`[build-partitioning] outputPath: ${__ARG_outputPath}`); + console.log(`[build-partitioning] ---------------`); + console.log(``); + + const graphJson = await import(path.resolve(".", __ARG_graphJsonPath!)); + const packageDirsByName = new Map(graphJson.serializedPackagesLocationByName); + const packageNamesByDir = new Map([...packageDirsByName.entries()].map(([k, v]) => [v, k])); + + const allLeafPackages = new Set(packageDirsByName.keys()); + for (const link of graphJson.serializedDatavisGraph.links) { + allLeafPackages.delete(link.target); + } + allLeafPackages.delete(__ROOT_PKG_NAME); + + console.log(`[build-partitioning] All leaf packages:`); + console.log(allLeafPackages); + + console.log(`[build-partitioning] P0:`); + console.log(__P0); + + const p1 = new Set([...allLeafPackages].filter((leaf) => !__P0.has(leaf))); + console.log(`[build-partitioning] P1:`); + console.log(p1); + + const partitionDefinitions: PartitionDefinition[] = [ + { + name: "Partition 0", + leafPackageNames: __P0, + dirs: await getDirsOfDependencies(__P0), + }, + { + name: "Partition 1", + leafPackageNames: p1, + dirs: await getDirsOfDependencies(p1), + }, + ]; + + await assertLeafPackagesInPartitionDefinitionsDontOverlap({ allLeafPackages, p0: __P0, p1: p1 }); + await assertLeafPackagesInPartitionsExist({ + packageNames: partitionDefinitions.flatMap((p) => [...p.leafPackageNames]), + allLeafPackages, + }); + + const allPackageDirs = new Set( + outputArray(await $`pnpm -F='!${__ROOT_PKG_NAME}...' exec pwd`.text()) + .map((s) => path.relative(".", s)) + .map((pkgDir) => packageNamesByDir.get(pkgDir)!) + ); + + await assertCompleteness({ packageDirsByName, partitions: partitionDefinitions, allPackageDirs }); + + const nonSourceFilesPatternsForGitDiff = __NON_SOURCE_FILES_PATTERNS.map((p) => `':!${p}'`).join(" "); + console.log(nonSourceFilesPatternsForGitDiff); + // TODO: Use these patterns + const changedSourcePaths = outputArray( + await new Response( + Bun.spawnSync(`git diff --name-only ${__ARG_baseSha} ${__ARG_headSha} -- ${""}`.split(" ")).stdout + ).text() + ); + console.log("[build-partitioning] Changed source paths:"); + console.log(new Set(changedSourcePaths)); + + const changedSourcePathsInRoot = changedSourcePaths.filter((path) => + __PACKAGES_ROOT_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`)) + ); + + const relevantPackageDirsInAllPartitions = outputArray(await $`pnpm -F=...[${__ARG_baseSha}]... exec pwd`.text()); + const affectedPackageDirsInAllPartitions = outputArray(await $`pnpm -F=...[${__ARG_baseSha}] exec pwd`.text()); + + return await Promise.all( + partitionDefinitions.map(async (partition) => { + if (__ARG_forceFull || changedSourcePathsInRoot.length > 0) { + console.log(`[build-partitioning] 'Full' build of '${partition.name}'.`); + console.log( + `[build-partitioning] Building ${partition.dirs.size}/${partition.dirs.size}/${allPackageDirs.size} packages.` + ); + return { + mode: "full", + name: partition.name, + bootstrapPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F='${l}...'`).join(" "), + fullBuildPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F='${l}...'`).join(" "), + }; + } + + const changedSourcePathsInPartition = changedSourcePaths.filter((path) => + [...partition.dirs].some((partitionDir) => path.startsWith(partitionDir)) + ); + if (changedSourcePathsInPartition.length === 0) { + console.log(`[build-partitioning] 'None' build of '${partition.name}'.`); + console.log(`[build-partitioning] Building 0/${partition.dirs.size}/${allPackageDirs.size} packages.`); + console.log(``); + return { + mode: "none", + name: partition.name, + }; + } + + const affectedPackageNamesInPartition = new Set( + affectedPackageDirsInAllPartitions + .map((pkgDir) => path.relative(".", pkgDir)) + .filter((pkgDir) => partition.dirs.has(pkgDir)) + .map((packageDir) => packageNamesByDir.get(packageDir)!) + ); + + const relevantPackageNamesInPartition = new Set( + [...(await getDirsOfDependencies(affectedPackageNamesInPartition))] + .map((pkgDir) => path.relative(".", pkgDir)) + .map((pkgDir) => packageNamesByDir.get(pkgDir)!) + ); + + console.log(`[build-partitioning] 'Partial' build of '${partition.name}'`); + console.log( + `[build-partitioning] Building ${relevantPackageNamesInPartition.size}/${relevantPackageDirsInAllPartitions.length}/${allPackageDirs.size} packages.` + ); + console.log(relevantPackageNamesInPartition); + + const upstreamPackageNamesInPartition = new Set( + [...relevantPackageNamesInPartition].filter((pkgName) => !affectedPackageNamesInPartition.has(pkgName)) + ); + + await assertOptimalPartialBuild({ + partition, + relevantPackageNamesInPartition, + upstreamPackageNamesInPartition, + affectedPackageNamesInPartition, + }); + + return { + mode: "partial", + name: partition.name, + bootstrapPnpmFilterString: [...relevantPackageNamesInPartition].map((p) => `-F='${p}'`).join(" "), + upstreamPnpmFilterString: [...upstreamPackageNamesInPartition].map((p) => `-F='${p}'`).join(" "), + affectedPnpmFilterString: [...affectedPackageNamesInPartition].map((p) => `-F='...${p}'`).join(" "), + }; + }) + ); +} + +// MAIN + +const partitions = await getPartitions(); +const partitionsJson = JSON.stringify(partitions, null, 2); +console.log(``); +console.log(`[build-partitioning] --- PARTITIONS JSON ---`); +console.log(partitionsJson); + +const resolvedOutputPath = path.resolve(".", __ARG_outputPath!); +fs.writeFileSync(resolvedOutputPath, partitionsJson); +console.log(`[build-partitioning] --> Written to '${resolvedOutputPath}'`); +console.log(`[build-partitioning] Done.`); + +process.exit(0); + +// + +async function getDirsOfDependencies(leafPackageNames: Set) { + const packagesFilter = [...leafPackageNames].map((pkgName) => `-F=${pkgName}...`).join(" "); + return new Set( + outputArray(await new Response(Bun.spawnSync(`pnpm ${packagesFilter} exec pwd`.split(" ")).stdout).text()) // + .map((pkgDir) => path.relative(".", pkgDir)) + ); +} + +function outputArray(output: string) { + return output.trim().split(/\s/); +} diff --git a/.github/supporting-files/ci/build-partitioning/globals.ts b/.github/supporting-files/ci/build-partitioning/globals.ts new file mode 100644 index 00000000000..2b352f68208 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/globals.ts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as path from "path"; +import * as fs from "fs"; + +export const __ROOT_PKG_NAME = "kie-tools-root"; + +export const __PACKAGES_ROOT_DIRS = ["packages", "examples"]; + +export const __P0 = new Set([ + "@kie-tools/serverless-logic-web-tools-swf-builder-image", + "@kie-tools/kn-plugin-workflow", + "@kie-tools/serverless-logic-web-tools", + "@kie-tools/serverless-logic-web-tools-base-builder-image", + "@kie-tools/serverless-logic-web-tools-swf-dev-mode-image", + "@kie-tools/dashbuilder-swf-monitoring-dashboard", + "@kie-tools/dashbuilder-viewer-image", + "chrome-extension-serverless-workflow-editor", + "vscode-extension-dashbuilder-editor", + "yard-vscode-extension", + "swf-vscode-extension", +]); + +export const __NON_SOURCE_FILES_PATTERNS = fs + .readFileSync(path.resolve(__dirname, "../patterns/non-source-files-patterns.txt"), "utf-8") + .split(/\s/); diff --git a/.github/supporting-files/ci/build-partitioning/types.ts b/.github/supporting-files/ci/build-partitioning/types.ts new file mode 100644 index 00000000000..63f996b2848 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/types.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type None = { + mode: "none"; + name: string; +}; + +export type Full = { + mode: "full"; + name: string; + bootstrapPnpmFilterString: string; + fullBuildPnpmFilterString: string; +}; + +export type Partial = { + mode: "partial"; + name: string; + bootstrapPnpmFilterString: string; + upstreamPnpmFilterString: string; + affectedPnpmFilterString: string; +}; + +export type PartitionDefinition = { + name: string; + leafPackageNames: Set; + dirs: Set; +}; diff --git a/.github/supporting-files/ci/build-artifacts-patterns.txt b/.github/supporting-files/ci/patterns/build-artifacts-patterns.txt similarity index 100% rename from .github/supporting-files/ci/build-artifacts-patterns.txt rename to .github/supporting-files/ci/patterns/build-artifacts-patterns.txt diff --git a/.github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt b/.github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt similarity index 100% rename from .github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt rename to .github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt diff --git a/.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt b/.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt similarity index 100% rename from .github/supporting-files/ci/end-to-end-tests-reports-patterns.txt rename to .github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt diff --git a/.github/supporting-files/ci/non-source-files-patterns.txt b/.github/supporting-files/ci/patterns/non-source-files-patterns.txt similarity index 100% rename from .github/supporting-files/ci/non-source-files-patterns.txt rename to .github/supporting-files/ci/patterns/non-source-files-patterns.txt diff --git a/.github/supporting-files/ci/tests-reports-patterns.txt b/.github/supporting-files/ci/patterns/tests-reports-patterns.txt similarity index 100% rename from .github/supporting-files/ci/tests-reports-patterns.txt rename to .github/supporting-files/ci/patterns/tests-reports-patterns.txt diff --git a/.github/supporting-files/ci/tests-source-files-patterns.txt b/.github/supporting-files/ci/patterns/tests-source-files-patterns.txt similarity index 100% rename from .github/supporting-files/ci/tests-source-files-patterns.txt rename to .github/supporting-files/ci/patterns/tests-source-files-patterns.txt diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml index 01f346e2f19..f948980d592 100644 --- a/.github/workflows/ci_build.yml +++ b/.github/workflows/ci_build.yml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-13, windows-latest] + partition: [0, 1] runs-on: ${{ matrix.os }} steps: - name: "Support longpaths" @@ -36,6 +37,11 @@ jobs: with: ref: ${{ github.base_ref }} + - name: "Setup Node" + uses: actions/setup-node@v3 + with: + node-version: 18.14.0 + - name: "Setup CI patterns" id: ci_patterns uses: ./.github/actions/setup-ci-patterns @@ -44,43 +50,37 @@ jobs: id: setup_build_mode shell: bash run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export CHANGED_SOURCE_PATHS_IN_ROOT=($(printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" | grep -v -e "^packages" -e "^examples")) - echo "Changed source paths in root:" - echo ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_ROOT[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: Build` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} -eq 0 ]; then - echo 'No source files changed in root; `CI :: Build` (partial) will run.' - echo "mode=partial" >> $GITHUB_OUTPUT - else - echo 'Source files changed in root; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" + + npm -g install bun@1.1.6 + + ./.github/supporting-files/ci/build-partitioning/build_partitioning.ts \ + --outputPath='/tmp/partitions.json' \ + --forceFull='${{ !github.event.pull_request }}' \ + --baseSha='${{ steps.checkout_pr.outputs.base_sha }}' \ + --headSha='${{steps.checkout_pr.outputs.head_sha }}' \ + --graphJsonPath='./repo/graph.json' + + npm -g uninstall bun + + echo "mode=$(jq --raw-output '.[${{ matrix.partition }}].mode' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "bootstrapPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].bootstrapPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "fullBuildPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].fullBuildPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "upstreamPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].upstreamPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "affectedPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].affectedPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "Done." - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' uses: ./.github/actions/setup-env - - name: "FULL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' + - name: "Bootstrap" + if: steps.setup_build_mode.outputs.mode != 'none' env: PLAYWRIGHT_BASE__installDeps: "true" uses: ./.github/actions/bootstrap + with: + pnpm_filter_string: ${{ steps.setup_build_mode.outputs.bootstrapPnpmFilterString }} - - name: "FULL → Build (without SWF Stack)" + - name: "FULL → Build" if: steps.setup_build_mode.outputs.mode == 'full' env: WEBPACK__minimize: "false" @@ -96,35 +96,18 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: >- - pnpm - -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='!@kie-tools/kn-plugin-workflow...' - -F='!@kie-tools/serverless-logic-web-tools...' - -r --workspace-concurrency=1 build:prod - - - name: "PARTIAL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'partial' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F "...[${{ steps.checkout_pr.outputs.base_sha }}]..." -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' + eval "pnpm ${{ steps.setup_build_mode.outputs.fullBuildPnpmFilterString }} --workspace-concurrency=1 build:prod" - - name: "PARTIAL → Build dependencies" + - name: "PARTIAL → Build upstream" if: steps.setup_build_mode.outputs.mode == 'partial' shell: bash env: KIE_TOOLS_BUILD__buildContainerImages: ${{ runner.os != 'Windows' }} KIE_TOOLS_BUILD__buildExamples: "true" run: | - export ALL_DEPENDENCIES_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F=$(jq --raw-output .name package.json)^..."') - export CHANGED_PKGS_EXCLUSION_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F='"'"'!$(jq --raw-output .name package.json)'"'"'"') - echo $ALL_DEPENDENCIES_FILTER - echo $CHANGED_PKGS_EXCLUSION_FILTER - eval "pnpm $ALL_DEPENDENCIES_FILTER $CHANGED_PKGS_EXCLUSION_FILTER -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' build:dev" + eval "pnpm ${{ steps.setup_build_mode.outputs.upstreamPnpmFilterString }} build:dev" - - name: "PARTIAL → Build changed and dependents" + - name: "PARTIAL → Build changed and downstream" if: steps.setup_build_mode.outputs.mode == 'partial' env: WEBPACK__minimize: "false" @@ -137,7 +120,7 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: | - pnpm -F "...[${{ steps.checkout_pr.outputs.base_sha }}]" -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' --workspace-concurrency=1 build:prod + eval "pnpm ${{ steps.setup_build_mode.outputs.affectedPnpmFilterString }} --workspace-concurrency=1 build:prod" - name: "Check tests result (`main` only)" if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' diff --git a/.github/workflows/ci_build_swf_stack.yml b/.github/workflows/ci_build_swf_stack.yml deleted file mode 100644 index 07962dfd11a..00000000000 --- a/.github/workflows/ci_build_swf_stack.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: "CI :: Build (SWF Stack)" - -on: - push: - branches: [main] - pull_request: - branches: ["**"] - types: [opened, reopened, ready_for_review, synchronize] - -concurrency: - group: ${{ github.event.pull_request && format('ci-build-swf-stack-full-pr-{0}', github.event.pull_request.number) || format('ci-build-swf-stack-full-push-main-{0}', github.sha) }} - cancel-in-progress: true - -env: - TMPDIR: "/tmp" - -jobs: - run: - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - name: "Checkout @ GitHub default" - uses: actions/checkout@v3 - - - name: "Checkout @ Simulated squashed-merge if PR" - id: checkout_pr - uses: ./.github/actions/checkout-pr - with: - ref: ${{ github.base_ref }} - - - name: "Setup CI patterns" - id: ci_patterns - uses: ./.github/actions/setup-ci-patterns - - - name: "Setup build mode {none,full}" - id: setup_build_mode - shell: bash - run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export PKGS_IN_SWF_STACK=("@kie-tools/serverless-logic-web-tools-swf-builder-image" "@kie-tools/kn-plugin-workflow" "@kie-tools/serverless-logic-web-tools" "@kie-tools/serverless-logic-web-tools-base-builder-image") - - npm install -g graph-data-structure@2.0.0 - export GREP_RES=$(NODE_PATH=$(npm prefix -g)/lib/node_modules node scripts/sparse-checkout/list_packages_dependencies.js ./repo "${PKGS_IN_SWF_STACK[@]}" | tr -d '\n' | xargs -d ' ' -I{} echo -n " -e \"{}\"") - echo "GREP RES:" - echo $GREP_RES - npm uninstall -g graph-data-structure - - export CHANGED_SOURCE_PATHS_IN_SWF_STACK=($(eval "printf '%s\\n' "${CHANGED_SOURCE_PATHS[@]}" | grep $GREP_RES")) - echo "Changed source paths in "SWF Stack":" - echo ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} -eq 0 ]; then - echo 'No source files changed in "SWF Stack"; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - else - echo 'Source files changed in "SWF Stack"; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" - - - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/setup-env - - - name: "Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='@kie-tools/kn-plugin-workflow...' -F='@kie-tools/serverless-logic-web-tools...' - - - name: "Build (only SWF Stack)" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - WEBPACK__minimize: "false" - WEBPACK__tsLoaderTranspileOnly: "false" - KIE_TOOLS_BUILD__runLinters: "true" - KIE_TOOLS_BUILD__runTests: "true" - KIE_TOOLS_BUILD__runEndToEndTests: "true" - KIE_TOOLS_BUILD__buildContainerImages: "true" - KIE_TOOLS_BUILD__buildExamples: "true" - KIE_TOOLS_BUILD__ignoreTestFailures: ${{ !github.event.pull_request }} - KIE_TOOLS_BUILD__ignoreEndToEndTestFailures: ${{ !github.event.pull_request }} - DISPLAY: ":99.0" - START_SERVER_AND_TEST_INSECURE: "true" - NODE_OPTIONS: "--max_old_space_size=4096" - run: >- - pnpm - -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='@kie-tools/kn-plugin-workflow...' - -F='@kie-tools/serverless-logic-web-tools...' - --workspace-concurrency=1 build:prod - - - name: "Check tests result (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: actions/github-script@v6 - env: - KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS: |- - ${{ steps.ci_patterns.outputs.tests_reports_patterns }} - ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns }} - with: - result-encoding: string - script: | - const patterns = process.env["KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS"] - .split("\n") - .map(p => p.trim()) - .filter(p => p); - - const script = require("./scripts/check-junit-report-results/src/index.js"); - await script({ core, glob, patterns }); - - - name: "Check hanging uncommitted files (you should commit those!)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - shell: bash - run: | - git diff - [ "0" == "$(git diff | wc -l | tr -d ' ')" ] - - - name: "Upload reports and artifacts" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/upload-ci-reports-and-artifacts - - - name: "Upload end-to-end tests results to Buildkite (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' && !github.event.pull_request - shell: bash - env: - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} - BUILDKITE_BRANCH: ${{ github.ref_name }} - BUILDKITE_MESSAGE: ${{ github.event.commits[0].message }} - run: | - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" - echo "---------------------------- starting upload -----------------------" - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" | xargs -I{} curl -X POST \ - -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \ - -F "format=junit" \ - -F "data=@{}" \ - -F "run_env[CI]=github_actions" \ - -F "run_env[key]=$GITHUB_ACTION-$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" \ - -F "run_env[number]=$GITHUB_RUN_NUMBER" \ - -F "run_env[branch]=$BUILDKITE_BRANCH" \ - -F "run_env[commit_sha]=$GITHUB_SHA" \ - -F "run_env[message]=$BUILDKITE_MESSAGE" \ - -F "run_env[url]=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - https://analytics-api.buildkite.com/v1/uploads - - - name: "Print storage usage (after build)" - if: always() && !cancelled() - shell: bash - run: | - df -h . diff --git a/package.json b/package.json index bbf4be36980..9314aeadac2 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@kie-tools-scripts/update-version": "workspace:*", "@nice-move/prettier-plugin-package-json": "^0.6.1", "@prettier/plugin-xml": "^2", + "@types/bun": "^1.1.0", "@types/node": "^18.13.0", "husky": "^6.0.0", "postinstall-postinstall": "^2.1.0", @@ -32,8 +33,7 @@ "react-dropzone": "^11.4.2" }, "engines": { - "node": ">=18", - "pnpm": "8.7.0" + "node": ">=18" }, "packageManager": "pnpm@8.7.0", "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2a960fe165..1fc373e232e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,6 +54,9 @@ importers: "@prettier/plugin-xml": specifier: ^2 version: 2.2.0 + "@types/bun": + specifier: ^1.1.0 + version: 1.1.0 "@types/node": specifier: ^18.13.0 version: 18.13.0 @@ -13266,6 +13269,21 @@ importers: specifier: ^4.6.2 version: 4.8.4 + scripts/build-partitioning: + devDependencies: + "@types/bun": + specifier: ^1.1.0 + version: 1.1.0 + bun: + specifier: ^1.1.5 + version: 1.1.5 + graph-data-structure: + specifier: ^2.0.0 + version: 2.0.0 + typescript: + specifier: ^4.6.2 + version: 4.8.4 + scripts/check-junit-report-results: dependencies: fast-xml-parser: @@ -23727,6 +23745,78 @@ packages: engines: { node: ">=8.0" } dev: true + /@oven/bun-darwin-aarch64@1.1.5: + resolution: + { integrity: sha512-z0k3W2XEfa11OVUW0vp9pWLlpcPKY6TtpwPvREXCA0nFWj9LxbIRr5FZ1U6+M0gCRlx+5XOpSsWQ7+/HJcDgSg== } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64-baseline@1.1.5: + resolution: + { integrity: sha512-w3bHmbgrU0L8fCzW0uf0MYPo2DMekHG0w2SSQ+ZspkGQ11ppTu7FJZxZMAw8UQAAX3FgltFnaI1p6J9gkv4MMw== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64@1.1.5: + resolution: + { integrity: sha512-Pv1QDb69u6JMXfWiAE638aUWzsARGxWDQ1J1YLxOJVzbEe9f8qvMPZX7yP4OcfFMyc1ZcFrd7kDSkXQHZ/DeTg== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-aarch64@1.1.5: + resolution: + { integrity: sha512-iObapm21/EsuQN3Z3m/DwAyjeGQaYuwC+KocPT29n7Tv+C/bubIEtikrmSGfA2CiSA5n6dFsFGk/t/uFfUuVMA== } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64-baseline@1.1.5: + resolution: + { integrity: sha512-ENUbLzO/tKpzUp04e9u+jZi+RFIpIJ/RQy7WKTstzce3fZqAsfh65CJE7TqKvFpHEDNxaRqJRg7yUonSCO1BmA== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64@1.1.5: + resolution: + { integrity: sha512-KrvxD5sUf0WcdjVJm2LyGJqPzBldYWT4EQy+N5c5gzFnrEKSXOsj5lFE2vPNICS8Zvwzt8zLr0OwKq/WU93xTQ== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64-baseline@1.1.5: + resolution: + { integrity: sha512-OCkUFyhbLR0kzAzuZg7pAa48TQFukYlaHTPH7ZWJkuS4ewFsDiYE+ET1BFn0LoRPlK/IhR8F6S21cH1elrlWCw== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64@1.1.5: + resolution: + { integrity: sha512-kLRNRrQhCqXXoc0E6y9lBYbec6Sd7zIZtasoO6V7pYatMEaDB1ufIktmM5YVtCGYqNJ1+zNsENuU9xxPuprPDA== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@patternfly/patternfly@4.224.2: resolution: { integrity: sha512-HGNV26uyHSIECuhjPg/WGn0mXbAotcs6ODfhAOkfYjIgGylddgiwElxUe1rpEHV5mQJJ2rMn4OdeJIIpzRX61g== } @@ -28998,6 +29088,13 @@ packages: "@types/node": 18.17.18 dev: true + /@types/bun@1.1.0: + resolution: + { integrity: sha512-QGK0yU4jh0OK1A7DyhPkQuKjHQCC5jSJa3dpWIEhHv/rPfb6zLfdArc4/uUUZBMTcjilsafRXnPWO+1owb572Q== } + dependencies: + bun-types: 1.1.0 + dev: true + /@types/cacheable-request@6.0.1: resolution: { integrity: sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ== } @@ -29720,6 +29817,13 @@ packages: resolution: { integrity: sha512-/4QOuy3ZpV7Ya1GTRz5CYSz3DgkKpyUptXuQ5PPce7uuyJAOR7r9FhkmxJfvcNUXyklbC63a+YvB3jxy7s9ngw== } + /@types/node@20.11.30: + resolution: + { integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw== } + dependencies: + undici-types: 5.26.5 + dev: true + /@types/normalize-package-data@2.4.0: resolution: { integrity: sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== } @@ -30091,6 +30195,13 @@ packages: source-map: 0.6.1 dev: true + /@types/ws@8.5.10: + resolution: + { integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A== } + dependencies: + "@types/node": 18.17.18 + dev: true + /@types/ws@8.5.5: resolution: { integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg== } @@ -33340,6 +33451,32 @@ packages: semver: 7.5.4 dev: true + /bun-types@1.1.0: + resolution: + { integrity: sha512-GhMDD7TosdJzQPGUOcQD5PZshvXVxDfwGAZs2dq+eSaPsRn3iUCzvpFlsg7Q51bXVzLAUs+FWHlnmpgZ5UggIg== } + dependencies: + "@types/node": 20.11.30 + "@types/ws": 8.5.10 + dev: true + + /bun@1.1.5: + resolution: + { integrity: sha512-9gVjgPLGQqKEAcuTL1x/hn7PUYvWHlx/tm3//xIRkJk55IVWXcmFM6kQtV+Jn9k5x3wvtAd8a3RRyk2alyKPjA== } + cpu: [arm64, x64] + os: [darwin, linux, win32] + hasBin: true + requiresBuild: true + optionalDependencies: + "@oven/bun-darwin-aarch64": 1.1.5 + "@oven/bun-darwin-x64": 1.1.5 + "@oven/bun-darwin-x64-baseline": 1.1.5 + "@oven/bun-linux-aarch64": 1.1.5 + "@oven/bun-linux-x64": 1.1.5 + "@oven/bun-linux-x64-baseline": 1.1.5 + "@oven/bun-windows-x64": 1.1.5 + "@oven/bun-windows-x64-baseline": 1.1.5 + dev: true + /busboy@1.6.0: resolution: { integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== } @@ -51785,6 +51922,11 @@ packages: resolution: { integrity: sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g== } + /undici-types@5.26.5: + resolution: + { integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== } + dev: true + /undoo@0.5.0: resolution: { integrity: sha512-SPlDcde+AUHoFKeVlH2uBJxqVkw658I4WR2rPoygC1eRCzm3GeoP8S6xXZVJeBVOQQid8X2xUBW0N4tOvvHH3Q== }