From bee8942fa5676d2a5c33167332aea2db12cda4d6 Mon Sep 17 00:00:00 2001 From: Tiago Bento Date: Sun, 28 Apr 2024 03:03:41 -0400 Subject: [PATCH] Test --- .github/actions/setup-ci-patterns/action.yml | 14 +- .../ci/build-partitioning/README.md | 28 ++ .../ci/build-partitioning/assertions.ts | 119 +++++++++ .../build-partitioning/build_partitioning.ts | 248 ++++++++++++++++++ .../ci/build-partitioning/globals.ts | 36 +++ .../ci/build-partitioning/types.ts | 44 ++++ .../ci/partitions/partition0.txt | 1 + .../ci/partitions/partition1.txt | 10 + .../build-artifacts-patterns.txt | 0 .../end-to-end-tests-artifacts-patterns.txt | 0 .../end-to-end-tests-reports-patterns.txt | 0 .../non-source-files-patterns.txt | 0 .../{ => patterns}/tests-reports-patterns.txt | 0 .../tests-source-files-patterns.txt | 0 .github/workflows/ci_build.yml | 92 +++---- .github/workflows/ci_build_swf_stack.yml | 168 ------------ ..._kogito_serverless_operator_e2e_tests.yaml | 123 --------- package.json | 3 +- .../kogito-serverless-operator/env/index.js | 9 +- .../kogito-serverless-operator/package.json | 8 +- packages/kogito-swf-common/resources/Makefile | 2 +- pnpm-lock.yaml | 139 ++++++++++ 22 files changed, 680 insertions(+), 364 deletions(-) create mode 100644 .github/supporting-files/ci/build-partitioning/README.md create mode 100644 .github/supporting-files/ci/build-partitioning/assertions.ts create mode 100644 .github/supporting-files/ci/build-partitioning/build_partitioning.ts create mode 100644 .github/supporting-files/ci/build-partitioning/globals.ts create mode 100644 .github/supporting-files/ci/build-partitioning/types.ts create mode 100644 .github/supporting-files/ci/partitions/partition0.txt create mode 100644 .github/supporting-files/ci/partitions/partition1.txt rename .github/supporting-files/ci/{ => patterns}/build-artifacts-patterns.txt (100%) rename .github/supporting-files/ci/{ => patterns}/end-to-end-tests-artifacts-patterns.txt (100%) rename .github/supporting-files/ci/{ => patterns}/end-to-end-tests-reports-patterns.txt (100%) rename .github/supporting-files/ci/{ => patterns}/non-source-files-patterns.txt (100%) rename .github/supporting-files/ci/{ => patterns}/tests-reports-patterns.txt (100%) rename .github/supporting-files/ci/{ => patterns}/tests-source-files-patterns.txt (100%) delete mode 100644 .github/workflows/ci_build_swf_stack.yml delete mode 100644 .github/workflows/ci_kogito_serverless_operator_e2e_tests.yaml diff --git a/.github/actions/setup-ci-patterns/action.yml b/.github/actions/setup-ci-patterns/action.yml index 6261751e347..b3ddb660d2b 100644 --- a/.github/actions/setup-ci-patterns/action.yml +++ b/.github/actions/setup-ci-patterns/action.yml @@ -54,9 +54,9 @@ runs: const pnpmWorkspacePackagesRootPaths = ["packages", "examples"].map(p => path.join(cwd, p)).join("\n"); - const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/non-source-files-patterns.txt"), "utf-8"); + const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/non-source-files-patterns.txt"), "utf-8"); const nonSourceFilesPatternsForGitDiff = nonSourceFilesPatterns.split("\n").filter(p => p.trim() !== "").map(p => `':!${p}'`).join(" "); - const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/tests-source-files-patterns.txt"), "utf-8"); + const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/tests-source-files-patterns.txt"), "utf-8"); core.setOutput("pnpm_workspace_packages_root_paths", pnpmWorkspacePackagesRootPaths); core.setOutput("non_source_files_patterns", nonSourceFilesPatterns); @@ -78,14 +78,14 @@ runs: core.setOutput(outputName, patterns); } - await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/tests-reports-patterns.txt")); - await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt")); - await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt")); - await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/build-artifacts-patterns.txt")); + await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/tests-reports-patterns.txt")); + await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt")); + await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt")); + await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/build-artifacts-patterns.txt")); core.setOutput( "end_to_end_tests_reports_patterns_for_find", - prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt", "utf-8")) + prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt", "utf-8")) .split("\n") .map(p => `-path '${p}'`) .join(" -o ") diff --git a/.github/supporting-files/ci/build-partitioning/README.md b/.github/supporting-files/ci/build-partitioning/README.md new file mode 100644 index 00000000000..1b51ee15dbf --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/README.md @@ -0,0 +1,28 @@ +# Build partitioning + +This script uses `bun` as runtime and has the purpose of splitting the monorepo build in 2 partitions that, when built, will ensure the entire monorepo has been covered. + +Each partition is completely independent from each other, and partition builds will be assigned a mode: + +- `none`: No source code changes or none of the changes affect this partition. +- `full`: All packages in the partition need to be built and tested. +- `partial`: Only selected packages of the partition were affected by the changes, so only this subset needs to be rebuilt and tested. + +### Usage + +At the root of the monorepo, you can run + +```bash +npx bun .github/supporting-files/ci/build-partitioning/build_partitioning.ts \ + --outputPath='/tmp/partitions.json' \ + --forceFull="false" \ + --baseSha="$(git rev-parse HEAD~1)" \ + --headSha="$(git rev-parse HEAD~0)" \ + --graphJsonPath='./repo/graph.json' +``` + +- `--outputPath`: Where the resulting JSON should be written to. +- `--forceFull`: Wheter or not to force `full` mode +- `--baseSha`: Base Git commit SHA. +- `--headSha`: HEAD Git commit SHA. +- `--graphJsonPath`: Where to find the `graph.json` of the monorepo in Datavis graph format. diff --git a/.github/supporting-files/ci/build-partitioning/assertions.ts b/.github/supporting-files/ci/build-partitioning/assertions.ts new file mode 100644 index 00000000000..4d21bb5cedb --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/assertions.ts @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PartitionDefinition } from "./types"; + +export async function assertLeafPackagesInPartitionsExist({ + packageNames, + allLeafPackages, +}: { + packageNames: string[]; + allLeafPackages: Set; +}) { + const nonLeafPackagesInPartitions = new Set(packageNames.filter((l) => !allLeafPackages.has(l))); + console.log( + `[build-partitioning] Partition definitions only use leaf packages (${ + nonLeafPackagesInPartitions.size > 0 ? "❌" : "✅" + }):` + ); + if (nonLeafPackagesInPartitions.size > 0) { + console.error(`[build-partitioning] Non-leaf packages found in partition definitions. Aborting.`); + console.error(nonLeafPackagesInPartitions); + process.exit(1); + } +} + +export async function assertLeafPackagesInPartitionDefinitionsDontOverlap({ + allLeafPackages, + partitionDefinitions, +}: { + allLeafPackages: Set; + partitionDefinitions: PartitionDefinition[]; +}) { + const leafPackagesOverlap = partitionDefinitions.flatMap((s) => [...s.leafPackageNames]); + const hasPartitionOverlap = allLeafPackages.size !== leafPackagesOverlap.length; + + console.log(`[build-partitioning] Partition definitions don't overlap (${hasPartitionOverlap ? "❌" : "✅"}):`); + if (hasPartitionOverlap) { + console.error(`[build-partitioning] Partitions definitions declare overlapping leaf packages. Aborting.`); + console.error(leafPackagesOverlap); + process.exit(1); + } +} + +export async function assertCompleteness({ + packageDirsByName, + partitions, + allPackageDirs, +}: { + packageDirsByName: Map; + partitions: PartitionDefinition[]; + allPackageDirs: Set; +}) { + const partitionsByPkgDir = new Map(); + for (const p of partitions) { + for (const pkgDir of p.dirs) { + partitionsByPkgDir.set(pkgDir, [...(partitionsByPkgDir.get(pkgDir) ?? []), p.name]); + } + } + + const redundancyOnPartitionCombinations = new Map(); + partitionsByPkgDir.forEach((partitions, pkgDir) => { + if (partitions.length > 1) { + // We're only interested in packages that are going to be built as part of more than one partition. + const key = partitions.sort().join(" & "); + redundancyOnPartitionCombinations.set(key, [...(redundancyOnPartitionCombinations.get(key) ?? []), pkgDir]); + } + }); + + console.log(`[build-partitioning] Packages that will be built by more than one partition:`); + console.log(redundancyOnPartitionCombinations); + + const completenessCheck = [...allPackageDirs].filter((pkgDir) => !partitionsByPkgDir.has(pkgDir)).length <= 0; + console.log(`[build-partitioning] Partition definitions completeness (${!completenessCheck ? "❌" : "✅"}):`); + if (!completenessCheck) { + console.error(`[build-partitioning] All packages count: ${allPackageDirs.size}`); + for (const p of partitions) { + console.error(`[build-partitioning] ${p.name} packages count: ${p.dirs.size}`); + } + process.exit(1); + } + return allPackageDirs; +} + +export async function assertOptimalPartialBuild(args: { + partition: PartitionDefinition; + upstreamPackageNamesInPartition: Set; + affectedPackageNamesInPartition: Set; + relevantPackageNamesInPartition: Set; +}) { + const isOptimalPartialPartitionBuild = + args.upstreamPackageNamesInPartition.size + args.affectedPackageNamesInPartition.size === + args.relevantPackageNamesInPartition.size; + + console.log( + `[build-partitioning] 'Partial' build of '${args.partition.name}': Optimal build check ((${ + !isOptimalPartialPartitionBuild ? "❌" : "✅" + }))` + ); + if (!isOptimalPartialPartitionBuild) { + console.error(`[build-partitioning] Non-optimal 'Partial' build. Aborting.`); + process.exit(1); + } +} diff --git a/.github/supporting-files/ci/build-partitioning/build_partitioning.ts b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts new file mode 100644 index 00000000000..7215b35ba95 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts @@ -0,0 +1,248 @@ +#!/usr/bin/env bun + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Partial, None, Full, PartitionDefinition } from "./types"; +import { __ROOT_PKG_NAME, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_DIRS, stdoutArray } from "./globals"; +import { + assertLeafPackagesInPartitionDefinitionsDontOverlap, + assertLeafPackagesInPartitionsExist, + assertCompleteness, + assertOptimalPartialBuild, +} from "./assertions"; + +import * as path from "path"; +import * as fs from "fs"; +import { execSync } from "child_process"; +import { parseArgs } from "util"; + +const { + values: { + outputPath: __ARG_outputPath, + forceFull: forceFull, + baseSha: __ARG_baseSha, + headSha: __ARG_headSha, + graphJsonPath: __ARG_graphJsonPath, + partition: __ARG_partitionFilePaths, + }, +} = parseArgs({ + args: Bun.argv, + options: { + graphJsonPath: { type: "string" }, + baseSha: { type: "string" }, + headSha: { type: "string" }, + forceFull: { type: "string" }, + outputPath: { type: "string" }, + partition: { type: "string", multiple: true }, + }, + strict: true, + allowPositionals: true, +}); + +const __ARG_forceFull = forceFull === "true"; + +const partitions = await getPartitions(); +const partitionsJson = JSON.stringify(partitions, null, 2); +console.log(``); +console.log(`[build-partitioning] --- PARTITIONS JSON ---`); +console.log(partitionsJson); + +const resolvedOutputPath = path.resolve(".", __ARG_outputPath!); +fs.writeFileSync(resolvedOutputPath, partitionsJson); +console.log(`[build-partitioning] --> Written to '${resolvedOutputPath}'`); +console.log(`[build-partitioning] Done.`); + +process.exit(0); + +// + +async function getPartitions(): Promise> { + console.log(``); + console.log(`[build-partitioning] --- Summary ---`); + console.log(`[build-partitioning] graphJsonPath: ${__ARG_graphJsonPath}`); + console.log(`[build-partitioning] baseSha: ${__ARG_baseSha}`); + console.log(`[build-partitioning] headSha: ${__ARG_headSha}`); + console.log(`[build-partitioning] forceFull: ${forceFull}`); + console.log(`[build-partitioning] outputPath: ${__ARG_outputPath}`); + console.log(`[build-partitioning] ---------------`); + console.log(``); + + const graphJson = await import(path.resolve(".", __ARG_graphJsonPath!)); + const packageDirsByName = new Map(graphJson.serializedPackagesLocationByName); + const packageNamesByDir = new Map([...packageDirsByName.entries()].map(([k, v]) => [v, k])); + + const allLeafPackages = new Set(packageDirsByName.keys()); + for (const link of graphJson.serializedDatavisGraph.links) { + allLeafPackages.delete(link.target); + } + allLeafPackages.delete(__ROOT_PKG_NAME); + + console.log(`[build-partitioning] All leaf packages:`); + console.log(allLeafPackages); + + let leafPackagesDefinedByPartitions = new Set(); + const partitionDefinitions: PartitionDefinition[] = await Promise.all( + (__ARG_partitionFilePaths ?? []).map(async (partitionFile) => { + const leafPackageNames = new Set( + fs + .readFileSync(path.resolve(".", partitionFile), "utf-8") + .trim() + .split("\n") + .map((pkgName) => pkgName.trim()) + ); + + leafPackagesDefinedByPartitions = new Set([...leafPackagesDefinedByPartitions, ...leafPackageNames]); + + return { + name: partitionFile, + leafPackageNames: leafPackageNames, + dirs: await getDirsOfDependencies(leafPackageNames), + }; + }) + ); + + // Adds the remaining packages as a last partition + const remainingLeafPackages = new Set( + [...allLeafPackages].filter((leaf) => !leafPackagesDefinedByPartitions.has(leaf)) + ); + partitionDefinitions.push({ + name: "Partition N (remaining leaf packages)", + leafPackageNames: remainingLeafPackages, + dirs: await getDirsOfDependencies(remainingLeafPackages), + }); + + for (const p of partitionDefinitions) { + console.log(`[build-partitioning] ${p.name}:`); + console.log(p.leafPackageNames); + } + + await assertLeafPackagesInPartitionDefinitionsDontOverlap({ allLeafPackages, partitionDefinitions }); + await assertLeafPackagesInPartitionsExist({ + packageNames: partitionDefinitions.flatMap((p) => [...p.leafPackageNames]), + allLeafPackages, + }); + + const allPackageDirs = new Set( + stdoutArray(await execSync(`pnpm -F '!${__ROOT_PKG_NAME}...' exec bash -c pwd`).toString()) + .map((pkgDir) => path.relative(".", pkgDir)) + .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep)) + ); + + await assertCompleteness({ packageDirsByName, partitions: partitionDefinitions, allPackageDirs }); + + const nonSourceFilesPatternsForGitDiff = __NON_SOURCE_FILES_PATTERNS.map((p) => `':!${p}'`).join(" "); + const changedSourcePaths = stdoutArray( + execSync(`git diff --name-only ${__ARG_baseSha} ${__ARG_headSha} -- ${nonSourceFilesPatternsForGitDiff}`).toString() + ); + console.log("[build-partitioning] Changed source paths:"); + console.log(new Set(changedSourcePaths)); + + const changedSourcePathsInRoot = changedSourcePaths.filter((path) => + __PACKAGES_ROOT_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`)) + ); + + const relevantPackageDirsInAllPartitions = stdoutArray( + await execSync(`pnpm -F ...[${__ARG_baseSha}]... exec bash -c pwd`).toString() + ) + .map((pkgDir) => path.relative(".", pkgDir)) + .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep)); + + const affectedPackageDirsInAllPartitions = stdoutArray( + await execSync(`pnpm -F ...[${__ARG_baseSha}] exec bash -c pwd`).toString() + ) + .map((pkgDir) => path.relative(".", pkgDir)) + .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep)); + + return await Promise.all( + partitionDefinitions.map(async (partition) => { + if (__ARG_forceFull || changedSourcePathsInRoot.length > 0) { + console.log(`[build-partitioning] 'Full' build of '${partition.name}'.`); + console.log( + `[build-partitioning] Building ${partition.dirs.size}/${partition.dirs.size}/${allPackageDirs.size} packages.` + ); + return { + mode: "full", + name: partition.name, + bootstrapPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F '${l}...'`).join(" "), + fullBuildPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F '${l}...'`).join(" "), + }; + } + + const changedSourcePathsInPartition = changedSourcePaths.filter((path) => + [...partition.dirs].some((partitionDir) => path.startsWith(partitionDir)) + ); + if (changedSourcePathsInPartition.length === 0) { + console.log(`[build-partitioning] 'None' build of '${partition.name}'.`); + console.log(`[build-partitioning] Building 0/${partition.dirs.size}/${allPackageDirs.size} packages.`); + console.log(``); + return { + mode: "none", + name: partition.name, + }; + } + + const affectedPackageNamesInPartition = new Set( + affectedPackageDirsInAllPartitions + .filter((pkgDir) => partition.dirs.has(pkgDir)) + .map((packageDir) => packageNamesByDir.get(packageDir)!) + ); + + const relevantPackageNamesInPartition = new Set( + [...(await getDirsOfDependencies(affectedPackageNamesInPartition))].map( + (pkgDir) => packageNamesByDir.get(pkgDir)! + ) + ); + + console.log(`[build-partitioning] 'Partial' build of '${partition.name}'`); + console.log( + `[build-partitioning] Building ${relevantPackageNamesInPartition.size}/${relevantPackageDirsInAllPartitions.length}/${allPackageDirs.size} packages.` + ); + console.log(relevantPackageNamesInPartition); + + const upstreamPackageNamesInPartition = new Set( + [...relevantPackageNamesInPartition].filter((pkgName) => !affectedPackageNamesInPartition.has(pkgName)) + ); + + await assertOptimalPartialBuild({ + partition, + relevantPackageNamesInPartition, + upstreamPackageNamesInPartition, + affectedPackageNamesInPartition, + }); + + return { + mode: "partial", + name: partition.name, + bootstrapPnpmFilterString: [...relevantPackageNamesInPartition].map((p) => `-F '${p}'`).join(" "), + upstreamPnpmFilterString: [...upstreamPackageNamesInPartition].map((p) => `-F '${p}'`).join(" "), + affectedPnpmFilterString: [...affectedPackageNamesInPartition].map((p) => `-F '...${p}'`).join(" "), + }; + }) + ); +} + +async function getDirsOfDependencies(leafPackageNames: Set) { + const packagesFilter = [...leafPackageNames].map((pkgName) => `-F ${pkgName}...`).join(" "); + return new Set( + stdoutArray(execSync(`pnpm ${packagesFilter} exec bash -c pwd`).toString()) // + .map((pkgDir) => path.relative(".", pkgDir)) + .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep)) + ); +} diff --git a/.github/supporting-files/ci/build-partitioning/globals.ts b/.github/supporting-files/ci/build-partitioning/globals.ts new file mode 100644 index 00000000000..c052e0da5a7 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/globals.ts @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as path from "path"; +import * as fs from "fs"; + +export const __ROOT_PKG_NAME = "kie-tools-root"; + +export const __PACKAGES_ROOT_DIRS = ["packages", "examples"]; + +export const __NON_SOURCE_FILES_PATTERNS = stdoutArray( + fs.readFileSync(path.resolve(__dirname, "../patterns/non-source-files-patterns.txt"), "utf-8") +); + +export function stdoutArray(output: string) { + return output + .trim() + .split(/\s/) + .filter((s) => !!s); +} diff --git a/.github/supporting-files/ci/build-partitioning/types.ts b/.github/supporting-files/ci/build-partitioning/types.ts new file mode 100644 index 00000000000..63f996b2848 --- /dev/null +++ b/.github/supporting-files/ci/build-partitioning/types.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type None = { + mode: "none"; + name: string; +}; + +export type Full = { + mode: "full"; + name: string; + bootstrapPnpmFilterString: string; + fullBuildPnpmFilterString: string; +}; + +export type Partial = { + mode: "partial"; + name: string; + bootstrapPnpmFilterString: string; + upstreamPnpmFilterString: string; + affectedPnpmFilterString: string; +}; + +export type PartitionDefinition = { + name: string; + leafPackageNames: Set; + dirs: Set; +}; diff --git a/.github/supporting-files/ci/partitions/partition0.txt b/.github/supporting-files/ci/partitions/partition0.txt new file mode 100644 index 00000000000..3792439c1ac --- /dev/null +++ b/.github/supporting-files/ci/partitions/partition0.txt @@ -0,0 +1 @@ +@kie-tools/kn-plugin-workflow \ No newline at end of file diff --git a/.github/supporting-files/ci/partitions/partition1.txt b/.github/supporting-files/ci/partitions/partition1.txt new file mode 100644 index 00000000000..c70c6cca6ec --- /dev/null +++ b/.github/supporting-files/ci/partitions/partition1.txt @@ -0,0 +1,10 @@ +@kie-tools/serverless-logic-web-tools-swf-builder-image +@kie-tools/serverless-logic-web-tools +@kie-tools/serverless-logic-web-tools-base-builder-image +@kie-tools/serverless-logic-web-tools-swf-dev-mode-image +@kie-tools/dashbuilder-swf-monitoring-dashboard +@kie-tools/dashbuilder-viewer-image +chrome-extension-serverless-workflow-editor +vscode-extension-dashbuilder-editor +yard-vscode-extension +swf-vscode-extension diff --git a/.github/supporting-files/ci/build-artifacts-patterns.txt b/.github/supporting-files/ci/patterns/build-artifacts-patterns.txt similarity index 100% rename from .github/supporting-files/ci/build-artifacts-patterns.txt rename to .github/supporting-files/ci/patterns/build-artifacts-patterns.txt diff --git a/.github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt b/.github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt similarity index 100% rename from .github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt rename to .github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt diff --git a/.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt b/.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt similarity index 100% rename from .github/supporting-files/ci/end-to-end-tests-reports-patterns.txt rename to .github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt diff --git a/.github/supporting-files/ci/non-source-files-patterns.txt b/.github/supporting-files/ci/patterns/non-source-files-patterns.txt similarity index 100% rename from .github/supporting-files/ci/non-source-files-patterns.txt rename to .github/supporting-files/ci/patterns/non-source-files-patterns.txt diff --git a/.github/supporting-files/ci/tests-reports-patterns.txt b/.github/supporting-files/ci/patterns/tests-reports-patterns.txt similarity index 100% rename from .github/supporting-files/ci/tests-reports-patterns.txt rename to .github/supporting-files/ci/patterns/tests-reports-patterns.txt diff --git a/.github/supporting-files/ci/tests-source-files-patterns.txt b/.github/supporting-files/ci/patterns/tests-source-files-patterns.txt similarity index 100% rename from .github/supporting-files/ci/tests-source-files-patterns.txt rename to .github/supporting-files/ci/patterns/tests-source-files-patterns.txt diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml index 01f346e2f19..e53ba07f28c 100644 --- a/.github/workflows/ci_build.yml +++ b/.github/workflows/ci_build.yml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-13, windows-latest] + partition: [0, 1, 2] runs-on: ${{ matrix.os }} steps: - name: "Support longpaths" @@ -36,6 +37,16 @@ jobs: with: ref: ${{ github.base_ref }} + - name: "Setup pnpm" + uses: pnpm/action-setup@v2 + with: + version: 8.7.0 + + - name: "Setup Node" + uses: actions/setup-node@v3 + with: + node-version: 18.14.0 + - name: "Setup CI patterns" id: ci_patterns uses: ./.github/actions/setup-ci-patterns @@ -44,43 +55,39 @@ jobs: id: setup_build_mode shell: bash run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export CHANGED_SOURCE_PATHS_IN_ROOT=($(printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" | grep -v -e "^packages" -e "^examples")) - echo "Changed source paths in root:" - echo ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_ROOT[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: Build` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_ROOT[@]} -eq 0 ]; then - echo 'No source files changed in root; `CI :: Build` (partial) will run.' - echo "mode=partial" >> $GITHUB_OUTPUT - else - echo 'Source files changed in root; `CI :: Build` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" + + npm -g install bun@1.1.6 + + bun ./.github/supporting-files/ci/build-partitioning/build_partitioning.ts \ + --outputPath='/tmp/partitions.json' \ + --forceFull='${{ !github.event.pull_request }}' \ + --baseSha='${{ steps.checkout_pr.outputs.base_sha }}' \ + --headSha='${{steps.checkout_pr.outputs.head_sha }}' \ + --graphJsonPath='./repo/graph.json' \ + --partition='./.github/supporting-files/ci/partitions/partition0.txt' \ + --partition='./.github/supporting-files/ci/partitions/partition1.txt' + + npm -g uninstall bun + + echo "mode=$(jq --raw-output '.[${{ matrix.partition }}].mode' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "bootstrapPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].bootstrapPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "fullBuildPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].fullBuildPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "upstreamPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].upstreamPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "affectedPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].affectedPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT + echo "Done." - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' uses: ./.github/actions/setup-env - - name: "FULL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' + - name: "Bootstrap" + if: steps.setup_build_mode.outputs.mode != 'none' env: PLAYWRIGHT_BASE__installDeps: "true" uses: ./.github/actions/bootstrap + with: + pnpm_filter_string: ${{ steps.setup_build_mode.outputs.bootstrapPnpmFilterString }} - - name: "FULL → Build (without SWF Stack)" + - name: "FULL → Build" if: steps.setup_build_mode.outputs.mode == 'full' env: WEBPACK__minimize: "false" @@ -96,35 +103,18 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: >- - pnpm - -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='!@kie-tools/kn-plugin-workflow...' - -F='!@kie-tools/serverless-logic-web-tools...' - -r --workspace-concurrency=1 build:prod - - - name: "PARTIAL → Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'partial' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F "...[${{ steps.checkout_pr.outputs.base_sha }}]..." -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' + eval "pnpm ${{ steps.setup_build_mode.outputs.fullBuildPnpmFilterString }} --workspace-concurrency=1 build:prod" - - name: "PARTIAL → Build dependencies" + - name: "PARTIAL → Build upstream" if: steps.setup_build_mode.outputs.mode == 'partial' shell: bash env: KIE_TOOLS_BUILD__buildContainerImages: ${{ runner.os != 'Windows' }} KIE_TOOLS_BUILD__buildExamples: "true" run: | - export ALL_DEPENDENCIES_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F=$(jq --raw-output .name package.json)^..."') - export CHANGED_PKGS_EXCLUSION_FILTER=$(pnpm -F="...[${{ steps.checkout_pr.outputs.base_sha }}]" exec bash -c 'echo -n " -F='"'"'!$(jq --raw-output .name package.json)'"'"'"') - echo $ALL_DEPENDENCIES_FILTER - echo $CHANGED_PKGS_EXCLUSION_FILTER - eval "pnpm $ALL_DEPENDENCIES_FILTER $CHANGED_PKGS_EXCLUSION_FILTER -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' build:dev" + eval "pnpm ${{ steps.setup_build_mode.outputs.upstreamPnpmFilterString }} build:dev" - - name: "PARTIAL → Build changed and dependents" + - name: "PARTIAL → Build changed and downstream" if: steps.setup_build_mode.outputs.mode == 'partial' env: WEBPACK__minimize: "false" @@ -137,7 +127,7 @@ jobs: START_SERVER_AND_TEST_INSECURE: "true" NODE_OPTIONS: "--max_old_space_size=4096" run: | - pnpm -F "...[${{ steps.checkout_pr.outputs.base_sha }}]" -F='!@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='!@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='!@kie-tools/kn-plugin-workflow...' -F='!@kie-tools/serverless-logic-web-tools...' --workspace-concurrency=1 build:prod + eval "pnpm ${{ steps.setup_build_mode.outputs.affectedPnpmFilterString }} --workspace-concurrency=1 build:prod" - name: "Check tests result (`main` only)" if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' diff --git a/.github/workflows/ci_build_swf_stack.yml b/.github/workflows/ci_build_swf_stack.yml deleted file mode 100644 index 07962dfd11a..00000000000 --- a/.github/workflows/ci_build_swf_stack.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: "CI :: Build (SWF Stack)" - -on: - push: - branches: [main] - pull_request: - branches: ["**"] - types: [opened, reopened, ready_for_review, synchronize] - -concurrency: - group: ${{ github.event.pull_request && format('ci-build-swf-stack-full-pr-{0}', github.event.pull_request.number) || format('ci-build-swf-stack-full-push-main-{0}', github.sha) }} - cancel-in-progress: true - -env: - TMPDIR: "/tmp" - -jobs: - run: - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - name: "Checkout @ GitHub default" - uses: actions/checkout@v3 - - - name: "Checkout @ Simulated squashed-merge if PR" - id: checkout_pr - uses: ./.github/actions/checkout-pr - with: - ref: ${{ github.base_ref }} - - - name: "Setup CI patterns" - id: ci_patterns - uses: ./.github/actions/setup-ci-patterns - - - name: "Setup build mode {none,full}" - id: setup_build_mode - shell: bash - run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export PKGS_IN_SWF_STACK=("@kie-tools/serverless-logic-web-tools-swf-builder-image" "@kie-tools/kn-plugin-workflow" "@kie-tools/serverless-logic-web-tools" "@kie-tools/serverless-logic-web-tools-base-builder-image") - - npm install -g graph-data-structure@2.0.0 - export GREP_RES=$(NODE_PATH=$(npm prefix -g)/lib/node_modules node scripts/sparse-checkout/list_packages_dependencies.js ./repo "${PKGS_IN_SWF_STACK[@]}" | tr -d '\n' | xargs -d ' ' -I{} echo -n " -e \"{}\"") - echo "GREP RES:" - echo $GREP_RES - npm uninstall -g graph-data-structure - - export CHANGED_SOURCE_PATHS_IN_SWF_STACK=($(eval "printf '%s\\n' "${CHANGED_SOURCE_PATHS[@]}" | grep $GREP_RES")) - echo "Changed source paths in "SWF Stack":" - echo ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_SWF_STACK[@]} -eq 0 ]; then - echo 'No source files changed in "SWF Stack"; `CI :: SWF Stack` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - else - echo 'Source files changed in "SWF Stack"; `CI :: SWF Stack` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" - - - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/setup-env - - - name: "Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - PLAYWRIGHT_BASE__installDeps: "true" - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' -F='@kie-tools/kn-plugin-workflow...' -F='@kie-tools/serverless-logic-web-tools...' - - - name: "Build (only SWF Stack)" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - WEBPACK__minimize: "false" - WEBPACK__tsLoaderTranspileOnly: "false" - KIE_TOOLS_BUILD__runLinters: "true" - KIE_TOOLS_BUILD__runTests: "true" - KIE_TOOLS_BUILD__runEndToEndTests: "true" - KIE_TOOLS_BUILD__buildContainerImages: "true" - KIE_TOOLS_BUILD__buildExamples: "true" - KIE_TOOLS_BUILD__ignoreTestFailures: ${{ !github.event.pull_request }} - KIE_TOOLS_BUILD__ignoreEndToEndTestFailures: ${{ !github.event.pull_request }} - DISPLAY: ":99.0" - START_SERVER_AND_TEST_INSECURE: "true" - NODE_OPTIONS: "--max_old_space_size=4096" - run: >- - pnpm - -F='@kie-tools/serverless-logic-web-tools-swf-builder-image...' - -F='@kie-tools/serverless-logic-web-tools-base-builder-image...' - -F='@kie-tools/kn-plugin-workflow...' - -F='@kie-tools/serverless-logic-web-tools...' - --workspace-concurrency=1 build:prod - - - name: "Check tests result (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: actions/github-script@v6 - env: - KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS: |- - ${{ steps.ci_patterns.outputs.tests_reports_patterns }} - ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns }} - with: - result-encoding: string - script: | - const patterns = process.env["KIE_TOOLS_CI__JUNIT_REPORT_RESULTS_PATTERNS"] - .split("\n") - .map(p => p.trim()) - .filter(p => p); - - const script = require("./scripts/check-junit-report-results/src/index.js"); - await script({ core, glob, patterns }); - - - name: "Check hanging uncommitted files (you should commit those!)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - shell: bash - run: | - git diff - [ "0" == "$(git diff | wc -l | tr -d ' ')" ] - - - name: "Upload reports and artifacts" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/upload-ci-reports-and-artifacts - - - name: "Upload end-to-end tests results to Buildkite (`main` only)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' && !github.event.pull_request - shell: bash - env: - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} - BUILDKITE_BRANCH: ${{ github.ref_name }} - BUILDKITE_MESSAGE: ${{ github.event.commits[0].message }} - run: | - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" - echo "---------------------------- starting upload -----------------------" - eval "find -P * -type f ${{ steps.ci_patterns.outputs.end_to_end_tests_reports_patterns_for_find }}" | xargs -I{} curl -X POST \ - -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \ - -F "format=junit" \ - -F "data=@{}" \ - -F "run_env[CI]=github_actions" \ - -F "run_env[key]=$GITHUB_ACTION-$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" \ - -F "run_env[number]=$GITHUB_RUN_NUMBER" \ - -F "run_env[branch]=$BUILDKITE_BRANCH" \ - -F "run_env[commit_sha]=$GITHUB_SHA" \ - -F "run_env[message]=$BUILDKITE_MESSAGE" \ - -F "run_env[url]=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - https://analytics-api.buildkite.com/v1/uploads - - - name: "Print storage usage (after build)" - if: always() && !cancelled() - shell: bash - run: | - df -h . diff --git a/.github/workflows/ci_kogito_serverless_operator_e2e_tests.yaml b/.github/workflows/ci_kogito_serverless_operator_e2e_tests.yaml deleted file mode 100644 index 431d5be89ac..00000000000 --- a/.github/workflows/ci_kogito_serverless_operator_e2e_tests.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: "CI :: Kogito Serverless Operator :: E2E Tests" - -on: - push: - branches: [main] - pull_request: - branches: ["**"] - types: [opened, reopened, ready_for_review, synchronize] - -concurrency: - group: ${{ github.event.pull_request && format('ci-build-full-pr-kogito-serverless-operator-{0}', github.event.pull_request.number) || format('ci-build-full-push-main-kogito-serverless-operator-{0}', github.sha) }} - cancel-in-progress: true - -env: - TMPDIR: "/tmp" - -jobs: - run: - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - name: "Checkout @ GitHub default" - uses: actions/checkout@v3 - - - name: "Checkout @ Simulated squashed-merge if PR" - id: checkout_pr - uses: ./.github/actions/checkout-pr - with: - ref: ${{ github.base_ref }} - - - name: "Setup CI patterns" - id: ci_patterns - uses: ./.github/actions/setup-ci-patterns - - - name: "Setup build mode {none,full}" - id: setup_build_mode - shell: bash - run: | - export CHANGED_SOURCE_PATHS=($(eval "git diff --name-only ${{ steps.checkout_pr.outputs.base_sha }} ${{ steps.checkout_pr.outputs.head_sha }} -- ${{ steps.ci_patterns.outputs.non_source_files_patterns_for_git_diff }}")) - echo "Changed source paths:" - echo ${#CHANGED_SOURCE_PATHS[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS[@]}" - - export PKGS_IN_KOGITO_SERVERLESS_OPERATOR_STACK=("@kie-tools/kogito-serverless-operator") - - npm install -g graph-data-structure@2.0.0 - export GREP_RES=$(NODE_PATH=$(npm prefix -g)/lib/node_modules node scripts/sparse-checkout/list_packages_dependencies.js ./repo "${PKGS_IN_KOGITO_SERVERLESS_OPERATOR_STACK[@]}" | tr -d '\n' | xargs -d ' ' -I{} echo -n " -e \"{}\"") - echo "GREP RES:" - echo $GREP_RES - npm uninstall -g graph-data-structure - - export CHANGED_SOURCE_PATHS_IN_KOGITO_SERVERLESS_OPERATOR_OR_UPSTREAM=($(eval "printf '%s\\n' "${CHANGED_SOURCE_PATHS[@]}" | grep $GREP_RES")) - echo "Changed source paths in "@kie-tools/kogito-serverless-operator" or upstream:" - echo ${#CHANGED_SOURCE_PATHS_IN_KOGITO_SERVERLESS_OPERATOR_OR_UPSTREAM[@]} - printf '%s\n' "${CHANGED_SOURCE_PATHS_IN_KOGITO_SERVERLESS_OPERATOR_OR_UPSTREAM[@]}" - - if [ ${#CHANGED_SOURCE_PATHS[@]} -eq 0 ]; then - echo 'No source files changed; `CI :: Kogito Serverless Operator :: E2E Tests` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - elif [ ! ${{ github.event.pull_request }} ]; then - echo 'Push to the `main` branch happened; `CI :: Kogito Serverless Operator :: E2E Tests` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - elif [ ${#CHANGED_SOURCE_PATHS_IN_KOGITO_SERVERLESS_OPERATOR_OR_UPSTREAM[@]} -eq 0 ]; then - echo 'No source files changed in "@kie-tools/kogito-serverless-operator" or upstream; `CI :: Kogito Serverless Operator :: E2E Tests` (none) will run.' - echo "mode=none" >> $GITHUB_OUTPUT - else - echo 'Source files changed in "@kie-tools/kogito-serverless-operator" or upstream; `CI :: Kogito Serverless Operator :: E2E Tests` (full) will run.' - echo "mode=full" >> $GITHUB_OUTPUT - fi - - echo "Done" - - - name: "Setup environment" - if: steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/setup-env - - - name: "Bootstrap" - if: steps.setup_build_mode.outputs.mode == 'full' - uses: ./.github/actions/bootstrap - with: - pnpm_filter_string: -F "@kie-tools/kogito-serverless-operator..." - - - name: "Build upstream" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - KIE_TOOLS_BUILD__runLinters: "false" - KIE_TOOLS_BUILD__runTests: "false" - KIE_TOOLS_BUILD__runEndToEndTests: "false" - KIE_TOOLS_BUILD__buildContainerImages: "true" - run: | - pnpm -F "@kie-tools/kogito-serverless-operator^..." build:prod - - - name: "Build and test @kie-tools/kogito-serverless-operator" - if: steps.setup_build_mode.outputs.mode == 'full' - env: - KIE_TOOLS_BUILD__runLinters: "false" # No need.. the normal CI will do this. - KIE_TOOLS_BUILD__runTests: "false" # Only E2E! - KIE_TOOLS_BUILD__runEndToEndTests: "true" - KIE_TOOLS_BUILD__buildContainerImages: "true" - KOGITO_SERVERLESS_OPERATOR__runEndToEndTests: "true" - run: | - pnpm -F "@kie-tools/kogito-serverless-operator" build:prod - - - name: "Check hanging uncommitted files (you should commit those!)" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - shell: bash - run: | - git diff - [ "0" == "$(git diff | wc -l | tr -d ' ')" ] - - - name: "Upload reports and artifacts" - if: always() && !cancelled() && steps.setup_build_mode.outputs.mode != 'none' - uses: ./.github/actions/upload-ci-reports-and-artifacts - - - name: "Print storage usage (after build)" - if: always() && !cancelled() - shell: bash - run: | - df -h . diff --git a/package.json b/package.json index bbf4be36980..dfe0656b4ce 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,7 @@ "react-dropzone": "^11.4.2" }, "engines": { - "node": ">=18", - "pnpm": "8.7.0" + "node": ">=18" }, "packageManager": "pnpm@8.7.0", "pnpm": { diff --git a/packages/kogito-serverless-operator/env/index.js b/packages/kogito-serverless-operator/env/index.js index 404ee6dea93..d3de48848c7 100644 --- a/packages/kogito-serverless-operator/env/index.js +++ b/packages/kogito-serverless-operator/env/index.js @@ -17,7 +17,7 @@ * under the License. */ -const { varsWithName, composeEnv, getOrDefault, str2bool } = require("@kie-tools-scripts/build-env"); +const { varsWithName, composeEnv, getOrDefault } = require("@kie-tools-scripts/build-env"); module.exports = composeEnv([require("@kie-tools/root-env/env")], { vars: varsWithName({ @@ -37,10 +37,6 @@ module.exports = composeEnv([require("@kie-tools/root-env/env")], { default: "latest", description: "The image tag", }, - KOGITO_SERVERLESS_OPERATOR__runEndToEndTests: { - default: `${false}`, - description: "Boolean flag to enable or disable running E2E tests.", - }, }), get env() { return { @@ -49,9 +45,6 @@ module.exports = composeEnv([require("@kie-tools/root-env/env")], { account: getOrDefault(this.vars.KOGITO_SERVERLESS_OPERATOR__account), name: getOrDefault(this.vars.KOGITO_SERVERLESS_OPERATOR__name), tag: getOrDefault(this.vars.KOGITO_SERVERLESS_OPERATOR__buildTag), - endToEndTests: { - run: str2bool(getOrDefault(this.vars.KOGITO_SERVERLESS_OPERATOR__runEndToEndTests)), - }, version: require("../package.json").version, }, }; diff --git a/packages/kogito-serverless-operator/package.json b/packages/kogito-serverless-operator/package.json index 03e9ac2337c..f001fc03fc5 100644 --- a/packages/kogito-serverless-operator/package.json +++ b/packages/kogito-serverless-operator/package.json @@ -17,7 +17,7 @@ "build:dev:darwin:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && make build && pnpm image:build && pnpm format", "build:dev:win32": ".\\node_modules\\@kie-tools\\python-venv\\venv\\Scripts\\Activate.bat && make build && pnpm image:build && pnpm format", "build:prod": "run-script-os", - "build:prod:darwin:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && make build && pnpm image:build && pnpm test && pnpm test:e2e && pnpm format", + "build:prod:darwin:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && make build && pnpm image:build && pnpm test && pnpm test-e2e && pnpm format", "build:prod:win32": "echo 'Build not supported on Windows'", "bump-version": "run-script-os", "bump-version:darwin:linux": "./hack/bump-version.sh $(pnpm build-env kogitoServerlessOperator.version)", @@ -28,10 +28,10 @@ "image:build:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && run-script-if --bool \"$(build-env containerImages.build)\" --then \"make container-build\"", "install": "go mod tidy && pnpm bump-version && pnpm format", "test": "run-script-os", + "test-e2e": "run-script-os", + "test-e2e:darwin:win32": "echo 'E2E tests not supported on Windows and macOS'", + "test-e2e:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && run-script-if --ignore-errors \"$(build-env tests.ignoreFailures)\" --bool \"$(build-env endToEndTests.run)\" --then \"make full-test-e2e\"", "test:darwin:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && run-script-if --ignore-errors \"$(build-env tests.ignoreFailures)\" --bool \"$(build-env tests.run)\" --then \"make test\"", - "test:e2e": "run-script-os", - "test:e2e:darwin:win32": "echo 'E2E tests not supported on Windows and macOS'", - "test:e2e:linux": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && run-script-if --ignore-errors \"$(build-env tests.ignoreFailures)\" --bool \"$(build-env endToEndTests.run)\" --bool \"$(build-env kogitoServerlessOperator.endToEndTests.run)\" --then \"make full-test-e2e\"", "test:win32": ".\\node_modules\\@kie-tools\\python-venv\\venv\\Scripts\\Activate.bat && run-script-if --ignore-errors \"$(build-env tests.ignoreFailures)\" --bool \"$(build-env tests.run)\" --then \"make test\"" }, "dependencies": {}, diff --git a/packages/kogito-swf-common/resources/Makefile b/packages/kogito-swf-common/resources/Makefile index 438c87b5187..8c2303c2d4c 100644 --- a/packages/kogito-swf-common/resources/Makefile +++ b/packages/kogito-swf-common/resources/Makefile @@ -19,7 +19,7 @@ # Image build envs CEKIT_BUILD_OPTIONS ?= -CEKIT_CMD := cekit -v ${CEKIT_BUILD_OPTIONS} +CEKIT_CMD := cekit ${CEKIT_BUILD_OPTIONS} BUILD_ENGINE ?= docker BUILD_ENGINE_OPTIONS ?= diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2a960fe165..53cf51e6de1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13266,6 +13266,21 @@ importers: specifier: ^4.6.2 version: 4.8.4 + scripts/build-partitioning: + devDependencies: + "@types/bun": + specifier: ^1.1.0 + version: 1.1.0 + bun: + specifier: ^1.1.5 + version: 1.1.5 + graph-data-structure: + specifier: ^2.0.0 + version: 2.0.0 + typescript: + specifier: ^4.6.2 + version: 4.8.4 + scripts/check-junit-report-results: dependencies: fast-xml-parser: @@ -23727,6 +23742,78 @@ packages: engines: { node: ">=8.0" } dev: true + /@oven/bun-darwin-aarch64@1.1.5: + resolution: + { integrity: sha512-z0k3W2XEfa11OVUW0vp9pWLlpcPKY6TtpwPvREXCA0nFWj9LxbIRr5FZ1U6+M0gCRlx+5XOpSsWQ7+/HJcDgSg== } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64-baseline@1.1.5: + resolution: + { integrity: sha512-w3bHmbgrU0L8fCzW0uf0MYPo2DMekHG0w2SSQ+ZspkGQ11ppTu7FJZxZMAw8UQAAX3FgltFnaI1p6J9gkv4MMw== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-darwin-x64@1.1.5: + resolution: + { integrity: sha512-Pv1QDb69u6JMXfWiAE638aUWzsARGxWDQ1J1YLxOJVzbEe9f8qvMPZX7yP4OcfFMyc1ZcFrd7kDSkXQHZ/DeTg== } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-aarch64@1.1.5: + resolution: + { integrity: sha512-iObapm21/EsuQN3Z3m/DwAyjeGQaYuwC+KocPT29n7Tv+C/bubIEtikrmSGfA2CiSA5n6dFsFGk/t/uFfUuVMA== } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64-baseline@1.1.5: + resolution: + { integrity: sha512-ENUbLzO/tKpzUp04e9u+jZi+RFIpIJ/RQy7WKTstzce3fZqAsfh65CJE7TqKvFpHEDNxaRqJRg7yUonSCO1BmA== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-linux-x64@1.1.5: + resolution: + { integrity: sha512-KrvxD5sUf0WcdjVJm2LyGJqPzBldYWT4EQy+N5c5gzFnrEKSXOsj5lFE2vPNICS8Zvwzt8zLr0OwKq/WU93xTQ== } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64-baseline@1.1.5: + resolution: + { integrity: sha512-OCkUFyhbLR0kzAzuZg7pAa48TQFukYlaHTPH7ZWJkuS4ewFsDiYE+ET1BFn0LoRPlK/IhR8F6S21cH1elrlWCw== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@oven/bun-windows-x64@1.1.5: + resolution: + { integrity: sha512-kLRNRrQhCqXXoc0E6y9lBYbec6Sd7zIZtasoO6V7pYatMEaDB1ufIktmM5YVtCGYqNJ1+zNsENuU9xxPuprPDA== } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@patternfly/patternfly@4.224.2: resolution: { integrity: sha512-HGNV26uyHSIECuhjPg/WGn0mXbAotcs6ODfhAOkfYjIgGylddgiwElxUe1rpEHV5mQJJ2rMn4OdeJIIpzRX61g== } @@ -28998,6 +29085,13 @@ packages: "@types/node": 18.17.18 dev: true + /@types/bun@1.1.0: + resolution: + { integrity: sha512-QGK0yU4jh0OK1A7DyhPkQuKjHQCC5jSJa3dpWIEhHv/rPfb6zLfdArc4/uUUZBMTcjilsafRXnPWO+1owb572Q== } + dependencies: + bun-types: 1.1.0 + dev: true + /@types/cacheable-request@6.0.1: resolution: { integrity: sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ== } @@ -29720,6 +29814,13 @@ packages: resolution: { integrity: sha512-/4QOuy3ZpV7Ya1GTRz5CYSz3DgkKpyUptXuQ5PPce7uuyJAOR7r9FhkmxJfvcNUXyklbC63a+YvB3jxy7s9ngw== } + /@types/node@20.11.30: + resolution: + { integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw== } + dependencies: + undici-types: 5.26.5 + dev: true + /@types/normalize-package-data@2.4.0: resolution: { integrity: sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== } @@ -30091,6 +30192,13 @@ packages: source-map: 0.6.1 dev: true + /@types/ws@8.5.10: + resolution: + { integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A== } + dependencies: + "@types/node": 18.17.18 + dev: true + /@types/ws@8.5.5: resolution: { integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg== } @@ -33340,6 +33448,32 @@ packages: semver: 7.5.4 dev: true + /bun-types@1.1.0: + resolution: + { integrity: sha512-GhMDD7TosdJzQPGUOcQD5PZshvXVxDfwGAZs2dq+eSaPsRn3iUCzvpFlsg7Q51bXVzLAUs+FWHlnmpgZ5UggIg== } + dependencies: + "@types/node": 20.11.30 + "@types/ws": 8.5.10 + dev: true + + /bun@1.1.5: + resolution: + { integrity: sha512-9gVjgPLGQqKEAcuTL1x/hn7PUYvWHlx/tm3//xIRkJk55IVWXcmFM6kQtV+Jn9k5x3wvtAd8a3RRyk2alyKPjA== } + cpu: [arm64, x64] + os: [darwin, linux, win32] + hasBin: true + requiresBuild: true + optionalDependencies: + "@oven/bun-darwin-aarch64": 1.1.5 + "@oven/bun-darwin-x64": 1.1.5 + "@oven/bun-darwin-x64-baseline": 1.1.5 + "@oven/bun-linux-aarch64": 1.1.5 + "@oven/bun-linux-x64": 1.1.5 + "@oven/bun-linux-x64-baseline": 1.1.5 + "@oven/bun-windows-x64": 1.1.5 + "@oven/bun-windows-x64-baseline": 1.1.5 + dev: true + /busboy@1.6.0: resolution: { integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== } @@ -51785,6 +51919,11 @@ packages: resolution: { integrity: sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g== } + /undici-types@5.26.5: + resolution: + { integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== } + dev: true + /undoo@0.5.0: resolution: { integrity: sha512-SPlDcde+AUHoFKeVlH2uBJxqVkw658I4WR2rPoygC1eRCzm3GeoP8S6xXZVJeBVOQQid8X2xUBW0N4tOvvHH3Q== }