forked from apache/incubator-kie-tools
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
8b34a43
commit 3efd639
Showing
22 changed files
with
680 additions
and
364 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # Build partitioning | ||
|
|
||
| This script uses `bun` as runtime and has the purpose of splitting the monorepo build in 2 partitions that, when built, will ensure the entire monorepo has been covered. | ||
|
|
||
| Each partition is completely independent from each other, and partition builds will be assigned a mode: | ||
|
|
||
| - `none`: No source code changes or none of the changes affect this partition. | ||
| - `full`: All packages in the partition need to be built and tested. | ||
| - `partial`: Only selected packages of the partition were affected by the changes, so only this subset needs to be rebuilt and tested. | ||
|
|
||
| ### Usage | ||
|
|
||
| At the root of the monorepo, you can run | ||
|
|
||
| ```bash | ||
| npx bun .github/supporting-files/ci/build-partitioning/build_partitioning.ts \ | ||
| --outputPath='/tmp/partitions.json' \ | ||
| --forceFull="false" \ | ||
| --baseSha="$(git rev-parse HEAD~1)" \ | ||
| --headSha="$(git rev-parse HEAD~0)" \ | ||
| --graphJsonPath='./repo/graph.json' | ||
| ``` | ||
|
|
||
| - `--outputPath`: Where the resulting JSON should be written to. | ||
| - `--forceFull`: Wheter or not to force `full` mode | ||
| - `--baseSha`: Base Git commit SHA. | ||
| - `--headSha`: HEAD Git commit SHA. | ||
| - `--graphJsonPath`: Where to find the `graph.json` of the monorepo in Datavis graph format. |
119 changes: 119 additions & 0 deletions
119
.github/supporting-files/ci/build-partitioning/assertions.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| import { PartitionDefinition } from "./types"; | ||
|
|
||
| export async function assertLeafPackagesInPartitionsExist({ | ||
| packageNames, | ||
| allLeafPackages, | ||
| }: { | ||
| packageNames: string[]; | ||
| allLeafPackages: Set<string>; | ||
| }) { | ||
| const nonLeafPackagesInPartitions = new Set(packageNames.filter((l) => !allLeafPackages.has(l))); | ||
| console.log( | ||
| `[build-partitioning] Partition definitions only use leaf packages (${ | ||
| nonLeafPackagesInPartitions.size > 0 ? "❌" : "✅" | ||
| }):` | ||
| ); | ||
| if (nonLeafPackagesInPartitions.size > 0) { | ||
| console.error(`[build-partitioning] Non-leaf packages found in partition definitions. Aborting.`); | ||
| console.error(nonLeafPackagesInPartitions); | ||
| process.exit(1); | ||
| } | ||
| } | ||
|
|
||
| export async function assertLeafPackagesInPartitionDefinitionsDontOverlap({ | ||
| allLeafPackages, | ||
| partitionDefinitions, | ||
| }: { | ||
| allLeafPackages: Set<string>; | ||
| partitionDefinitions: PartitionDefinition[]; | ||
| }) { | ||
| const leafPackagesOverlap = partitionDefinitions.flatMap((s) => [...s.leafPackageNames]); | ||
| const hasPartitionOverlap = allLeafPackages.size !== leafPackagesOverlap.length; | ||
|
|
||
| console.log(`[build-partitioning] Partition definitions don't overlap (${hasPartitionOverlap ? "❌" : "✅"}):`); | ||
| if (hasPartitionOverlap) { | ||
| console.error(`[build-partitioning] Partitions definitions declare overlapping leaf packages. Aborting.`); | ||
| console.error(leafPackagesOverlap); | ||
| process.exit(1); | ||
| } | ||
| } | ||
|
|
||
| export async function assertCompleteness({ | ||
| packageDirsByName, | ||
| partitions, | ||
| allPackageDirs, | ||
| }: { | ||
| packageDirsByName: Map<string, string>; | ||
| partitions: PartitionDefinition[]; | ||
| allPackageDirs: Set<string>; | ||
| }) { | ||
| const partitionsByPkgDir = new Map<string, string[]>(); | ||
| for (const p of partitions) { | ||
| for (const pkgDir of p.dirs) { | ||
| partitionsByPkgDir.set(pkgDir, [...(partitionsByPkgDir.get(pkgDir) ?? []), p.name]); | ||
| } | ||
| } | ||
|
|
||
| const redundancyOnPartitionCombinations = new Map<string, string[]>(); | ||
| partitionsByPkgDir.forEach((partitions, pkgDir) => { | ||
| if (partitions.length > 1) { | ||
| // We're only interested in packages that are going to be built as part of more than one partition. | ||
| const key = partitions.sort().join(" & "); | ||
| redundancyOnPartitionCombinations.set(key, [...(redundancyOnPartitionCombinations.get(key) ?? []), pkgDir]); | ||
| } | ||
| }); | ||
|
|
||
| console.log(`[build-partitioning] Packages that will be built by more than one partition:`); | ||
| console.log(redundancyOnPartitionCombinations); | ||
|
|
||
| const completenessCheck = [...allPackageDirs].filter((pkgDir) => !partitionsByPkgDir.has(pkgDir)).length <= 0; | ||
| console.log(`[build-partitioning] Partition definitions completeness (${!completenessCheck ? "❌" : "✅"}):`); | ||
| if (!completenessCheck) { | ||
| console.error(`[build-partitioning] All packages count: ${allPackageDirs.size}`); | ||
| for (const p of partitions) { | ||
| console.error(`[build-partitioning] ${p.name} packages count: ${p.dirs.size}`); | ||
| } | ||
| process.exit(1); | ||
| } | ||
| return allPackageDirs; | ||
| } | ||
|
|
||
| export async function assertOptimalPartialBuild(args: { | ||
| partition: PartitionDefinition; | ||
| upstreamPackageNamesInPartition: Set<string>; | ||
| affectedPackageNamesInPartition: Set<string>; | ||
| relevantPackageNamesInPartition: Set<string>; | ||
| }) { | ||
| const isOptimalPartialPartitionBuild = | ||
| args.upstreamPackageNamesInPartition.size + args.affectedPackageNamesInPartition.size === | ||
| args.relevantPackageNamesInPartition.size; | ||
|
|
||
| console.log( | ||
| `[build-partitioning] 'Partial' build of '${args.partition.name}': Optimal build check ((${ | ||
| !isOptimalPartialPartitionBuild ? "❌" : "✅" | ||
| }))` | ||
| ); | ||
| if (!isOptimalPartialPartitionBuild) { | ||
| console.error(`[build-partitioning] Non-optimal 'Partial' build. Aborting.`); | ||
| process.exit(1); | ||
| } | ||
| } |
Oops, something went wrong.