kie-issues#1490: Use turbo's diffing algorithm to know which packag…

…es are affected by changes to `pnpm-lock.yaml`, avoiding FULL builds (apache#2632)

.github/supporting-files/ci/build-partitioning/build_partitioning.ts

@@ -20,7 +20,7 @@
  */
 
 import { Partial, None, Full, PartitionDefinition } from "./types";
-import { __ROOT_PKG_NAME, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_DIRS, stdoutArray } from "./globals";
+import { __ROOT_PKG_NAME, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_PATHS, stdoutArray } from "./globals";
 import {
   assertLeafPackagesInPartitionDefinitionsDontOverlap,
   assertLeafPackagesInPartitionsExist,
@@ -56,7 +56,7 @@ const {
   allowPositionals: true,
 });
 
-const cwd = execSync("bash -c pwd").toString().trim();
+const absolutePosixRepoRootPath = execSync("bash -c pwd").toString().trim();
 
 const __ARG_forceFull = forceFull === "true";
 
@@ -142,9 +142,9 @@ async function getPartitions(): Promise<Array<None | Full | Partial>> {
   });
 
   const allPackageDirs = new Set(
-    stdoutArray(await execSync(`bash -c "pnpm -F !${__ROOT_PKG_NAME}... exec bash -c pwd"`).toString())
-      .map((pkgDir) => path.relative(cwd, pkgDir))
-      .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep))
+    stdoutArray(await execSync(`bash -c "pnpm -F !${__ROOT_PKG_NAME}... exec bash -c pwd"`).toString()).map((pkgDir) =>
+      convertToPosixPathRelativeToRepoRoot(pkgDir)
+    )
   );
 
   await assertCompleteness({ packageDirsByName, partitions: partitionDefinitions, allPackageDirs });
@@ -155,18 +155,31 @@ async function getPartitions(): Promise<Array<None | Full | Partial>> {
       `bash -c "git diff --name-only ${__ARG_baseSha} ${__ARG_headSha} -- ${nonSourceFilesPatternsForGitDiff}"`
     ).toString()
   );
+  const changedPackages: Array<{ path: string; name: string }> = JSON.parse(
+    execSync(`bash -c "turbo ls --filter='[${__ARG_baseSha}...${__ARG_headSha}]' --output json"`).toString()
+  ).packages.items.map((item: { path: string; name: string }) => ({
+    path: convertToPosixPathRelativeToRepoRoot(item.path),
+    name: item.name,
+  }));
+
+  const changedPackagesDirs = changedPackages.map((item) => item.path);
+
+  const changedPackagesNames = changedPackages.map((item) => item.name);
+
   console.log("[build-partitioning] Changed source paths:");
   console.log(new Set(changedSourcePaths));
 
   const changedSourcePathsInRoot = changedSourcePaths.filter((path) =>
-    __PACKAGES_ROOT_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`))
+    __PACKAGES_ROOT_PATHS.every((rootPath) => !path.startsWith(rootPath))
   );
 
-  const affectedPackageDirsInAllPartitions = stdoutArray(
-    await execSync(`pnpm -F ...[${__ARG_baseSha}] exec bash -c pwd`).toString()
+  const affectedPackageDirsInAllPartitions: Array<string> = await JSON.parse(
+    execSync(
+      `bash -c "turbo ls ${changedPackagesNames.map((packageName) => `--filter='...${packageName}'`).join(" ")} --output json"`
+    ).toString()
   )
-    .map((pkgDir) => path.relative(cwd, pkgDir))
-    .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep));
+    .packages.items.map((item: { path: string }) => item.path)
+    .map((pkgDir) => convertToPosixPathRelativeToRepoRoot(pkgDir));
 
   return await Promise.all(
     partitionDefinitions.map(async (partition) => {
@@ -183,8 +196,8 @@ async function getPartitions(): Promise<Array<None | Full | Partial>> {
         };
       }
 
-      const changedSourcePathsInPartition = changedSourcePaths.filter((path) =>
-        [...partition.dirs].some((partitionDir) => path.startsWith(`${partitionDir}/`))
+      const changedSourcePathsInPartition = changedPackagesDirs.filter((path) =>
+        [...partition.dirs].some((partitionDir) => path.startsWith(`${partitionDir}`))
       );
 
       if (changedSourcePathsInPartition.length === 0) {
@@ -204,12 +217,11 @@ async function getPartitions(): Promise<Array<None | Full | Partial>> {
       );
 
       const relevantPackageNamesInPartition = new Set(
-        [...(await getDirsOfDependencies(affectedPackageNamesInPartition, partition.name))].map(
+        [...(await getDirsOfDependencies(affectedPackageNamesInPartition))].map(
           (pkgDir) => packageNamesByDir.get(pkgDir)!
         )
       );
 
-      console.log(`[build-partitioning] 'Partial' build of '${partition.name}'`);
       console.log(
         `[build-partitioning] Building ${relevantPackageNamesInPartition.size}/${partition.dirs.size}/${allPackageDirs.size} packages.`
       );
@@ -240,8 +252,17 @@ async function getPartitions(): Promise<Array<None | Full | Partial>> {
 async function getDirsOfDependencies(leafPackageNames: Set<string>) {
   const packagesFilter = [...leafPackageNames].map((pkgName) => `-F ${pkgName}...`).join(" ");
   return new Set(
-    stdoutArray(execSync(`pnpm ${packagesFilter} exec bash -c pwd`).toString()) //
-      .map((pkgDir) => path.relative(cwd, pkgDir))
-      .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep))
+    stdoutArray(execSync(`bash -c "pnpm ${packagesFilter} exec bash -c pwd"`).toString()) //
+      .map((pkgDir) => convertToPosixPathRelativeToRepoRoot(pkgDir))
   );
 }
+
+function convertToPosixPathRelativeToRepoRoot(targetPath: string) {
+  // If it's not an absolute path we can assume it's relative to the repository root (the current directory).
+  const isTargetPathAbsolute = targetPath.startsWith(path.sep) || targetPath.startsWith(path.posix.sep);
+
+  // Replace separators to make sure they are posix style.
+  return `${isTargetPathAbsolute ? path.relative(absolutePosixRepoRootPath, targetPath) : targetPath}`
+    .split(path.sep)
+    .join(path.posix.sep);
+}

.github/supporting-files/ci/build-partitioning/globals.ts

@@ -22,7 +22,15 @@ import * as fs from "fs";
 
 export const __ROOT_PKG_NAME = "kie-tools-root";
 
-export const __PACKAGES_ROOT_DIRS = ["packages", "examples"];
+/**
+ * Root paths that can be ignored when checking individual changed files.
+ *
+ * `packages/` for the kie-tools packages, analysed by `turbo ls`.
+ * `examples/` for the kie-tools examples, analysed by `turbo ls`.
+ * `repo/` for the dependency graph. No PR should update the repository dependency graph by itself.
+ * `pnpm-lock.yaml` is the lockfile for the repository dependencies, also analysed by `turbo ls`.
+ */
+export const __PACKAGES_ROOT_PATHS = ["packages/", "examples/", "repo/", "pnpm-lock.yaml"];
 
 export const __NON_SOURCE_FILES_PATTERNS = stdoutArray(
   fs.readFileSync(path.resolve(__dirname, "../patterns/non-source-files-patterns.txt"), "utf-8")

.github/workflows/ci_build.yml

@@ -76,7 +76,11 @@ jobs:
         shell: bash
         run: |
 
-          npm -g install [email protected]
+          npm -g install [email protected] [email protected]
+
+          echo '{}' > turbo.json
+
+          turbo telemetry disable
 
           bun ./.github/supporting-files/ci/build-partitioning/build_partitioning.ts \
             --outputPath='/tmp/partitions.json' \
@@ -87,7 +91,9 @@ jobs:
             --partition='./.github/supporting-files/ci/partitions/partition0.txt' \
             --partition='./.github/supporting-files/ci/partitions/partition1.txt'
 
-          npm -g uninstall bun
+          npm -g uninstall bun turbo
+
+          rm turbo.json
 
           echo "mode=$(jq --raw-output '.[${{ matrix.partition }}].mode' /tmp/partitions.json)" >> $GITHUB_OUTPUT
           echo "bootstrapPnpmFilterString=$(jq --raw-output '.[${{ matrix.partition }}].bootstrapPnpmFilterString' /tmp/partitions.json)" >> $GITHUB_OUTPUT

.gitignore

@@ -336,3 +336,7 @@ packages/python-venv/venv
 
 # devbox
 .devbox
+
+# turbo
+.turbo
+turbo.json

devbox.json

@@ -16,7 +16,8 @@
     "DEVBOX_COREPACK_ENABLED": "true",
     "GOFLAGS": "-modcacherw",
     "GOPATH": "$DEVBOX_PROJECT_ROOT/.devbox/gopkgs",
-    "PATH": "$DEVBOX_PROJECT_ROOT/.devbox/gopkgs/bin:$PATH"
+    "PNPM_HOME": "$DEVBOX_PROJECT_ROOT/.devbox/pnpm-store",
+    "PATH": "$DEVBOX_PROJECT_ROOT/.devbox/gopkgs/bin:$DEVBOX_PROJECT_ROOT/.devbox/pnpm-store:$PATH"
   },
   "shell": {
     "init_hook": [