Test

.github/actions/setup-ci-patterns/action.yml

@@ -54,9 +54,9 @@ runs:
 
           const pnpmWorkspacePackagesRootPaths = ["packages", "examples"].map(p => path.join(cwd, p)).join("\n");
 
-          const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/non-source-files-patterns.txt"), "utf-8");
+          const nonSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/non-source-files-patterns.txt"), "utf-8");
           const nonSourceFilesPatternsForGitDiff = nonSourceFilesPatterns.split("\n").filter(p => p.trim() !== "").map(p => `':!${p}'`).join(" ");
-          const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/tests-source-files-patterns.txt"), "utf-8");
+          const testSourceFilesPatterns = fs.readFileSync(path.join(cwd, "./.github/supporting-files/ci/patterns/tests-source-files-patterns.txt"), "utf-8");
 
           core.setOutput("pnpm_workspace_packages_root_paths", pnpmWorkspacePackagesRootPaths);
           core.setOutput("non_source_files_patterns", nonSourceFilesPatterns);
@@ -78,14 +78,14 @@ runs:
             core.setOutput(outputName, patterns);
           }
 
-          await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/tests-reports-patterns.txt"));
-          await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt"));
-          await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/end-to-end-tests-artifacts-patterns.txt"));
-          await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/build-artifacts-patterns.txt"));
+          await outputPatternsPrefixedWithRoots("tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/tests-reports-patterns.txt"));
+          await outputPatternsPrefixedWithRoots("end_to_end_tests_reports_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt"));
+          await outputPatternsPrefixedWithRoots("end_to_end_tests_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/end-to-end-tests-artifacts-patterns.txt"));
+          await outputPatternsPrefixedWithRoots("build_artifacts_patterns", path.join(cwd, "./.github/supporting-files/ci/patterns/build-artifacts-patterns.txt"));
 
           core.setOutput(
             "end_to_end_tests_reports_patterns_for_find", 
-            prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/end-to-end-tests-reports-patterns.txt", "utf-8"))
+            prefixWithRoots(fs.readFileSync("./.github/supporting-files/ci/patterns/end-to-end-tests-reports-patterns.txt", "utf-8"))
               .split("\n")
               .map(p => `-path '${p}'`)
               .join(" -o ")

.github/supporting-files/ci/build-partitioning/assertions.ts

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { PartitionDefinition } from "./types";
+
+export async function assertLeafPackagesInPartitionsExist({
+  packageNames,
+  allLeafPackages,
+}: {
+  packageNames: string[];
+  allLeafPackages: Set<string>;
+}) {
+  const nonLeafPackagesInPartitions = new Set(packageNames.filter((l) => !allLeafPackages.has(l)));
+  console.log(`[build-partitioning] Leaf check (${nonLeafPackagesInPartitions.size > 0 ? "❌" : "✅"}):`);
+  if (nonLeafPackagesInPartitions.size > 0) {
+    console.error(`[build-partitioning] Non-leaf packages found in partition definitions. Aborting.`);
+    console.error(nonLeafPackagesInPartitions);
+    process.exit(1);
+  }
+}
+
+export async function assertLeafPackagesInPartitionDefinitionsDontOverlap({
+  allLeafPackages,
+  p0,
+  p1,
+}: {
+  allLeafPackages: Set<string>;
+  p0: Set<string>;
+  p1: Set<string>;
+}) {
+  const leafPackagesOverlap = [...allLeafPackages].filter((leaf) => p0.has(leaf) && p1.has(leaf));
+  const hasPartitionOverlap = leafPackagesOverlap.length > 0;
+
+  console.log(`[build-partitioning] Overlap check (${hasPartitionOverlap ? "❌" : "✅"}):`);
+  if (hasPartitionOverlap) {
+    console.error(`[build-partitioning] Partitions overlap. Aborting.`);
+    console.error(leafPackagesOverlap);
+    process.exit(1);
+  }
+}
+
+export async function assertCompleteness({
+  packageDirsByName,
+  partitions,
+  allPackageDirs,
+}: {
+  packageDirsByName: Map<string, string>;
+  partitions: PartitionDefinition[];
+  allPackageDirs: Set<string>;
+}) {
+  const redundantPackageNames = new Set(
+    [...packageDirsByName.entries()]
+      .filter(([pkgName, pkgDir]) => partitions[0].dirs.has(pkgDir) && partitions[1].dirs.has(pkgDir))
+      .map(([pkgName, pkgDir]) => pkgName)
+  );
+  console.log(`[build-partitioning] Redundancy:`);
+  console.log(redundantPackageNames);
+
+  const completenessCheck =
+    partitions[0].dirs.size + partitions[1].dirs.size - (partitions.length - 1) * redundantPackageNames.size ===
+    allPackageDirs.size;
+  console.log(`[build-partitioning] Completeness check (${!completenessCheck ? "❌" : "✅"}):`);
+  if (!completenessCheck) {
+    console.error(`[build-partitioning] All packages count: ${allPackageDirs.size}`);
+    console.error(`[build-partitioning] ${partitions[0].name} packages count: ${partitions[0].dirs.size}`);
+    console.error(`[build-partitioning] ${partitions[1].name} packages count: ${partitions[1].dirs.size}`);
+    console.error(`[build-partitioning] Redundant packages count: ${redundantPackageNames.size}`);
+    process.exit(1);
+  }
+  return allPackageDirs;
+}
+
+export async function assertOptimalPartialBuild(args: {
+  partition: PartitionDefinition;
+  upstreamPackageNamesInPartition: Set<string>;
+  affectedPackageNamesInPartition: Set<string>;
+  relevantPackageNamesInPartition: Set<string>;
+}) {
+  const isOptimalPartialPartitionBuild =
+    args.upstreamPackageNamesInPartition.size + args.affectedPackageNamesInPartition.size ===
+    args.relevantPackageNamesInPartition.size;
+
+  console.log(
+    `[build-partitioning] 'Partial' build of '${args.partition.name}': Optimal build check ((${
+      !isOptimalPartialPartitionBuild ? "❌" : "✅"
+    }))`
+  );
+  if (!isOptimalPartialPartitionBuild) {
+    console.error(`[build-partitioning] Non-optimal 'Partial' build. Aborting.`);
+    process.exit(1);
+  }
+}

.github/supporting-files/ci/build-partitioning/build_partitioning.ts

@@ -0,0 +1,229 @@
+#!/usr/bin/env bun
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Partial, None, Full, PartitionDefinition } from "./types";
+import { __ROOT_PKG_NAME, __P0, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_DIRS } from "./globals";
+import {
+  assertLeafPackagesInPartitionDefinitionsDontOverlap,
+  assertLeafPackagesInPartitionsExist,
+  assertCompleteness,
+  assertOptimalPartialBuild,
+} from "./assertions";
+
+import { $ } from "bun";
+import { parseArgs } from "util";
+import * as path from "path";
+import * as fs from "fs";
+
+const {
+  values: {
+    outputPath: __ARG_outputPath,
+    forceFull: forceFull,
+    baseSha: __ARG_baseSha,
+    headSha: __ARG_headSha,
+    graphJsonPath: __ARG_graphJsonPath,
+  },
+} = parseArgs({
+  args: Bun.argv,
+  options: {
+    graphJsonPath: { type: "string" },
+    baseSha: { type: "string" },
+    headSha: { type: "string" },
+    forceFull: { type: "string" },
+    outputPath: { type: "string" },
+  },
+  strict: true,
+  allowPositionals: true,
+});
+
+const __ARG_forceFull = forceFull === "true";
+
+const partitions = await getPartitions();
+const partitionsJson = JSON.stringify(partitions, null, 2);
+console.log(``);
+console.log(`[build-partitioning] --- PARTITIONS JSON ---`);
+console.log(partitionsJson);
+
+const resolvedOutputPath = path.resolve(".", __ARG_outputPath!);
+fs.writeFileSync(resolvedOutputPath, partitionsJson);
+console.log(`[build-partitioning] --> Written to '${resolvedOutputPath}'`);
+console.log(`[build-partitioning] Done.`);
+
+process.exit(0);
+
+//
+
+async function getPartitions(): Promise<Array<None | Full | Partial>> {
+  console.log(``);
+  console.log(`[build-partitioning] --- Summary ---`);
+  console.log(`[build-partitioning] graphJsonPath: ${__ARG_graphJsonPath}`);
+  console.log(`[build-partitioning] baseSha:       ${__ARG_baseSha}`);
+  console.log(`[build-partitioning] headSha:       ${__ARG_headSha}`);
+  console.log(`[build-partitioning] forceFull:     ${forceFull}`);
+  console.log(`[build-partitioning] outputPath:    ${__ARG_outputPath}`);
+  console.log(`[build-partitioning] ---------------`);
+  console.log(``);
+
+  const graphJson = await import(path.resolve(".", __ARG_graphJsonPath!));
+  const packageDirsByName = new Map<string, string>(graphJson.serializedPackagesLocationByName);
+  const packageNamesByDir = new Map([...packageDirsByName.entries()].map(([k, v]) => [v, k]));
+
+  const allLeafPackages = new Set(packageDirsByName.keys());
+  for (const link of graphJson.serializedDatavisGraph.links) {
+    allLeafPackages.delete(link.target);
+  }
+  allLeafPackages.delete(__ROOT_PKG_NAME);
+
+  console.log(`[build-partitioning] All leaf packages:`);
+  console.log(allLeafPackages);
+
+  console.log(`[build-partitioning] P0:`);
+  console.log(__P0);
+
+  const p1 = new Set([...allLeafPackages].filter((leaf) => !__P0.has(leaf)));
+  console.log(`[build-partitioning] P1:`);
+  console.log(p1);
+
+  const partitionDefinitions: PartitionDefinition[] = [
+    {
+      name: "Partition 0",
+      leafPackageNames: __P0,
+      dirs: await getDirsOfDependencies(__P0),
+    },
+    {
+      name: "Partition 1",
+      leafPackageNames: p1,
+      dirs: await getDirsOfDependencies(p1),
+    },
+  ];
+
+  await assertLeafPackagesInPartitionDefinitionsDontOverlap({ allLeafPackages, p0: __P0, p1: p1 });
+  await assertLeafPackagesInPartitionsExist({
+    packageNames: partitionDefinitions.flatMap((p) => [...p.leafPackageNames]),
+    allLeafPackages,
+  });
+
+  const allPackageDirs = new Set(
+    outputArray(await $`pnpm -F='!${__ROOT_PKG_NAME}...' exec pwd`.text())
+      .map((s) => path.relative(".", s))
+      .map((pkgDir) => packageNamesByDir.get(pkgDir)!)
+  );
+
+  await assertCompleteness({ packageDirsByName, partitions: partitionDefinitions, allPackageDirs });
+
+  const nonSourceFilesPatternsForGitDiff = __NON_SOURCE_FILES_PATTERNS.map((p) => `':!${p}'`).join(" ");
+  console.log(nonSourceFilesPatternsForGitDiff);
+  // TODO: Use these patterns
+  const changedSourcePaths = outputArray(
+    await new Response(
+      Bun.spawnSync(`git diff --name-only ${__ARG_baseSha} ${__ARG_headSha} -- ${""}`.split(" ")).stdout
+    ).text()
+  );
+  console.log("[build-partitioning] Changed source paths:");
+  console.log(new Set(changedSourcePaths));
+
+  const changedSourcePathsInRoot = changedSourcePaths.filter((path) =>
+    __PACKAGES_ROOT_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`))
+  );
+
+  const relevantPackageDirsInAllPartitions = outputArray(await $`pnpm -F=...[${__ARG_baseSha}]... exec pwd`.text());
+  const affectedPackageDirsInAllPartitions = outputArray(await $`pnpm -F=...[${__ARG_baseSha}] exec pwd`.text());
+
+  return await Promise.all(
+    partitionDefinitions.map(async (partition) => {
+      if (__ARG_forceFull || changedSourcePathsInRoot.length > 0) {
+        console.log(`[build-partitioning] 'Full' build of '${partition.name}'.`);
+        console.log(
+          `[build-partitioning] Building ${partition.dirs.size}/${partition.dirs.size}/${allPackageDirs.size} packages.`
+        );
+        return {
+          mode: "full",
+          name: partition.name,
+          bootstrapPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F='${l}...'`).join(" "),
+          fullBuildPnpmFilterString: [...partition.leafPackageNames].map((l) => `-F='${l}...'`).join(" "),
+        };
+      }
+
+      const changedSourcePathsInPartition = changedSourcePaths.filter((path) =>
+        [...partition.dirs].some((partitionDir) => path.startsWith(partitionDir))
+      );
+      if (changedSourcePathsInPartition.length === 0) {
+        console.log(`[build-partitioning] 'None' build of '${partition.name}'.`);
+        console.log(`[build-partitioning] Building 0/${partition.dirs.size}/${allPackageDirs.size} packages.`);
+        console.log(``);
+        return {
+          mode: "none",
+          name: partition.name,
+        };
+      }
+
+      const affectedPackageNamesInPartition = new Set(
+        affectedPackageDirsInAllPartitions
+          .map((pkgDir) => path.relative(".", pkgDir))
+          .filter((pkgDir) => partition.dirs.has(pkgDir))
+          .map((packageDir) => packageNamesByDir.get(packageDir)!)
+      );
+
+      const relevantPackageNamesInPartition = new Set(
+        [...(await getDirsOfDependencies(affectedPackageNamesInPartition))]
+          .map((pkgDir) => path.relative(".", pkgDir))
+          .map((pkgDir) => packageNamesByDir.get(pkgDir)!)
+      );
+
+      console.log(`[build-partitioning] 'Partial' build of '${partition.name}'`);
+      console.log(
+        `[build-partitioning] Building ${relevantPackageNamesInPartition.size}/${relevantPackageDirsInAllPartitions.length}/${allPackageDirs.size} packages.`
+      );
+      console.log(relevantPackageNamesInPartition);
+
+      const upstreamPackageNamesInPartition = new Set(
+        [...relevantPackageNamesInPartition].filter((pkgName) => !affectedPackageNamesInPartition.has(pkgName))
+      );
+
+      await assertOptimalPartialBuild({
+        partition,
+        relevantPackageNamesInPartition,
+        upstreamPackageNamesInPartition,
+        affectedPackageNamesInPartition,
+      });
+
+      return {
+        mode: "partial",
+        name: partition.name,
+        bootstrapPnpmFilterString: [...relevantPackageNamesInPartition].map((p) => `-F='${p}'`).join(" "),
+        upstreamPnpmFilterString: [...upstreamPackageNamesInPartition].map((p) => `-F='${p}'`).join(" "),
+        affectedPnpmFilterString: [...affectedPackageNamesInPartition].map((p) => `-F='...${p}'`).join(" "),
+      };
+    })
+  );
+}
+
+async function getDirsOfDependencies(leafPackageNames: Set<string>) {
+  const packagesFilter = [...leafPackageNames].map((pkgName) => `-F=${pkgName}...`).join(" ");
+  return new Set(
+    outputArray(await new Response(Bun.spawnSync(`pnpm ${packagesFilter} exec pwd`.split(" ")).stdout).text()) //
+      .map((pkgDir) => path.relative(".", pkgDir))
+  );
+}
+
+function outputArray(output: string) {
+  return output.trim().split(/\s/);
+}