feat: support shared browser install for concurrent getBinary() calls

deno.jsonc

@@ -2,7 +2,7 @@
   "tasks": {
     // The task to automatically generate `./src/celestial.ts`
     "bind": "deno run -A ./bindings/_tools/generate/mod.ts && deno fmt",
-    "test": "deno test -A --trace-ops",
+    "test": "deno test -A --trace-ops --parallel",
     "bench": "deno bench -A",
     "www": "cd docs && pyro dev"
   },

src/browser.ts

@@ -7,6 +7,7 @@ import { WEBSOCKET_ENDPOINT_REGEX, websocketReady } from "./util.ts";
 
 async function runCommand(
   command: Deno.Command,
+  { retries = 60 } = {},
 ): Promise<{ process: Deno.ChildProcess; endpoint: string }> {
   const process = command.spawn();
   let endpoint = null;
@@ -45,6 +46,13 @@ async function runCommand(
   }
 
   if (error) {
+    const { code } = await process.status;
+    stack.push(`Process exited with code ${code}`);
+    // Handle recoverable error code 21 on Windows
+    // https://source.chromium.org/chromium/chromium/src/+/main:net/base/net_error_list.h;l=90-91
+    if ((Deno.build.os === "windows") && (code === 21) && retries > 0) {
+      return runCommand(command, { retries: retries - 1 });
+    }
     console.error(stack.join("\n"));
     throw new Error("Your binary refused to boot");
   }

src/cache.ts

@@ -5,7 +5,8 @@ import { dirname } from "https://deno.land/[email protected]/path/dirname.ts";
 import { join } from "https://deno.land/[email protected]/path/join.ts";
 import { ZipReader } from "https://deno.land/x/[email protected]/index.js";
 import ProgressBar from "https://deno.land/x/[email protected]/mod.ts";
-import { exists } from "https://deno.land/[email protected]/fs/exists.ts";
+import { exists, existsSync } from "https://deno.land/[email protected]/fs/exists.ts";
+import { retry } from "https://deno.land/[email protected]/async/retry.ts";
 
 export const SUPPORTED_VERSIONS = {
   chrome: "118.0.5943.0",
@@ -18,6 +19,8 @@ const HOME_PATH = Deno.build.os === "windows"
 const BASE_PATH = resolve(HOME_PATH, ".astral");
 const CONFIG_FILE = "cache.json";
 
+const LOCK_FILES = {} as { [cache: string]: { [product: string]: Lock } };
+
 interface KnownGoodVersions {
   timestamps: string;
   versions: {
@@ -53,6 +56,7 @@ function getCachedConfig({ cache = BASE_PATH }): Record<string, string> {
 export async function cleanCache({ cache = BASE_PATH } = {}) {
   try {
     if (await exists(cache)) {
+      delete LOCK_FILES[cache];
       await Deno.remove(cache, { recursive: true });
     }
   } catch (error) {
@@ -122,17 +126,22 @@ async function decompressArchive(source: string, destination: string) {
  */
 export async function getBinary(
   browser: "chrome" | "firefox",
-  { cache = BASE_PATH } = {},
+  { cache = BASE_PATH, timeout = 60000 } = {},
 ): Promise<string> {
   // TODO(lino-levan): fix firefox downloading
   const VERSION = SUPPORTED_VERSIONS[browser];
-
+  const product = `${browser}-${SUPPORTED_VERSIONS[browser]}`;
   const config = getCachedConfig({ cache });
 
+  // If the config doesn't have the revision and there is a lock file, reload config after release
+  if (!config[VERSION] && LOCK_FILES[cache]?.[product]?.exists()) {
+    await LOCK_FILES[cache]?.[product]?.waitRelease({ timeout });
+    Object.assign(config, getCachedConfig({ cache }));
+  }
+
   // If the config doesn't have the revision, download it and return that
   if (!config[VERSION]) {
     const quiet = await isQuietInstall();
-    ensureDirSync(cache);
     const versions = await knownGoodVersions();
     const version = versions.versions.filter((val) =>
       val.version === VERSION
@@ -152,47 +161,61 @@ export async function getBinary(
       );
     })[0];
 
-    const req = await fetch(download.url);
-    if (!req.body) {
-      throw new Error(
-        "Download failed, please check your internet connection and try again",
-      );
-    }
-    if (quiet) {
-      await Deno.writeFile(resolve(cache, `raw_${VERSION}.zip`), req.body);
-    } else {
-      const reader = req.body.getReader();
-      const archive = await Deno.open(resolve(cache, `raw_${VERSION}.zip`), {
-        write: true,
-        truncate: true,
-        create: true,
-      });
-      const bar = new ProgressBar({
-        title: `Downloading ${browser} ${VERSION}`,
-        total: Number(req.headers.get("Content-Length") ?? 0),
-        clear: true,
-        display: ":title :bar :percent",
-      });
-      let downloaded = 0;
-      do {
-        const { done, value } = await reader.read();
-        if (done) {
-          break;
-        }
-        await Deno.write(archive.rid, value);
-        downloaded += value.length;
-        bar.render(downloaded);
-      } while (true);
-      Deno.close(archive.rid);
-      console.log(`Download complete (${browser} version ${VERSION})`);
+    ensureDirSync(cache);
+    const lock = new Lock({ cache });
+    LOCK_FILES[cache] ??= {};
+    LOCK_FILES[cache][product] = lock;
+    if (!lock.create()) {
+      return getBinary(browser, { cache, timeout });
     }
-    await decompressArchive(
-      resolve(cache, `raw_${VERSION}.zip`),
-      resolve(cache, VERSION),
-    );
+    try {
+      const req = await fetch(download.url);
+      if (!req.body) {
+        throw new Error(
+          "Download failed, please check your internet connection and try again",
+        );
+      }
+      if (quiet) {
+        await Deno.writeFile(resolve(cache, `raw_${VERSION}.zip`), req.body);
+      } else {
+        const reader = req.body.getReader();
+        const archive = await Deno.open(resolve(cache, `raw_${VERSION}.zip`), {
+          write: true,
+          truncate: true,
+          create: true,
+        });
+        const bar = new ProgressBar({
+          title: `Downloading ${browser} ${VERSION}`,
+          total: Number(req.headers.get("Content-Length") ?? 0),
+          clear: true,
+          display: ":title :bar :percent",
+        });
+        let downloaded = 0;
+        do {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+          await Deno.write(archive.rid, value);
+          downloaded += value.length;
+          bar.render(downloaded);
+        } while (true);
+        Deno.close(archive.rid);
+        console.log(`Download complete (${browser} version ${VERSION})`);
+      }
+      await decompressArchive(
+        resolve(cache, `raw_${VERSION}.zip`),
+        resolve(cache, VERSION),
+      );
 
-    config[VERSION] = resolve(cache, VERSION);
-    Deno.writeTextFileSync(resolve(cache, CONFIG_FILE), JSON.stringify(config));
+      config[VERSION] = resolve(cache, VERSION);
+      Deno.writeTextFileSync(
+        resolve(cache, CONFIG_FILE),
+        JSON.stringify(config),
+      );
+    } finally {
+      LOCK_FILES[cache]?.[product]?.release();
+    }
   }
 
   // It now exists, return the path to the known good binary
@@ -225,3 +248,63 @@ export async function getBinary(
     "Unsupported platform, provide a path to a chromium or firefox binary instead",
   );
 }
+
+/**
+ * Create a lock file in cache
+ * Only the process with the same PID can release created lock file through this API
+ * TODO: Use Deno.flock/Deno.funlock when stabilized (https://deno.land/[email protected]?s=Deno.flock&unstable)
+ */
+class Lock {
+  readonly path;
+
+  constructor({ cache = BASE_PATH } = {}) {
+    this.path = resolve(cache, ".lock");
+  }
+
+  /** Returns true if lock file exists */
+  exists() {
+    return existsSync(this.path);
+  }
+
+  /** Create a lock file and returns true if it succeeds, false if it was already existing */
+  create() {
+    try {
+      Deno.writeTextFileSync(this.path, `${Deno.pid}`, { createNew: true });
+      return true;
+    } catch (error) {
+      if (!(error instanceof Deno.errors.AlreadyExists)) {
+        throw error;
+      }
+      return false;
+    }
+  }
+
+  /** Release lock file */
+  release() {
+    try {
+      if (Deno.readTextFileSync(this.path) === `${Deno.pid}`) {
+        Deno.removeSync(this.path);
+      }
+    } catch (error) {
+      if (!(error instanceof Deno.errors.NotFound)) {
+        throw error;
+      }
+    }
+  }
+
+  /** Wait for lock release */
+  async waitRelease({ timeout = 60000 } = {}) {
+    await retry(() => {
+      if (this.exists()) {
+        throw new Error(
+          `Timeout while waiting for lockfile release at: ${this.path}`,
+        );
+      }
+    }, {
+      maxTimeout: timeout,
+      maxAttempts: Infinity,
+      multiplier: 1,
+      minTimeout: 100,
+    });
+  }
+}

tests/_get_binary_test.ts → tests/get_binary_test.ts

@@ -9,7 +9,15 @@ import { assertStringIncludes } from "https://deno.land/[email protected]/assert/asser
 Deno.env.set("ASTRAL_QUIET_INSTALL", "true");
 const cache = await Deno.makeTempDir({ prefix: "astral_test_get_binary" });
 const permissions = {
-  write: [cache],
+  write: [
+    cache,
+    // Chromium lock on Linux
+    `${Deno.env.get("HOME")}/.config/chromium/SingletonLock`,
+    // Chromium lock on MacOS
+    `${
+      Deno.env.get("HOME")
+    }/Library/Application Support/Chromium/SingletonLock`,
+  ],
   read: [cache],
   net: true,
   env: true,
@@ -61,5 +69,6 @@ Deno.test("Test download after failure", { permissions }, async () => {
   assert(await getBinary("chrome", { cache: testCache }));
 });
 
-// Cleaning
-await Deno.remove(cache, { recursive: true });
+Deno.test("Clean cache after tests", async () => {
+  await cleanCache({ cache });
+});

tests/install_lock_file_test.ts

@@ -0,0 +1,31 @@
+import { cleanCache, getBinary, launch } from "../mod.ts";
+import { deadline } from "https://deno.land/[email protected]/async/deadline.ts";
+// Tests should be performed in directory different from others tests as cache is cleaned during this one
+//Deno.env.set("ASTRAL_QUIET_INSTALL", "true");
+const cache = await Deno.makeTempDir({
+  prefix: "astral_test_install_lock_file",
+});
+
+Deno.test("Test concurrent getBinary calls", async () => {
+  // Spawn concurrent getBinary calls
+  await cleanCache({ cache });
+  const promises = [];
+  for (let i = 0; i < 20; i++) {
+    promises.push(getBinary("chrome", { cache }));
+  }
+  const path = await Promise.race(promises);
+
+  // Ensure binary sent by first promise is executable
+  const browser = await launch({ path });
+
+  // Other promises should resolve at around the same time as they wait for lock file
+  await deadline(Promise.all(promises), 250);
+
+  // Ensure binary is still working (no files overwritten)
+  await browser.newPage("https://example.com");
+  await browser.close();
+});
+
+Deno.test("Clean cache after tests", async () => {
+  await cleanCache({ cache });
+});