feat: auto browser download (#1029)

feat: auto browser download

package.json

@@ -54,6 +54,7 @@
     "@babel/code-frame": "7.24.2",
     "@gemini-testing/commander": "2.15.4",
     "@jspm/core": "2.0.1",
+    "@puppeteer/browsers": "2.4.0",
     "@types/debug": "4.1.12",
     "@types/yallist": "4.0.4",
     "@vitest/spy": "2.1.4",
@@ -65,12 +66,16 @@
     "bluebird": "3.5.1",
     "chalk": "2.4.2",
     "clear-require": "1.0.1",
+    "cli-progress": "3.12.0",
     "debug": "2.6.9",
     "devtools": "8.39.0",
+    "edgedriver": "5.6.1",
     "error-stack-parser": "2.1.4",
     "expect-webdriverio": "3.6.0",
+    "extract-zip": "2.0.1",
     "fastq": "1.13.0",
     "fs-extra": "5.0.0",
+    "geckodriver": "4.5.0",
     "gemini-configparser": "1.4.1",
     "get-port": "5.1.1",
     "glob-extra": "5.0.2",
@@ -96,6 +101,7 @@
     "urijs": "1.19.11",
     "url-join": "4.0.1",
     "vite": "5.1.6",
+    "wait-port": "1.1.0",
     "webdriverio": "8.39.0",
     "worker-farm": "1.7.0",
     "yallist": "3.1.1"
@@ -118,6 +124,7 @@
     "@types/chai": "4.3.4",
     "@types/chai-as-promised": "7.1.5",
     "@types/clear-require": "3.2.1",
+    "@types/cli-progress": "3.11.6",
     "@types/escape-string-regexp": "2.0.1",
     "@types/fs-extra": "11.0.4",
     "@types/lodash": "4.14.191",

src/browser-installer/chrome/browser.ts

@@ -0,0 +1,61 @@
+import { resolveBuildId, canDownload, install as puppeteerInstall } from "@puppeteer/browsers";
+import { MIN_CHROME_FOR_TESTING_VERSION } from "../constants";
+import {
+    browserInstallerDebug,
+    getBrowserPlatform,
+    getBrowsersDir,
+    getMilestone,
+    Browser,
+    type DownloadProgressCallback,
+} from "../utils";
+import { getBinaryPath, getMatchedBrowserVersion, installBinary } from "../registry";
+import { normalizeChromeVersion } from "../utils";
+
+export const installChrome = async (version: string, { force = false } = {}): Promise<string> => {
+    const milestone = getMilestone(version);
+
+    if (Number(milestone) < MIN_CHROME_FOR_TESTING_VERSION) {
+        browserInstallerDebug(`couldn't install chrome@${version}, installing chromium instead`);
+
+        const { installChromium } = await import("../chromium");
+
+        return installChromium(version, { force });
+    }
+
+    const platform = getBrowserPlatform();
+    const existingLocallyBrowserVersion = getMatchedBrowserVersion(Browser.CHROME, platform, version);
+
+    if (existingLocallyBrowserVersion && !force) {
+        browserInstallerDebug(`A locally installed chrome@${version} browser was found. Skipping the installation`);
+
+        return getBinaryPath(Browser.CHROME, platform, existingLocallyBrowserVersion);
+    }
+
+    const normalizedVersion = normalizeChromeVersion(version);
+    const buildId = await resolveBuildId(Browser.CHROME, platform, normalizedVersion);
+
+    const cacheDir = getBrowsersDir();
+    const canBeInstalled = await canDownload({ browser: Browser.CHROME, platform, buildId, cacheDir });
+
+    if (!canBeInstalled) {
+        throw new Error(
+            [
+                `chrome@${version} can't be installed.`,
+                `Probably the version '${version}' is invalid, please try another version.`,
+                "Version examples: '120', '120.0'",
+            ].join("\n"),
+        );
+    }
+
+    const installFn = (downloadProgressCallback: DownloadProgressCallback): Promise<string> =>
+        puppeteerInstall({
+            platform,
+            buildId,
+            cacheDir,
+            downloadProgressCallback,
+            browser: Browser.CHROME,
+            unpack: true,
+        }).then(result => result.executablePath);
+
+    return installBinary(Browser.CHROME, platform, buildId, installFn);
+};

src/browser-installer/chrome/driver.ts

@@ -0,0 +1,63 @@
+import { resolveBuildId, install as puppeteerInstall, canDownload } from "@puppeteer/browsers";
+import { MIN_CHROMEDRIVER_FOR_TESTING_VERSION } from "../constants";
+import {
+    browserInstallerDebug,
+    getBrowserPlatform,
+    getChromeDriverDir,
+    getMilestone,
+    Driver,
+    type DownloadProgressCallback,
+} from "../utils";
+import { getBinaryPath, getMatchedDriverVersion, installBinary } from "../registry";
+
+export const installChromeDriver = async (chromeVersion: string, { force = false } = {}): Promise<string> => {
+    const platform = getBrowserPlatform();
+    const existingLocallyDriverVersion = getMatchedDriverVersion(Driver.CHROMEDRIVER, platform, chromeVersion);
+
+    if (existingLocallyDriverVersion && !force) {
+        browserInstallerDebug(
+            `A locally installed chromedriver for chrome@${chromeVersion} was found. Skipping the installation`,
+        );
+
+        return getBinaryPath(Driver.CHROMEDRIVER, platform, existingLocallyDriverVersion);
+    }
+
+    const milestone = getMilestone(chromeVersion);
+
+    if (Number(milestone) < MIN_CHROMEDRIVER_FOR_TESTING_VERSION) {
+        browserInstallerDebug(
+            `installing chromedriver for chrome@${chromeVersion} from chromedriver.storage.googleapis.com manually`,
+        );
+
+        const { installChromeDriverManually } = await import("../chromium");
+
+        return installChromeDriverManually(milestone);
+    }
+
+    const buildId = await resolveBuildId(Driver.CHROMEDRIVER, platform, milestone);
+
+    const cacheDir = getChromeDriverDir();
+    const canBeInstalled = await canDownload({ browser: Driver.CHROMEDRIVER, platform, buildId, cacheDir });
+
+    if (!canBeInstalled) {
+        throw new Error(
+            [
+                `chromedriver@${buildId} can't be installed.`,
+                `Probably the major browser version '${milestone}' is invalid`,
+                "Correct chrome version examples: '123', '124'",
+            ].join("\n"),
+        );
+    }
+
+    const installFn = (downloadProgressCallback: DownloadProgressCallback): Promise<string> =>
+        puppeteerInstall({
+            platform,
+            buildId,
+            cacheDir: getChromeDriverDir(),
+            browser: Driver.CHROMEDRIVER,
+            unpack: true,
+            downloadProgressCallback,
+        }).then(result => result.executablePath);
+
+    return installBinary(Driver.CHROMEDRIVER, platform, buildId, installFn);
+};

src/browser-installer/chrome/index.ts

@@ -0,0 +1,37 @@
+import { spawn, type ChildProcess } from "child_process";
+import getPort from "get-port";
+import waitPort from "wait-port";
+import { pipeLogsWithPrefix } from "../../dev-server/utils";
+import { DRIVER_WAIT_TIMEOUT } from "../constants";
+import { getMilestone } from "../utils";
+import { installChrome } from "./browser";
+import { installChromeDriver } from "./driver";
+
+export { installChrome, installChromeDriver };
+
+export const runChromeDriver = async (
+    chromeVersion: string,
+    { debug = false } = {},
+): Promise<{ gridUrl: string; process: ChildProcess; port: number }> => {
+    const [chromeDriverPath] = await Promise.all([installChromeDriver(chromeVersion), installChrome(chromeVersion)]);
+
+    const milestone = getMilestone(chromeVersion);
+    const randomPort = await getPort();
+
+    const chromeDriver = spawn(chromeDriverPath, [`--port=${randomPort}`, debug ? `--verbose` : "--silent"], {
+        windowsHide: true,
+        detached: false,
+    });
+
+    if (debug) {
+        pipeLogsWithPrefix(chromeDriver, `[chromedriver@${milestone}] `);
+    }
+
+    const gridUrl = `http://127.0.0.1:${randomPort}`;
+
+    process.once("exit", () => chromeDriver.kill());
+
+    await waitPort({ port: randomPort, output: "silent", timeout: DRIVER_WAIT_TIMEOUT });
+
+    return { gridUrl, process: chromeDriver, port: randomPort };
+};

src/browser-installer/chromium/browser.ts

@@ -0,0 +1,56 @@
+import { install as puppeteerInstall, canDownload } from "@puppeteer/browsers";
+import { installBinary, getBinaryPath, getMatchedBrowserVersion } from "../registry";
+import { getMilestone, browserInstallerDebug, getBrowsersDir, Browser, type DownloadProgressCallback } from "../utils";
+import { getChromiumBuildId } from "./utils";
+import { getChromePlatform } from "../utils";
+import { MIN_CHROMIUM_VERSION } from "../constants";
+
+export const installChromium = async (version: string, { force = false } = {}): Promise<string> => {
+    const milestone = getMilestone(version);
+
+    if (Number(milestone) < MIN_CHROMIUM_VERSION) {
+        throw new Error(
+            [
+                `chrome@${version} can't be installed.`,
+                `Automatic browser downloader is not available for chrome versions < ${MIN_CHROMIUM_VERSION}`,
+            ].join("\n"),
+        );
+    }
+
+    const platform = getChromePlatform(version);
+    const existingLocallyBrowserVersion = getMatchedBrowserVersion(Browser.CHROMIUM, platform, version);
+
+    if (existingLocallyBrowserVersion && !force) {
+        browserInstallerDebug(`A locally installed chromium@${version} browser was found. Skipping the installation`);
+
+        return getBinaryPath(Browser.CHROMIUM, platform, existingLocallyBrowserVersion);
+    }
+
+    const buildId = await getChromiumBuildId(platform, milestone);
+    const cacheDir = getBrowsersDir();
+    const canBeInstalled = await canDownload({ browser: Browser.CHROMIUM, platform, buildId, cacheDir });
+
+    if (!canBeInstalled) {
+        throw new Error(
+            [
+                `chrome@${version} can't be installed.`,
+                `Probably the version '${version}' is invalid, please try another version.`,
+                "Version examples: '93', '93.0'",
+            ].join("\n"),
+        );
+    }
+
+    browserInstallerDebug(`installing chromium@${buildId} (${milestone}) for ${platform}`);
+
+    const installFn = (downloadProgressCallback: DownloadProgressCallback): Promise<string> =>
+        puppeteerInstall({
+            platform,
+            buildId,
+            cacheDir,
+            downloadProgressCallback,
+            browser: Browser.CHROMIUM,
+            unpack: true,
+        }).then(result => result.executablePath);
+
+    return installBinary(Browser.CHROMIUM, platform, milestone, installFn);
+};

src/browser-installer/chromium/driver.ts

@@ -0,0 +1,54 @@
+import fs from "fs-extra";
+import path from "path";
+import { noop } from "lodash";
+import { CHROMEDRIVER_STORAGE_API, MIN_CHROMIUM_VERSION } from "../constants";
+import { installBinary } from "../registry";
+import {
+    downloadFile,
+    getChromiumDriverDir,
+    retryFetch,
+    unzipFile,
+    normalizeChromeVersion,
+    Driver,
+    getBrowserPlatform,
+} from "../utils";
+import { getChromeDriverArchiveTmpPath, getChromeDriverArchiveUrl } from "./utils";
+
+const getChromeDriverVersionByChromiumVersion = async (chromiumVersion: string | number): Promise<string> => {
+    const suffix = typeof chromiumVersion === "number" ? chromiumVersion : normalizeChromeVersion(chromiumVersion);
+
+    const result = await retryFetch(`${CHROMEDRIVER_STORAGE_API}/LATEST_RELEASE_${suffix}`).then(res => res.text());
+
+    return result;
+};
+
+export const installChromeDriverManually = async (milestone: string): Promise<string> => {
+    const platform = getBrowserPlatform();
+
+    if (Number(milestone) < MIN_CHROMIUM_VERSION) {
+        throw new Error(
+            [
+                `chromedriver@${milestone} can't be installed.`,
+                `Automatic driver downloader is not available for chrome versions < ${MIN_CHROMIUM_VERSION}`,
+            ].join("\n"),
+        );
+    }
+
+    const driverVersion = await getChromeDriverVersionByChromiumVersion(milestone);
+
+    const installFn = async (): Promise<string> => {
+        const archiveUrl = getChromeDriverArchiveUrl(driverVersion);
+        const archivePath = getChromeDriverArchiveTmpPath(driverVersion);
+        const chromeDriverDirPath = getChromiumDriverDir(driverVersion);
+        const chromeDriverPath = path.join(chromeDriverDirPath, "chromedriver");
+
+        await downloadFile(archiveUrl, archivePath);
+        await unzipFile(archivePath, chromeDriverDirPath);
+
+        fs.remove(archivePath).then(noop, noop);
+
+        return chromeDriverPath;
+    };
+
+    return installBinary(Driver.CHROMEDRIVER, platform, driverVersion, installFn);
+};

src/browser-installer/chromium/index.ts

@@ -0,0 +1,2 @@
+export { installChromium } from "./browser";
+export { installChromeDriverManually } from "./driver";

src/browser-installer/chromium/revisions/linux.ts

@@ -0,0 +1,42 @@
+export default {
+    73: 625983,
+    74: 638903,
+    75: 652459,
+    76: 665032,
+    77: 681154,
+    78: 694594,
+    79: 707231,
+    80: 722374,
+    81: 737198,
+    82: 750023,
+    83: 756143,
+    84: 769125,
+    85: 782822,
+    86: 800433,
+    87: 813060,
+    88: 827143,
+    89: 843934,
+    90: 858016,
+    91: 870827,
+    92: 885357,
+    93: 902296,
+    94: 911605,
+    95: 920070,
+    96: 929514,
+    97: 938637,
+    98: 950416,
+    99: 961779,
+    100: 972803,
+    101: 982577,
+    102: 992824,
+    103: 1002974,
+    104: 1012822,
+    105: 1027072,
+    106: 1036920,
+    107: 1047812,
+    108: 1059082,
+    109: 1070158,
+    110: 1084167,
+    111: 1097778,
+    112: 1107206,
+} as Record<number, number>;