From 773e847e40b077ff18e234102e820e6159647c81 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 13 Oct 2023 00:15:01 +0000 Subject: [PATCH 1/6] feat: support connecting to remote endpoints --- deno.lock | 5 ++++ src/browser.ts | 40 ++++++++++++++++++++++++------ tests/existing_ws_endpoint_test.ts | 33 ++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 tests/existing_ws_endpoint_test.ts diff --git a/deno.lock b/deno.lock index 1f5d00e..bed5597 100644 --- a/deno.lock +++ b/deno.lock @@ -89,6 +89,11 @@ "https://deno.land/std@0.201.0/testing/asserts.ts": "b4e4b1359393aeff09e853e27901a982c685cb630df30426ed75496961931946", "https://deno.land/std@0.201.0/testing/snapshot.ts": "fd91f03c258c316bc9faf815846d85e80e0c82622b28ee44b57ec66dd91d3408", "https://deno.land/std@0.203.0/fs/exists.ts": "cb59a853d84871d87acab0e7936a4dac11282957f8e195102c5a7acb42546bb8", + "https://deno.land/std@0.204.0/assert/assert_is_error.ts": "c21113094a51a296ffaf036767d616a78a2ae5f9f7bbd464cd0197476498b94b", + "https://deno.land/std@0.204.0/assert/assert_rejects.ts": "45c59724de2701e3b1f67c391d6c71c392363635aad3f68a1b3408f9efca0057", + "https://deno.land/std@0.204.0/assert/assert_throws.ts": "63784e951475cb7bdfd59878cd25a0931e18f6dc32a6077c454b2cd94f4f4bcd", + "https://deno.land/std@0.204.0/assert/assertion_error.ts": "4d0bde9b374dfbcbe8ac23f54f567b77024fb67dbb1906a852d67fe050d42f56", + "https://deno.land/std@0.204.0/fmt/colors.ts": "c51c4642678eb690dcf5ffee5918b675bf01a33fba82acf303701ae1a4f8c8d9", "https://deno.land/x/progress@v1.3.9/deps.ts": "83050e627263931d853ba28b7c15c80bf4be912bea7e0d3d13da2bc0aaf7889d", "https://deno.land/x/progress@v1.3.9/mod.ts": "ca14ba3c56fc5991c4beee622faeb882e59db8f28d4f5e529ac17b7b07d55741", "https://deno.land/x/progress@v1.3.9/multi.ts": "1de7edf67047ba0050edf63229970f2cbf9cd069342fcd29444a3800d4cfdcbc", diff --git a/src/browser.ts b/src/browser.ts index 4f007f0..f43bd54 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -70,10 +70,14 @@ export interface BrowserOptions { export class Browser { #options: BrowserOptions; #celestial: Celestial; - #process: Deno.ChildProcess; + #process: Deno.ChildProcess | null; readonly pages: Page[] = []; - constructor(ws: WebSocket, process: Deno.ChildProcess, opts: BrowserOptions) { + constructor( + ws: WebSocket, + process: Deno.ChildProcess | null, + opts: BrowserOptions, + ) { this.#celestial = new Celestial(ws); this.#process = process; this.#options = opts; @@ -84,8 +88,14 @@ export class Browser { */ async close() { await this.#celestial.close(); - this.#process.kill(); - await this.#process.status; + // Clean process if it exists + if (this.#process) { + this.#process.kill(); + await this.#process.status; + } // If we use a remote connection, then close all pages websockets + else { + await Promise.allSettled(this.pages.map((page) => page.close())); + } } /** @@ -96,16 +106,15 @@ export class Browser { url: "", }); const browserWsUrl = new URL(this.#celestial.ws.url); - const wsUrl = `${browserWsUrl.origin}/devtools/page/${targetId}`; + const wsUrl = + `${browserWsUrl.origin}/devtools/page/${targetId}${browserWsUrl.search}`; const websocket = new WebSocket(wsUrl); await websocketReady(websocket); - const page = new Page(targetId, url, websocket, this); this.pages.push(page); const celestial = page.unsafelyGetCelestialBindings(); const { userAgent } = await celestial.Browser.getVersion(); - await Promise.all([ celestial.Emulation.setUserAgentOverride({ userAgent: userAgent.replaceAll("Headless", ""), @@ -113,7 +122,6 @@ export class Browser { celestial.Page.enable(), celestial.Page.setInterceptFileChooserDialog({ enabled: true }), ]); - if (url) { await page.goto(url, options); } @@ -143,6 +151,13 @@ export class Browser { wsEndpoint() { return this.#celestial.ws.url; } + + /** + * The browser's websocket ready state + */ + wsReadyState() { + return this.#celestial.ws.readyState; + } } export interface LaunchOptions { @@ -150,6 +165,7 @@ export interface LaunchOptions { path?: string; product?: "chrome" | "firefox"; args?: string[]; + browserWSEndpoint?: string; } /** @@ -159,6 +175,7 @@ export async function launch(opts?: LaunchOptions) { const headless = opts?.headless ?? true; const product = opts?.product ?? "chrome"; const args = opts?.args ?? []; + const browserWSEndpoint = opts?.browserWSEndpoint; let path = opts?.path; if (!path) { @@ -170,6 +187,13 @@ export async function launch(opts?: LaunchOptions) { product, }; + // Connect to endpoint directly if one was specified + if (browserWSEndpoint) { + const ws = new WebSocket(browserWSEndpoint); + await websocketReady(ws); + return new Browser(ws, null, options); + } + // Launch child process const launch = new Deno.Command(path, { args: [ diff --git a/tests/existing_ws_endpoint_test.ts b/tests/existing_ws_endpoint_test.ts new file mode 100644 index 0000000..68ff3a4 --- /dev/null +++ b/tests/existing_ws_endpoint_test.ts @@ -0,0 +1,33 @@ +import { assertEquals } from "https://deno.land/std@0.201.0/assert/assert_equals.ts"; +import { launch } from "../mod.ts"; +import { assertThrows } from "https://deno.land/std@0.204.0/assert/assert_throws.ts"; + +Deno.test("Test existing ws endpoint", async () => { + // Spawn one browser instance and spawn another one connecting to the first one + const a = await launch(); + const b = await launch({ browserWSEndpoint: a.wsEndpoint() }); + + // Test that second instance works without any process attached + const page = await b.newPage("http://example.com"); + await page.waitForSelector("h1"); + + // Close first instance and ensure that b instance is inactive too + await a.close(); + assertEquals(a.wsReadyState(), WebSocket.CLOSED); + assertEquals(b.wsReadyState(), WebSocket.CLOSED); +}); + +Deno.test("Ensure pages are properly closed when closing existing endpoint", async () => { + // Spawn one browser instance and spawn another one connecting to the first one + const a = await launch(); + const b = await launch({ browserWSEndpoint: a.wsEndpoint() }); + + // Ensure closing existing endpoint properly clean resources + await b.newPage("http://example.com"); + await b.newPage("http://example.com"); + await b.close(); + assertThrows(() => b.pages[0].close(), "Page has already been closed"); + assertThrows(() => b.pages[1].close(), "Page has already been closed"); + assertEquals(b.wsReadyState(), WebSocket.CLOSED); + await a.close(); +}); From 389b950bf5f25228b06000eafcc01dc6751abc53 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 13 Oct 2023 00:18:14 +0000 Subject: [PATCH 2/6] limit diff --- src/browser.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index f43bd54..af8cb91 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -88,12 +88,10 @@ export class Browser { */ async close() { await this.#celestial.close(); - // Clean process if it exists - if (this.#process) { - this.#process.kill(); - await this.#process.status; - } // If we use a remote connection, then close all pages websockets - else { + this.#process?.kill(); + await this.#process?.status; + // If we use a remote connection, then close all pages websockets + if (!this.#process) { await Promise.allSettled(this.pages.map((page) => page.close())); } } @@ -110,11 +108,13 @@ export class Browser { `${browserWsUrl.origin}/devtools/page/${targetId}${browserWsUrl.search}`; const websocket = new WebSocket(wsUrl); await websocketReady(websocket); + const page = new Page(targetId, url, websocket, this); this.pages.push(page); const celestial = page.unsafelyGetCelestialBindings(); const { userAgent } = await celestial.Browser.getVersion(); + await Promise.all([ celestial.Emulation.setUserAgentOverride({ userAgent: userAgent.replaceAll("Headless", ""), @@ -122,6 +122,7 @@ export class Browser { celestial.Page.enable(), celestial.Page.setInterceptFileChooserDialog({ enabled: true }), ]); + if (url) { await page.goto(url, options); } From 42f070c029ec4c5190a8fd71279a6232fac9025c Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:56:30 +0000 Subject: [PATCH 3/6] fix: page.close() for remote connection --- src/browser.ts | 7 ++++++- src/page.ts | 16 ++++++++++++---- tests/existing_ws_endpoint_test.ts | 3 +++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index af8cb91..9061531 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -83,6 +83,11 @@ export class Browser { this.#options = opts; } + /** Returns true if browser is connected remotely instead of using a subprocess */ + get isRemoteConnection() { + return !this.#process; + } + /** * Closes the browser and all of its pages (if any were opened). The Browser object itself is considered to be disposed and cannot be used anymore. */ @@ -91,7 +96,7 @@ export class Browser { this.#process?.kill(); await this.#process?.status; // If we use a remote connection, then close all pages websockets - if (!this.#process) { + if (this.isRemoteConnection) { await Promise.allSettled(this.pages.map((page) => page.close())); } } diff --git a/src/page.ts b/src/page.ts index 2298102..1fb6c12 100644 --- a/src/page.ts +++ b/src/page.ts @@ -200,11 +200,19 @@ export class Page extends EventTarget { * Close this page in the browser */ async close() { - const wsUrl = new URL(this.#celestial.ws.url); - const req = await fetch(`http://${wsUrl.host}/json/close/${this.#id}`); - const res = await req.text(); + let success: boolean; + let res = ""; + if (this.#browser.isRemoteConnection) { + await this.#celestial.close(); + success = this.#browser.pages.includes(this); + } else { + const wsUrl = new URL(this.#celestial.ws.url); + const req = await fetch(`http://${wsUrl.host}/json/close/${this.#id}`); + res = await req.text(); + success = res === "Target is closing"; + } - if (res === "Target is closing") { + if (success) { const index = this.#browser.pages.indexOf(this); if (index > -1) { this.#browser.pages.splice(index, 1); diff --git a/tests/existing_ws_endpoint_test.ts b/tests/existing_ws_endpoint_test.ts index 68ff3a4..428069c 100644 --- a/tests/existing_ws_endpoint_test.ts +++ b/tests/existing_ws_endpoint_test.ts @@ -1,6 +1,7 @@ import { assertEquals } from "https://deno.land/std@0.201.0/assert/assert_equals.ts"; import { launch } from "../mod.ts"; import { assertThrows } from "https://deno.land/std@0.204.0/assert/assert_throws.ts"; +import { assert } from "https://deno.land/std@0.201.0/assert/assert.ts"; Deno.test("Test existing ws endpoint", async () => { // Spawn one browser instance and spawn another one connecting to the first one @@ -10,6 +11,8 @@ Deno.test("Test existing ws endpoint", async () => { // Test that second instance works without any process attached const page = await b.newPage("http://example.com"); await page.waitForSelector("h1"); + await page.close(); + assert(!b.pages.includes(page)); // Close first instance and ensure that b instance is inactive too await a.close(); From 5da33a557a2a6339d6bb0dfa2e2328e259c0bb3f Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:44:08 +0000 Subject: [PATCH 4/6] fix: don't require to download binary if remote endpoint is specified --- src/browser.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index 9061531..a023fa0 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -184,10 +184,6 @@ export async function launch(opts?: LaunchOptions) { const browserWSEndpoint = opts?.browserWSEndpoint; let path = opts?.path; - if (!path) { - path = await getBinary(product); - } - const options: BrowserOptions = { headless, product, @@ -200,6 +196,10 @@ export async function launch(opts?: LaunchOptions) { return new Browser(ws, null, options); } + if (!path) { + path = await getBinary(product); + } + // Launch child process const launch = new Deno.Command(path, { args: [ From 875b01b73ff4673d0d13016440b01368bba3138e Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Sun, 15 Oct 2023 00:05:05 +0000 Subject: [PATCH 5/6] fix: rename to wsEndpoint --- src/browser.ts | 8 ++++---- tests/existing_ws_endpoint_test.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index a023fa0..3a55616 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -171,7 +171,7 @@ export interface LaunchOptions { path?: string; product?: "chrome" | "firefox"; args?: string[]; - browserWSEndpoint?: string; + wsEndpoint?: string; } /** @@ -181,7 +181,7 @@ export async function launch(opts?: LaunchOptions) { const headless = opts?.headless ?? true; const product = opts?.product ?? "chrome"; const args = opts?.args ?? []; - const browserWSEndpoint = opts?.browserWSEndpoint; + const wsEndpoint = opts?.wsEndpoint; let path = opts?.path; const options: BrowserOptions = { @@ -190,8 +190,8 @@ export async function launch(opts?: LaunchOptions) { }; // Connect to endpoint directly if one was specified - if (browserWSEndpoint) { - const ws = new WebSocket(browserWSEndpoint); + if (wsEndpoint) { + const ws = new WebSocket(wsEndpoint); await websocketReady(ws); return new Browser(ws, null, options); } diff --git a/tests/existing_ws_endpoint_test.ts b/tests/existing_ws_endpoint_test.ts index 428069c..64dd2d2 100644 --- a/tests/existing_ws_endpoint_test.ts +++ b/tests/existing_ws_endpoint_test.ts @@ -6,7 +6,7 @@ import { assert } from "https://deno.land/std@0.201.0/assert/assert.ts"; Deno.test("Test existing ws endpoint", async () => { // Spawn one browser instance and spawn another one connecting to the first one const a = await launch(); - const b = await launch({ browserWSEndpoint: a.wsEndpoint() }); + const b = await launch({ wsEndpoint: a.wsEndpoint() }); // Test that second instance works without any process attached const page = await b.newPage("http://example.com"); @@ -23,7 +23,7 @@ Deno.test("Test existing ws endpoint", async () => { Deno.test("Ensure pages are properly closed when closing existing endpoint", async () => { // Spawn one browser instance and spawn another one connecting to the first one const a = await launch(); - const b = await launch({ browserWSEndpoint: a.wsEndpoint() }); + const b = await launch({ wsEndpoint: a.wsEndpoint() }); // Ensure closing existing endpoint properly clean resources await b.newPage("http://example.com"); From 764161f77c4dac7d25c566dab9061fabd637bddd Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Tue, 17 Oct 2023 23:18:35 +0000 Subject: [PATCH 6/6] fix: apply recommandations --- docs/pages/advanced/connect.md | 45 ++++++++++++++++++++++++++++++ src/browser.ts | 6 ++-- tests/existing_ws_endpoint_test.ts | 7 ++--- 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 docs/pages/advanced/connect.md diff --git a/docs/pages/advanced/connect.md b/docs/pages/advanced/connect.md new file mode 100644 index 0000000..a3b65bd --- /dev/null +++ b/docs/pages/advanced/connect.md @@ -0,0 +1,45 @@ +--- +title: Connect to existing browser +description: How to connect an existing browser process with astral +index: 3 +--- + +If you already have a browser process running somewhere else or you're using a +service that provides remote browsers for automation (such as +[browserless.io](https://www.browserless.io/)), it is possible to directly +connect to its endpoint rather than spawning a new process. + +## Code + +```ts +// Import Astral +import { launch } from "https://deno.land/x/astral/mod.ts"; + +// Connect to remote endpoint +const browser = await launch({ + wsEndpoint: "wss://remote-browser-endpoint.example.com", +}); + +// Do stuff +const page = await browser.newPage("http://example.com"); +console.log(await page.evaluate(() => document.title)); + +// Close connection +await browser.close(); +``` + +## Reusing a browser spawned by astral + +A browser instance expose its WebSocket endpoint through `browser.wsEndpoint()`. + +```ts +// Spawn a browser process +const browser = await launch(); + +// Connect to first browser instead +const anotherBrowser = await launch({ wsEndpoint: browser.wsEndpoint() }); +``` + +This is especially useful in unit testing as you can setup a shared browser +instance before all your tests, while also properly closing resources to avoid +operations leaks. diff --git a/src/browser.ts b/src/browser.ts index 3a55616..d79813a 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -159,10 +159,10 @@ export class Browser { } /** - * The browser's websocket ready state + * Returns true if the browser and its websocket have benn closed */ - wsReadyState() { - return this.#celestial.ws.readyState; + get closed() { + return this.#celestial.ws.readyState === WebSocket.CLOSED; } } diff --git a/tests/existing_ws_endpoint_test.ts b/tests/existing_ws_endpoint_test.ts index 64dd2d2..6a75d7a 100644 --- a/tests/existing_ws_endpoint_test.ts +++ b/tests/existing_ws_endpoint_test.ts @@ -1,4 +1,3 @@ -import { assertEquals } from "https://deno.land/std@0.201.0/assert/assert_equals.ts"; import { launch } from "../mod.ts"; import { assertThrows } from "https://deno.land/std@0.204.0/assert/assert_throws.ts"; import { assert } from "https://deno.land/std@0.201.0/assert/assert.ts"; @@ -16,8 +15,8 @@ Deno.test("Test existing ws endpoint", async () => { // Close first instance and ensure that b instance is inactive too await a.close(); - assertEquals(a.wsReadyState(), WebSocket.CLOSED); - assertEquals(b.wsReadyState(), WebSocket.CLOSED); + assert(a.closed); + assert(b.closed); }); Deno.test("Ensure pages are properly closed when closing existing endpoint", async () => { @@ -31,6 +30,6 @@ Deno.test("Ensure pages are properly closed when closing existing endpoint", asy await b.close(); assertThrows(() => b.pages[0].close(), "Page has already been closed"); assertThrows(() => b.pages[1].close(), "Page has already been closed"); - assertEquals(b.wsReadyState(), WebSocket.CLOSED); + assert(b.closed); await a.close(); });