feat: wait page load

package.json

@@ -45,8 +45,8 @@
   "dependencies": {
     "@gemini-testing/commander": "2.15.3",
     "@types/mocha": "^10.0.1",
-    "@wdio/globals": "^8.5.7",
-    "@wdio/types": "^8.4.0",
+    "@wdio/globals": "^8.10.7",
+    "@wdio/types": "^8.10.4",
     "@wdio/utils": "^7.26.0",
     "aliasify": "^1.9.0",
     "bluebird": "^3.5.1",
@@ -80,6 +80,7 @@
   "devDependencies": {
     "@commitlint/cli": "^17.1.2",
     "@commitlint/config-conventional": "^17.1.0",
+    "@sinonjs/fake-timers": "^10.3.0",
     "@swc/core": "^1.3.40",
     "@types/bluebird": "^3.5.38",
     "@types/chai": "^4.3.4",
@@ -89,6 +90,7 @@
     "@types/proxyquire": "^1.3.28",
     "@types/sharp": "^0.31.1",
     "@types/sinon": "^4.3.3",
+    "@types/sinonjs__fake-timers": "^8.1.2",
     "@typescript-eslint/eslint-plugin": "^5.54.1",
     "@typescript-eslint/parser": "^5.54.1",
     "app-module-path": "^2.2.0",

src/browser/commands/index.js

@@ -1,3 +1,3 @@
 "use strict";
 
-module.exports = ["assert-view", "getConfig", "getPuppeteer", "setOrientation", "scrollIntoView"];
+module.exports = ["assert-view", "getConfig", "getPuppeteer", "setOrientation", "scrollIntoView", "openAndWait"];

src/browser/commands/openAndWait.ts

@@ -0,0 +1,145 @@
+import _ from "lodash";
+import { Matches } from "webdriverio";
+import PageLoader from "../../utils/page-loader";
+
+interface Browser {
+    publicAPI: WebdriverIO.Browser;
+    config: {
+        desiredCapabilities: {
+            browserName: string;
+        };
+        automationProtocol: "webdriver" | "devtools";
+        pageLoadTimeout: number;
+        openAndWaitOpts: {
+            timeout?: number;
+            waitNetworkIdle: boolean;
+            waitNetworkIdleTimeout: number;
+            rejectOnNetworkError: boolean;
+            ignoreNetworkErrorsPatterns: Array<RegExp | string>;
+        };
+    };
+}
+
+interface WaitOpts {
+    selector?: string | string[];
+    predicate?: () => boolean;
+    waitNetworkIdle?: boolean;
+    waitNetworkIdleTimeout?: number;
+    rejectOnNetworkError?: boolean;
+    shouldThrowError?: (match: Matches) => boolean;
+    ignoreNetworkErrorsPatterns?: Array<RegExp | string>;
+    timeout?: number;
+}
+
+const emptyPageUrl = "about:blank";
+
+const is: Record<string, (match: Matches) => boolean> = {
+    image: match => match.headers?.Accept?.includes("image"),
+    stylesheet: match => match.headers?.Accept?.includes("text/css"),
+    font: match => _.isString(match.url) && [".ttf", ".woff", ".woff2"].some(ext => match.url.endsWith(ext)),
+    favicon: match => _.isString(match.url) && match.url.endsWith("/favicon.ico"),
+};
+
+export = (browser: Browser): void => {
+    const { publicAPI: session, config } = browser;
+    const { openAndWaitOpts } = config;
+    const isChrome = config.desiredCapabilities.browserName === "chrome";
+    const isCDP = config.automationProtocol === "devtools";
+
+    function openAndWait(
+        uri: string,
+        {
+            selector = [],
+            predicate,
+            waitNetworkIdle = openAndWaitOpts?.waitNetworkIdle,
+            waitNetworkIdleTimeout = openAndWaitOpts?.waitNetworkIdleTimeout,
+            rejectOnNetworkError = openAndWaitOpts?.rejectOnNetworkError,
+            shouldThrowError = shouldThrowErrorDefault,
+            ignoreNetworkErrorsPatterns = openAndWaitOpts?.ignoreNetworkErrorsPatterns,
+            timeout = openAndWaitOpts?.timeout || config?.pageLoadTimeout,
+        }: WaitOpts = {},
+    ): Promise<string | void> {
+        waitNetworkIdle &&= isChrome || isCDP;
+
+        if (!uri || uri === emptyPageUrl) {
+            return new Promise(resolve => {
+                session.url(uri).then(() => resolve());
+            });
+        }
+
+        const selectors = typeof selector === "string" ? [selector] : selector;
+
+        const pageLoader = new PageLoader(session, {
+            selectors,
+            predicate,
+            timeout,
+            waitNetworkIdle,
+            waitNetworkIdleTimeout,
+        });
+
+        let selectorsResolved = !selectors.length;
+        let predicateResolved = !predicate;
+        let networkResolved = !waitNetworkIdle;
+
+        return new Promise<void>((resolve, reject) => {
+            const handleError = (err: Error): void => {
+                reject(new Error(`url: ${err.message}`));
+            };
+
+            const checkLoaded = (): void => {
+                if (selectorsResolved && predicateResolved && networkResolved) {
+                    resolve();
+                }
+            };
+
+            const goToPage = async (): Promise<void> => {
+                await session.url(uri);
+            };
+
+            pageLoader.on("pageLoadError", handleError);
+            pageLoader.on("selectorsError", handleError);
+            pageLoader.on("predicateError", handleError);
+            pageLoader.on("networkError", match => {
+                if (!rejectOnNetworkError) {
+                    return;
+                }
+
+                const shouldIgnore = isMatchPatterns(ignoreNetworkErrorsPatterns, match.url);
+
+                if (!shouldIgnore && shouldThrowError(match)) {
+                    reject(new Error(`url: couldn't get content from ${match.url}: ${match.statusCode}`));
+                }
+            });
+            pageLoader.on("selectorsExist", () => {
+                selectorsResolved = true;
+                checkLoaded();
+            });
+
+            pageLoader.on("predicateResolved", () => {
+                predicateResolved = true;
+                checkLoaded();
+            });
+
+            pageLoader.on("networkResolved", () => {
+                networkResolved = true;
+                checkLoaded();
+            });
+
+            pageLoader.load(goToPage).then(checkLoaded);
+        }).finally(() => pageLoader.unsubscribe());
+    }
+
+    session.addCommand("openAndWait", openAndWait);
+};
+
+function isMatchPatterns(patterns: Array<RegExp | string> = [], str: string): boolean {
+    return patterns.some(pattern => (_.isString(pattern) ? str.includes(pattern) : pattern.exec(str)));
+}
+
+function shouldThrowErrorDefault(match: Matches): boolean {
+    if (is.favicon(match)) {
+        return false;
+    }
+
+    return is.image(match) || is.stylesheet(match) || is.font(match);
+}

src/config/browser-options.js

@@ -240,6 +240,16 @@ function buildBrowserOptions(defaultFactory, extra) {
             },
         }),
 
+        openAndWaitOpts: option({
+            defaultValue: defaultFactory("openAndWaitOpts"),
+            parseEnv: JSON.parse,
+            parseCli: JSON.parse,
+            validate: value => utils.assertOptionalObject(value, "openAndWaitOpts"),
+            map: value => {
+                return value === defaults.openAndWaitOpts ? value : { ...defaults.openAndWaitOpts, ...value };
+            },
+        }),
+
         meta: options.optionalObject("meta"),
 
         windowSize: option({

src/config/defaults.js

@@ -27,6 +27,12 @@ module.exports = {
         captureElementFromTop: true,
         allowViewportOverflow: false,
     },
+    openAndWaitOpts: {
+        waitNetworkIdle: true,
+        waitNetworkIdleTimeout: 500,
+        rejectOnNetworkError: true,
+        ignoreNetworkErrorsPatterns: [],
+    },
     calibrate: false,
     screenshotMode: "auto",
     screenshotDelay: 0,

src/utils/page-loader.ts

@@ -0,0 +1,157 @@
+import EventEmitter from "events";
+import type { Matches, Mock } from "webdriverio";
+import logger from "./logger";
+
+export interface PageLoaderOpts {
+    selectors: string[];
+    predicate?: () => boolean | Promise<boolean>;
+    timeout: number;
+    waitNetworkIdle: boolean;
+    waitNetworkIdleTimeout: number;
+}
+
+export default class PageLoader extends EventEmitter {
+    private session: WebdriverIO.Browser;
+    private mock?: Mock | null;
+    private selectors: string[];
+    private predicate?: () => boolean | Promise<boolean>;
+    private timeout: number;
+    private waitNetworkIdle: boolean;
+    private waitNetworkIdleTimeout: number;
+    private totalRequests = 0;
+    private networkResolved = false;
+
+    constructor(
+        session: WebdriverIO.Browser,
+        { selectors, predicate, timeout, waitNetworkIdle, waitNetworkIdleTimeout }: PageLoaderOpts,
+    ) {
+        super();
+
+        this.session = session;
+        this.selectors = selectors;
+        this.predicate = predicate;
+        this.timeout = timeout;
+        this.waitNetworkIdle = waitNetworkIdle;
+        this.waitNetworkIdleTimeout = waitNetworkIdleTimeout;
+    }
+
+    public async load(goToPage: () => Promise<void>): Promise<void> {
+        await this.initMock();
+
+        await goToPage().catch(err => {
+            this.emit("pageLoadError", err);
+        });
+
+        this.launchSelectorsPromise();
+        this.launchPredicatePromise();
+        this.launchNetworkPromise();
+    }
+
+    public unsubscribe(): Promise<void> | undefined {
+        return this.mock?.restore().catch(() => {
+            logger.warn("PageLoader: Got err while unsubscribing");
+        });
+    }
+
+    private launchSelectorsPromise(): void {
+        const selectorPromises = this.selectors.map(async selector => {
+            const element = await this.session.$(selector);
+            await element.waitForExist({ timeout: this.timeout });
+        });
+
+        Promise.all(selectorPromises)
+            .then(() => {
+                this.emit("selectorsExist");
+            })
+            .catch(err => {
+                this.emit("selectorsError", err);
+            });
+    }
+
+    private launchPredicatePromise(): void {
+        if (!this.predicate) {
+            return;
+        }
+
+        this.session
+            .waitUntil(this.predicate, { timeout: this.timeout })
+            .then(() => {
+                this.emit("predicateResolved");
+            })
+            .catch(() => {
+                this.emit("predicateError", new Error(`predicate was never truthy in ${this.timeout}ms`));
+            });
+    }
+
+    private launchNetworkPromise(): void {
+        if (!this.waitNetworkIdle) {
+            return;
+        }
+
+        setTimeout(() => {
+            const markSuccess = this.markNetworkIdle();
+            if (markSuccess) {
+                logger.warn(`PageLoader: Network idle timeout`);
+            }
+        }, this.timeout);
+        setTimeout(() => {
+            if (!this.totalRequests) {
+                this.markNetworkIdle();
+            }
+        }, this.waitNetworkIdleTimeout);
+    }
+
+    private async initMock(): Promise<void> {
+        if (!this.waitNetworkIdle) {
+            return;
+        }
+
+        this.mock = await this.session.mock("**").catch(() => {
+            logger.warn(`PageLoader: Could not create CDP interceptor`);
+
+            return null;
+        });
+
+        if (!this.mock) {
+            this.markNetworkIdle();
+
+            return;
+        }
+
+        let pendingRequests = 0;
+        let pendingIdleTimeout: NodeJS.Timeout;
+        this.mock.on("request", () => {
+            this.totalRequests++;
+            pendingRequests++;
+            clearTimeout(pendingIdleTimeout);
+        });
+
+        this.mock.on("continue", () => {
+            pendingRequests--;
+
+            if (!pendingRequests) {
+                pendingIdleTimeout = setTimeout(() => this.markNetworkIdle(), this.waitNetworkIdleTimeout);
+            }
+        });
+
+        this.mock.on("match", (match: Matches) => {
+            if (this.isMatchError(match)) {
+                this.emit("networkError", match);
+            }
+        });
+    }
+
+    private isMatchError(match: Matches): boolean {
+        return match.statusCode >= 400 && match.statusCode < 600;
+    }
+
+    private markNetworkIdle(): boolean {
+        if (this.networkResolved) {
+            return false;
+        }
+
+        this.networkResolved = true;
+        this.emit("networkResolved");
+        return true;
+    }
+}