From 7ba47be810f32bc352e9007300e9dc7f372406a1 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 8 Nov 2024 15:15:45 +0100 Subject: [PATCH] Add base for GUI tests --- .github/workflows/main.yml | 15 +++++++ .gitignore | 5 +++ CONTRIBUTING.md | 19 ++++++++- Cargo.toml | 7 ++++ tests/gui/runner.rs | 86 ++++++++++++++++++++++++++++++++++++++ tests/gui/sidebar.goml | 59 ++++++++++++++++++++++++++ 6 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tests/gui/runner.rs create mode 100644 tests/gui/sidebar.goml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a9b4570236..f70b69025c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,9 @@ on: pull_request: merge_group: +env: + BROWSER_UI_TEST_VERSION: '0.18.2' + jobs: test: runs-on: ${{ matrix.os }} @@ -47,7 +50,19 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust run: bash ci/install-rust.sh ${{ matrix.rust }} ${{ matrix.target }} + - name: Install npm + if: matrix.os != 'windows-latest' + uses: actions/setup-node@v3 + with: + node-version: 20 + - name: Install browser-ui-test + if: matrix.os != 'windows-latest' + run: npm install browser-ui-test@"${BROWSER_UI_TEST_VERSION}" + - name: Build and run tests (+ GUI) + if: matrix.os != 'windows-latest' + run: cargo test --locked --target ${{ matrix.target }} --test gui - name: Build and run tests + if: matrix.os == 'windows-latest' run: cargo test --locked --target ${{ matrix.target }} - name: Test no default run: cargo test --no-default-features --target ${{ matrix.target }} diff --git a/.gitignore b/.gitignore index a23c771e89..34e1d9b10f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ test_book/book/ # Ignore Vim temporary and swap files. *.sw? *~ + +# GUI tests +node_modules +package-lock.json +package.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98c31a9c78..293eb6e898 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,8 +138,23 @@ We generally strive to keep mdBook compatible with a relatively recent browser o That is, supporting Chrome, Safari, Firefox, Edge on Windows, macOS, Linux, iOS, and Android. If possible, do your best to avoid breaking older browser releases. -Any change to the HTML or styling is encouraged to manually check on as many browsers and platforms that you can. -Unfortunately at this time we don't have any automated UI or browser testing, so your assistance in testing is appreciated. +GUI tests are checked with the GUI testsuite. To run it, you need to install `npm` first. Then run: + +``` +cargo test --test gui +``` + +The first time, it'll fail and ask you to install the `browser-ui-test` package. Install it then re-run the tests. + +If you want to disable the headless mode, use the `DISABLE_HEADLESS_TEST=1` environment variable: + +``` +cargo test --test gui -- --disable-headless-test +``` + +The GUI tests are in the directory `tests/gui` in text files with the `.goml` extension. These tests are run +using a `node.js` framework called `browser-ui-test`. You can find documentation for this language on its +[repository](https://github.com/GuillaumeGomez/browser-UI-test/blob/master/goml-script.md). ## Updating highlight.js diff --git a/Cargo.toml b/Cargo.toml index 5d593b2700..3d37043904 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,3 +82,10 @@ name = "remove-emphasis" path = "examples/remove-emphasis/test.rs" crate-type = ["lib"] test = true + +[[test]] +harness = false +test = false +name = "gui" +path = "tests/gui/runner.rs" +crate-type = ["bin"] diff --git a/tests/gui/runner.rs b/tests/gui/runner.rs new file mode 100644 index 0000000000..0cab925f08 --- /dev/null +++ b/tests/gui/runner.rs @@ -0,0 +1,86 @@ +use std::env::current_dir; +use std::fs::{read_to_string, remove_dir_all}; +use std::process::Command; + +fn get_available_browser_ui_test_version_inner(global: bool) -> Option { + let mut command = Command::new("npm"); + command + .arg("list") + .arg("--parseable") + .arg("--long") + .arg("--depth=0"); + if global { + command.arg("--global"); + } + let stdout = command.output().expect("`npm` command not found").stdout; + let lines = String::from_utf8_lossy(&stdout); + lines + .lines() + .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) + .map(std::borrow::ToOwned::to_owned) +} + +fn get_available_browser_ui_test_version() -> Option { + get_available_browser_ui_test_version_inner(false) + .or_else(|| get_available_browser_ui_test_version_inner(true)) +} + +fn expected_browser_ui_test_version() -> String { + let content = read_to_string(".github/workflows/main.yml") + .expect("failed to read `.github/workflows/main.yml`"); + for line in content.lines() { + let line = line.trim(); + if let Some(version) = line.strip_prefix("BROWSER_UI_TEST_VERSION:") { + return version.trim().replace('\'', ""); + } + } + panic!("failed to retrieved `browser-ui-test` version"); +} + +fn main() { + let browser_ui_test_version = expected_browser_ui_test_version(); + match get_available_browser_ui_test_version() { + Some(version) => { + if version != browser_ui_test_version { + eprintln!( + "⚠️ Installed version of browser-ui-test (`{version}`) is different than the \ + one used in the CI (`{browser_ui_test_version}`) You can install this version \ + using `npm update browser-ui-test` or by using `npm install browser-ui-test\ + @{browser_ui_test_version}`", + ); + } + } + None => { + panic!( + "`browser-ui-test` is not installed. You can install this package using `npm \ + update browser-ui-test` or by using `npm install browser-ui-test\ + @{browser_ui_test_version}`", + ); + } + } + + let current_dir = current_dir().expect("failed to retrieve current directory"); + let test_book = current_dir.join("test_book"); + + // Result doesn't matter. + let _ = remove_dir_all(test_book.join("book")); + + let mut cmd = Command::new("cargo"); + cmd.arg("run").arg("build").arg(&test_book); + // Then we run the GUI tests on it. + assert!(cmd.status().is_ok_and(|status| status.success())); + + let book_dir = format!("file://{}", current_dir.join("test_book/book/").display()); + + let mut command = Command::new("npx"); + command + .arg("browser-ui-test") + .args(["--variable", "DOC_PATH", book_dir.as_str()]) + .args(["--test-folder", "tests/gui"]); + if std::env::args().any(|arg| arg == "--disable-headless-test") { + command.arg("--no-headless"); + } + + // Then we run the GUI tests on it. + assert!(command.status().is_ok_and(|status| status.success())); +} diff --git a/tests/gui/sidebar.goml b/tests/gui/sidebar.goml new file mode 100644 index 0000000000..7e4ec32b45 --- /dev/null +++ b/tests/gui/sidebar.goml @@ -0,0 +1,59 @@ +// This GUI test checks sidebar hide/show and also its behaviour on smaller +// width. + +// We disable the requests checks because `searchindex.json` will always fail +// locally. +fail-on-request-error: false +go-to: |DOC_PATH| + "index.html" +set-window-size: (1100, 600) +// Need to reload for the new size to be taken account by the JS. +reload: + +store-value: (content_indent, 308) + +define-function: ( + "hide-sidebar", + [], + block { + // The content should be "moved" to the right because of the sidebar. + assert-css: ("#sidebar", {"transform": "none"}) + assert-position: ("#page-wrapper", {"x": |content_indent|}) + + // We now hide the sidebar. + click: "#sidebar-toggle" + wait-for: "body.sidebar-hidden" + // `transform` is 0.3s so we need to wait a bit (0.5s) to ensure the animation is done. + wait-for: 5000 + assert-css-false: ("#sidebar", {"transform": "none"}) + // The page content should now be on the left. + assert-position: ("#page-wrapper", {"x": 0}) + }, +) + +define-function: ( + "show-sidebar", + [], + block { + // The page content should be on the left and the sidebar "moved out". + assert-css: ("#sidebar", {"transform": "matrix(1, 0, 0, 1, -308, 0)"}) + assert-position: ("#page-wrapper", {"x": 0}) + + // We expand the sidebar. + click: "#sidebar-toggle" + wait-for: "body.sidebar-visible" + // `transform` is 0.3s so we need to wait a bit (0.5s) to ensure the animation is done. + wait-for: 5000 + assert-css-false: ("#sidebar", {"transform": "matrix(1, 0, 0, 1, -308, 0)"}) + // The page content should be moved to the right. + assert-position: ("#page-wrapper", {"x": |content_indent|}) + }, +) + +call-function: ("hide-sidebar", {}) +call-function: ("show-sidebar", {}) + +// We now test on smaller width to ensure that the sidebar is collapsed by default. +set-window-size: (900, 600) +reload: +call-function: ("show-sidebar", {}) +call-function: ("hide-sidebar", {})