diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ac947ca6..d94f3c27a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,35 @@ jobs: - name: Clean up the database run: docker-compose down --volumes + GUI_test: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + - id: install + run: | + rustup override set stable + rustup update stable + + - name: restore build & cargo cache + uses: Swatinem/rust-cache@v2 + + - name: Launch postgres and min.io + run: | + cp .env.sample .env + mkdir -p ${DOCSRS_PREFIX}/public-html + docker-compose up -d db s3 + # Give the database enough time to start up + sleep 5 + # Make sure the database is actually working + psql "${DOCSRS_DATABASE_URL}" + + - name: Run GUI tests + run: ./dockerfiles/run-gui-tests.sh + + - name: Clean up the database + run: docker-compose down --volumes + build_tests: runs-on: ubuntu-latest needs: build diff --git a/README.md b/README.md index b16834f40..d50777109 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,15 @@ Note that you will need docker installed no matter what, since it's used for Rus cargo test ``` +To run GUI tests: + +``` +./dockerfiles/run-gui-tests.sh +``` + +They use the [browser-ui-test](https://github.com/GuillaumeGomez/browser-UI-test/) framework. You +can take a look at its documentation [here](https://github.com/GuillaumeGomez/browser-UI-test/blob/master/goml-script.md). + ### Pure docker-compose If you have trouble with the above commands, consider using `docker-compose up --build`, diff --git a/docker-compose.yml b/docker-compose.yml index 36180aa3d..ce503de9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,16 @@ services: timeout: 5s retries: 10 + gui_tests: + build: + context: . + dockerfile: ./dockerfiles/Dockerfile-gui-tests + network_mode: "host" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - "${PWD}:/build/out" + volumes: postgres-data: {} minio-data: {} diff --git a/dockerfiles/Dockerfile-gui-tests b/dockerfiles/Dockerfile-gui-tests new file mode 100644 index 000000000..4e4108d2d --- /dev/null +++ b/dockerfiles/Dockerfile-gui-tests @@ -0,0 +1,81 @@ +FROM ubuntu:22.04 AS build + +# Install packaged dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential git curl cmake gcc g++ pkg-config libmagic-dev \ + libssl-dev zlib1g-dev ca-certificates + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ + docker.io \ + gcc \ + git \ + libssl-dev \ + pkg-config \ + xz-utils + +# Install dependencies for chromium browser +RUN apt-get install -y \ + gconf-service \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgbm-dev \ + libgcc1 \ + libgconf-2-4 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libstdc++6 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + fonts-liberation \ + libappindicator1 \ + libnss3 \ + lsb-release \ + xdg-utils \ + wget + +# Install rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain nightly --no-modify-path --profile minimal +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN curl -sL https://nodejs.org/dist/v14.4.0/node-v14.4.0-linux-x64.tar.xz | tar -xJ +ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}" +ENV NODE_PATH="/node-v14.4.0-linux-x64/lib/node_modules/" + +WORKDIR /build + +RUN mkdir out + +# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries +# to create a new folder. For reference: +# https://github.com/puppeteer/puppeteer/issues/375 +# +# We also specify the version in case we need to update it to go around cache limitations. +RUN npm install -g browser-ui-test@0.16.10 --unsafe-perm=true + +EXPOSE 3000 + +CMD ["node", "/build/out/gui-tests/tester.js"] diff --git a/dockerfiles/run-gui-tests.sh b/dockerfiles/run-gui-tests.sh new file mode 100755 index 000000000..ea9fa4e06 --- /dev/null +++ b/dockerfiles/run-gui-tests.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -e + +# Just in case it's running, we stop the web server. +docker-compose stop web + +docker-compose up -d db s3 + +# If we have a .env file, we need to temporarily move it so +# it doesn't make sqlx fail compilation. +if [ -f .env ]; then + mv .env .tmp.env +fi + +# We add the information we need. +cargo run -- database migrate +cargo run -- build update-toolchain +cargo run -- build crate sysinfo 0.23.4 +cargo run -- build crate sysinfo 0.23.5 +cargo run -- build add-essential-files + +if [ -f .tmp.env ]; then + mv .tmp.env .env +fi + +# In case we don't have a `.env`, we create one. +if [ ! -f .env ]; then + cp .env.sample .env +fi + +. .env + +cargo run -- start-web-server & +SERVER_PID=$! + +# status="docker run . -v `pwd`:/build/out:ro gui_tests" +docker-compose run gui_tests +status=$? +exit $status diff --git a/gui-tests/404.goml b/gui-tests/404.goml new file mode 100644 index 000000000..546e09a80 --- /dev/null +++ b/gui-tests/404.goml @@ -0,0 +1,3 @@ +// Checks the content of the 404 page. +go-to: |DOC_PATH| + "/non-existing-crate" +assert-text: ("#crate-title", "The requested crate does not exist") diff --git a/gui-tests/basic.goml b/gui-tests/basic.goml new file mode 100644 index 000000000..3e50211bf --- /dev/null +++ b/gui-tests/basic.goml @@ -0,0 +1,14 @@ +// Checks that the "latest" URL leads us to the last version of the `sysinfo` crate. +go-to: |DOC_PATH| + "/sysinfo" +// We first check if the redirection worked as expected: +assert-document-property: ({"URL": "/sysinfo/latest/sysinfo/"}, ENDS_WITH) +// Now we go to the actual version we're interested into. +go-to: |DOC_PATH| + "/sysinfo/0.23.5/sysinfo/index.html" +assert: "//*[@class='title' and text()='sysinfo-0.23.5']" +// And we also confirm we're on a rustdoc page. +assert: "#rustdoc_body_wrapper" + +// Let's go to the docs.rs page of the crate. +go-to: |DOC_PATH| + "/crate/sysinfo/0.23.5" +assert-false: "#rustdoc_body_wrapper" +assert-text: ("#crate-title", "sysinfo 0.23.5", CONTAINS) diff --git a/gui-tests/tester.js b/gui-tests/tester.js new file mode 100644 index 000000000..ff3c9b7e9 --- /dev/null +++ b/gui-tests/tester.js @@ -0,0 +1,252 @@ +// This package needs to be install: +// +// ``` +// npm install browser-ui-test +// ``` + +const fs = require("fs"); +const path = require("path"); +const os = require('os'); +const {Options, runTest} = require('browser-ui-test'); + +function showHelp() { + console.log("docs-rs-gui-js options:"); + console.log(" --file [PATH] : file to run (can be repeated)"); + console.log(" --debug : show extra information about script run"); + console.log(" --show-text : render font in pages"); + console.log(" --no-headless : disable headless mode"); + console.log(" --help : show this message then quit"); + console.log(" --jobs [NUMBER] : number of threads to run tests on"); +} + +function isNumeric(s) { + return /^\d+$/.test(s); +} + +function parseOptions(args) { + var opts = { + "files": [], + "debug": false, + "show_text": false, + "no_headless": false, + "jobs": -1, + }; + var correspondances = { + "--debug": "debug", + "--show-text": "show_text", + "--no-headless": "no_headless", + }; + + for (var i = 0; i < args.length; ++i) { + if (args[i] === "--file" + || args[i] === "--jobs") { + i += 1; + if (i >= args.length) { + console.log("Missing argument after `" + args[i - 1] + "` option."); + return null; + } + if (args[i - 1] === "--jobs") { + if (!isNumeric(args[i])) { + console.log( + "`--jobs` option expects a positive number, found `" + args[i] + "`"); + return null; + } + opts["jobs"] = parseInt(args[i]); + } else if (args[i - 1] !== "--file") { + opts[correspondances[args[i - 1]]] = args[i]; + } else { + opts["files"].push(args[i]); + } + } else if (args[i] === "--help") { + showHelp(); + process.exit(0); + } else if (correspondances[args[i]]) { + opts[correspondances[args[i]]] = true; + } else { + console.log("Unknown option `" + args[i] + "`."); + console.log("Use `--help` to see the list of options"); + return null; + } + } + return opts; +} + +/// Print single char status information without \n +function char_printer(n_tests) { + const max_per_line = 10; + let current = 0; + return { + successful: function() { + current += 1; + if (current % max_per_line === 0) { + process.stdout.write(`. (${current}/${n_tests})${os.EOL}`); + } else { + process.stdout.write("."); + } + }, + erroneous: function() { + current += 1; + if (current % max_per_line === 0) { + process.stderr.write(`F (${current}/${n_tests})${os.EOL}`); + } else { + process.stderr.write("F"); + } + }, + finish: function() { + if (current % max_per_line === 0) { + // Don't output if we are already at a matching line end + console.log(""); + } else { + const spaces = " ".repeat(max_per_line - (current % max_per_line)); + process.stdout.write(`${spaces} (${current}/${n_tests})${os.EOL}${os.EOL}`); + } + }, + }; +} + +/// Sort array by .file_name property +function by_filename(a, b) { + return a.file_name - b.file_name; +} + +async function main(argv) { + let opts = parseOptions(argv.slice(2)); + if (opts === null) { + process.exit(1); + } + + // Print successful tests too + let debug = false; + // Run tests in sequentially + let headless = true; + const options = new Options(); + try { + // This is more convenient that setting fields one by one. + let args = [ + "--no-sandbox", + ]; + if (typeof process.env.SERVER_URL !== 'undefined') { + args.push("--variable", "DOC_PATH", process.env.SERVER_URL); + } else { + args.push("--variable", "DOC_PATH", "http://127.0.0.1:3000"); + } + if (opts["debug"]) { + debug = true; + args.push("--debug"); + } + if (opts["show_text"]) { + args.push("--show-text"); + } + if (opts["no_headless"]) { + args.push("--no-headless"); + headless = false; + } + options.parseArguments(args); + } catch (error) { + console.error(`invalid argument: ${error}`); + process.exit(1); + } + + let failed = false; + let files; + if (opts["files"].length === 0) { + files = fs.readdirSync(__dirname); + } else { + files = opts["files"]; + } + files = files.filter(file => path.extname(file) == ".goml"); + if (files.length === 0) { + console.error("No test selected"); + process.exit(2); + } + files.sort(); + + if (!headless) { + opts["jobs"] = 1; + console.log("`--no-headless` option is active, disabling concurrency for running tests."); + } + let jobs = opts["jobs"]; + + if (opts["jobs"] < 1) { + jobs = files.length; + process.setMaxListeners(files.length + 1); + } else if (headless) { + process.setMaxListeners(opts["jobs"] + 1); + } + console.log(`Running ${files.length} docs.rs GUI (${jobs} concurrently) ...`); + + const tests_queue = []; + let results = { + successful: [], + failed: [], + errored: [], + }; + const status_bar = char_printer(files.length); + for (let i = 0; i < files.length; ++i) { + const file_name = files[i]; + const testPath = path.join(__dirname, file_name); + const callback = runTest(testPath, {"options": options}) + .then(out => { + const [output, nb_failures] = out; + results[nb_failures === 0 ? "successful" : "failed"].push({ + file_name: testPath, + output: output, + }); + if (nb_failures > 0) { + status_bar.erroneous(); + failed = true; + } else { + status_bar.successful(); + } + }) + .catch(err => { + results.errored.push({ + file_name: testPath + file_name, + output: err, + }); + status_bar.erroneous(); + failed = true; + }) + .finally(() => { + // We now remove the promise from the tests_queue. + tests_queue.splice(tests_queue.indexOf(callback), 1); + }); + tests_queue.push(callback); + if (opts["jobs"] > 0 && tests_queue.length >= opts["jobs"]) { + await Promise.race(tests_queue); + } + } + if (tests_queue.length > 0) { + await Promise.all(tests_queue); + } + status_bar.finish(); + + if (debug) { + results.successful.sort(by_filename); + results.successful.forEach(r => { + console.log(r.output); + }); + } + + if (results.failed.length > 0) { + console.log(""); + results.failed.sort(by_filename); + results.failed.forEach(r => { + console.log(r.file_name, r.output); + }); + } + if (results.errored.length > 0) { + console.log(os.EOL); + // print run errors on the bottom so developers see them better + results.errored.sort(by_filename); + results.errored.forEach(r => { + console.error(r.file_name, r.output); + }); + } + + if (failed) { + process.exit(1); + } +} + +main(process.argv);