diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5dfbc81b8f..25318b5996 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,7 +25,6 @@ jobs: # Lets us manually bump the cache to rebuild ENV_CACHE_BUILD: "0" TEST_DATA_CACHE_BUILD: "2" - PY_VER: 3.8 steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -33,9 +32,9 @@ jobs: with: fetch-depth: 0 - - name: Install Nox + - name: Install ASV & Nox run: | - pip install nox + pip install asv nox - name: Cache environment directories id: cache-env-dir @@ -78,7 +77,7 @@ jobs: if [ "$first_commit" != "" ] then - nox --session="benchmarks(overnight)" -- $first_commit + python benchmarks/bm_runner.py overnight $first_commit fi - name: Create issues for performance shifts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 46cc319c49..6dd79e1c1c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -111,7 +111,7 @@ jobs: - name: "nox cache" uses: ./.github/workflows/composite/nox-cache with: - cache_build: 0 + cache_build: 1 env_name: ${{ env.ENV_NAME }} lock_file: ${{ env.LOCK_FILE }} diff --git a/benchmarks/README.md b/benchmarks/README.md index 8dffd473f3..1a7d0ddad3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -15,9 +15,20 @@ shifts in performance being flagged in a new GitHub issue. installed, as well as Nox (see [Benchmark environments](#benchmark-environments)). -[Iris' noxfile](../noxfile.py) includes a `benchmarks` session that provides -conveniences for setting up before benchmarking, and can also replicate the -automated overnight run locally. See the session docstring for detail. +The benchmark runner ([bm_runner.py](./bm_runner.py)) provides conveniences for +common benchmark setup and run tasks, including replicating the automated +overnight run locally. See `python bm_runner.py --help` for detail. + +A significant portion of benchmark run time is environment management. Run-time +can be reduced by placing the benchmark environment on the same file system as +your +[Conda package cache](https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html#specify-pkg-directories), +if it is not already. You can achieve this by either: + +- Temporarily reconfiguring `delegated_env_commands` and `delegated_env_parent` + in [asv.conf.json](asv.conf.json) to reference a location on the same file + system as the Conda package cache. +- Moving your Iris repo to the same file system as the Conda package cache. ### Environment variables @@ -26,8 +37,8 @@ automated overnight run locally. See the session docstring for detail. benchmark scripts. * `DATA_GEN_PYTHON` - required - path to a Python executable that can be used to generate benchmark test objects/files; see -[Data generation](#data-generation). The Nox session sets this automatically, -but will defer to any value already set in the shell. +[Data generation](#data-generation). The benchmark runner sets this +automatically, but will defer to any value already set in the shell. * `BENCHMARK_DATA` - optional - path to a directory for benchmark synthetic test data, which the benchmark scripts will create if it doesn't already exist. Defaults to `/benchmarks/.data/` if not set. Note that some of @@ -36,7 +47,7 @@ plan accordingly. * `ON_DEMAND_BENCHMARKS` - optional - when set (to any value): benchmarks decorated with `@on_demand_benchmark` are included in the ASV run. Usually coupled with the ASV `--bench` argument to only run the benchmark(s) of -interest. Is set during the Nox `cperf` and `sperf` sessions. +interest. Is set during the benchmark runner `cperf` and `sperf` sub-commands. ## Writing benchmarks diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 7337eaa8c7..1e726eaee5 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -19,8 +19,7 @@ // * No build-time environment variables. // * Is run in the same environment as the ASV install itself. "delegated_env_commands": [ - "sed -i 's/_PY_VERSIONS_ALL/_PY_VERSION_LATEST/g' noxfile.py", - "nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose" + "PY_VER=3.10 nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose" ], // The parent directory of the above environment. // The most recently modified environment in the directory will be used. diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py new file mode 100644 index 0000000000..86f6bd37c7 --- /dev/null +++ b/benchmarks/bm_runner.py @@ -0,0 +1,387 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Argparse conveniences for executing common types of benchmark runs. +""" + +from abc import ABC, abstractmethod +import argparse +from argparse import ArgumentParser +from datetime import datetime +from importlib import import_module +from os import environ +from pathlib import Path +import re +import subprocess +from tempfile import NamedTemporaryFile +from typing import Literal + +# The threshold beyond which shifts are 'notable'. See `asv compare`` docs +# for more. +COMPARE_FACTOR = 1.2 + + +# Common ASV arguments for all run_types except `custom`. +ASV_HARNESS = ( + "asv run {posargs} --attribute rounds=4 --interleave-rounds --strict " + "--show-stderr" +) + + +def _check_requirements(package: str) -> None: + try: + import_module(package) + except ImportError as exc: + message = ( + f"No {package} install detected. Benchmarks can only " + f"be run in an environment including {package}." + ) + raise Exception(message) from exc + + +def _prep_data_gen_env() -> None: + """ + Create/access a separate, unchanging environment for generating test data. + """ + + root_dir = Path(__file__).parents[1] + python_version = "3.10" + data_gen_var = "DATA_GEN_PYTHON" + if data_gen_var in environ: + print("Using existing data generation environment.") + else: + print("Setting up the data generation environment ...") + # Get Nox to build an environment for the `tests` session, but don't + # run the session. Will re-use a cached environment if appropriate. + subprocess.run( + [ + "nox", + f"--noxfile={root_dir / 'noxfile.py'}", + "--session=tests", + "--install-only", + f"--python={python_version}", + ] + ) + # Find the environment built above, set it to be the data generation + # environment. + data_gen_python = next( + (root_dir / ".nox").rglob(f"tests*/bin/python{python_version}") + ).resolve() + environ[data_gen_var] = str(data_gen_python) + + print("Installing Mule into data generation environment ...") + mule_dir = data_gen_python.parents[1] / "resources" / "mule" + if not mule_dir.is_dir(): + subprocess.run( + [ + "git", + "clone", + "https://github.com/metomi/mule.git", + str(mule_dir), + ] + ) + subprocess.run( + [ + str(data_gen_python), + "-m", + "pip", + "install", + str(mule_dir / "mule"), + ] + ) + + print("Data generation environment ready.") + + +def _setup_common() -> None: + _check_requirements("asv") + _check_requirements("nox") + + _prep_data_gen_env() + + print("Setting up ASV ...") + subprocess.run(["asv", "machine", "--yes"]) + + print("Setup complete.") + + +def _asv_compare(*commits: str, overnight_mode: bool = False) -> None: + """Run through a list of commits comparing each one to the next.""" + commits = [commit[:8] for commit in commits] + shifts_dir = Path(".asv") / "performance-shifts" + for i in range(len(commits) - 1): + before = commits[i] + after = commits[i + 1] + asv_command = ( + f"asv compare {before} {after} --factor={COMPARE_FACTOR} --split" + ) + subprocess.run(asv_command.split(" ")) + + if overnight_mode: + # Record performance shifts. + # Run the command again but limited to only showing performance + # shifts. + shifts = subprocess.run( + [*asv_command.split(" "), "--only-changed"], + capture_output=True, + text=True, + ).stdout + if shifts: + # Write the shifts report to a file. + # Dir is used by .github/workflows/benchmarks.yml, + # but not cached - intended to be discarded after run. + shifts_dir.mkdir(exist_ok=True, parents=True) + shifts_path = (shifts_dir / after).with_suffix(".txt") + with shifts_path.open("w") as shifts_file: + shifts_file.write(shifts) + + +class _SubParserGenerator(ABC): + """Convenience for holding all the necessary argparse info in 1 place.""" + + name: str = NotImplemented + description: str = NotImplemented + epilog: str = NotImplemented + + def __init__(self, subparsers: ArgumentParser.add_subparsers) -> None: + self.subparser: ArgumentParser = subparsers.add_parser( + self.name, + description=self.description, + epilog=self.epilog, + formatter_class=argparse.RawTextHelpFormatter, + ) + self.add_arguments() + self.subparser.add_argument( + "asv_args", + nargs=argparse.REMAINDER, + help="Any number of arguments to pass down to ASV.", + ) + self.subparser.set_defaults(func=self.func) + + @abstractmethod + def add_arguments(self) -> None: + """All self.subparser.add_argument() calls.""" + _ = NotImplemented + + @staticmethod + @abstractmethod + def func(args: argparse.Namespace): + """ + The function to return when the subparser is parsed. + + `func` is then called, performing the user's selected sub-command. + + """ + _ = args + return NotImplemented + + +class Overnight(_SubParserGenerator): + name = "overnight" + description = ( + "Benchmarks all commits between the input **first_commit** to ``HEAD``, " + "comparing each to its parent for performance shifts. If a commit causes " + "shifts, the output is saved to a file:\n" + "``.asv/performance-shifts/``\n\n" + "Designed for checking the previous 24 hours' commits, typically in a " + "scheduled script." + ) + epilog = ( + "e.g. python bm_runner.py overnight a1b23d4\n" + "e.g. python bm_runner.py overnight a1b23d4 --bench=regridding" + ) + + def add_arguments(self) -> None: + self.subparser.add_argument( + "first_commit", + type=str, + help="The first commit in the benchmarking commit sequence.", + ) + + @staticmethod + def func(args: argparse.Namespace) -> None: + _setup_common() + + commit_range = f"{args.first_commit}^^.." + asv_command = ASV_HARNESS.format(posargs=commit_range) + subprocess.run([*asv_command.split(" "), *args.asv_args]) + + # git rev-list --first-parent is the command ASV uses. + git_command = f"git rev-list --first-parent {commit_range}" + commit_string = subprocess.run( + git_command.split(" "), capture_output=True, text=True + ).stdout + commit_list = commit_string.rstrip().split("\n") + _asv_compare(*reversed(commit_list), overnight_mode=True) + + +class Branch(_SubParserGenerator): + name = "branch" + description = ( + "Performs the same operations as ``overnight``, but always on two commits " + "only - ``HEAD``, and ``HEAD``'s merge-base with the input " + "**base_branch**. Output from this run is never saved to a file. Designed " + "for testing if the active branch's changes cause performance shifts - " + "anticipating what would be caught by ``overnight`` once merged.\n\n" + "**For maximum accuracy, avoid using the machine that is running this " + "session. Run time could be >1 hour for the full benchmark suite.**" + ) + epilog = ( + "e.g. python bm_runner.py branch upstream/main\n" + "e.g. python bm_runner.py branch upstream/main --bench=regridding" + ) + + def add_arguments(self) -> None: + self.subparser.add_argument( + "base_branch", + type=str, + help="A branch that has the merge-base with ``HEAD`` - ``HEAD`` will be benchmarked against that merge-base.", + ) + + @staticmethod + def func(args: argparse.Namespace) -> None: + _setup_common() + + git_command = f"git merge-base HEAD {args.base_branch}" + merge_base = subprocess.run( + git_command.split(" "), capture_output=True, text=True + ).stdout[:8] + + with NamedTemporaryFile("w") as hashfile: + hashfile.writelines([merge_base, "\n", "HEAD"]) + hashfile.flush() + commit_range = f"HASHFILE:{hashfile.name}" + asv_command = ASV_HARNESS.format(posargs=commit_range) + subprocess.run([*asv_command.split(" "), *args.asv_args]) + + _asv_compare(merge_base, "HEAD") + + +class _CSPerf(_SubParserGenerator, ABC): + """Common code used by both CPerf and SPerf.""" + + description = ( + "Run the on-demand {} suite of benchmarks (part of the UK Met " + "Office NG-VAT project) for the ``HEAD`` of ``upstream/main`` only, " + "and publish the results to the input **publish_dir**, within a " + "unique subdirectory for this run." + ) + epilog = ( + "e.g. python bm_runner.py {0} my_publish_dir\n" + "e.g. python bm_runner.py {0} my_publish_dir --bench=regridding" + ) + + def add_arguments(self) -> None: + self.subparser.add_argument( + "publish_dir", + type=str, + help="HTML results will be published to a sub-dir in this dir.", + ) + + @staticmethod + def csperf( + args: argparse.Namespace, run_type: Literal["cperf", "sperf"] + ) -> None: + _setup_common() + + publish_dir = Path(args.publish_dir) + if not publish_dir.is_dir(): + message = ( + f"Input 'publish directory' is not a directory: {publish_dir}" + ) + raise NotADirectoryError(message) + publish_subdir = ( + publish_dir + / f"{run_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + publish_subdir.mkdir() + + # Activate on demand benchmarks (C/SPerf are deactivated for + # 'standard' runs). + environ["ON_DEMAND_BENCHMARKS"] = "True" + commit_range = "upstream/main^!" + + asv_command = ( + ASV_HARNESS.format(posargs=commit_range) + f" --bench={run_type}" + ) + # C/SPerf benchmarks are much bigger than the CI ones: + # Don't fail the whole run if memory blows on 1 benchmark. + asv_command = asv_command.replace(" --strict", "") + # Only do a single round. + asv_command = re.sub(r"rounds=\d", "rounds=1", asv_command) + subprocess.run([*asv_command.split(" "), *args.asv_args]) + + asv_command = f"asv publish {commit_range} --html-dir={publish_subdir}" + subprocess.run(asv_command.split(" ")) + + # Print completion message. + location = Path().cwd() / ".asv" + print( + f'New ASV results for "{run_type}".\n' + f'See "{publish_subdir}",' + f'\n or JSON files under "{location / "results"}".' + ) + + +class CPerf(_CSPerf): + name = "cperf" + description = _CSPerf.description.format("CPerf") + epilog = _CSPerf.epilog.format("cperf") + + @staticmethod + def func(args: argparse.Namespace) -> None: + _CSPerf.csperf(args, "cperf") + + +class SPerf(_CSPerf): + name = "sperf" + description = _CSPerf.description.format("SPerf") + epilog = _CSPerf.epilog.format("sperf") + + @staticmethod + def func(args: argparse.Namespace) -> None: + _CSPerf.csperf(args, "sperf") + + +class Custom(_SubParserGenerator): + name = "custom" + description = ( + "Run ASV with the input **ASV sub-command**, without any preset " + "arguments - must all be supplied by the user. So just like running " + "ASV manually, with the convenience of re-using the runner's " + "scripted setup steps." + ) + epilog = "e.g. python bm_runner.py custom continuous a1b23d4 HEAD --quick" + + def add_arguments(self) -> None: + self.subparser.add_argument( + "asv_sub_command", + type=str, + help="The ASV command to run.", + ) + + @staticmethod + def func(args: argparse.Namespace) -> None: + _setup_common() + subprocess.run(["asv", args.asv_sub_command, *args.asv_args]) + + +def main(): + parser = ArgumentParser( + description="Run the Iris performance benchmarks (using Airspeed Velocity).", + epilog="More help is available within each sub-command.", + ) + subparsers = parser.add_subparsers(required=True) + + for gen in (Overnight, Branch, CPerf, SPerf, Custom): + _ = gen(subparsers).subparser + + parsed = parser.parse_args() + parsed.func(parsed) + + +if __name__ == "__main__": + main() diff --git a/docs/src/developers_guide/contributing_benchmarks.rst b/docs/src/developers_guide/contributing_benchmarks.rst index 65bc9635b6..017617161f 100644 --- a/docs/src/developers_guide/contributing_benchmarks.rst +++ b/docs/src/developers_guide/contributing_benchmarks.rst @@ -23,7 +23,7 @@ previous day** to check if any commit has introduced performance shifts. Detected shifts are reported in a new Iris GitHub issue. If a pull request author/reviewer suspects their changes may cause performance -shifts, a convenience is available (currently via Nox) to replicate the +shifts, a convenience script is available to replicate the overnight benchmark run but comparing the current ``HEAD`` with a requested branch (e.g. ``upstream/main``). Read more in `benchmarks/README.md`_. diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 7b8f3965bf..904cb106ed 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -46,7 +46,7 @@ This document explains the changes made to Iris for this release :meth:`iris.cube.Cube.rolling_window`). This automatically adapts cube units if necessary. (:pull:`5084`) -#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added :func:`iris.plot.hist` +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added :func:`iris.plot.hist` and :func:`iris.quickplot.hist`. (:pull:`5189`) @@ -144,6 +144,9 @@ This document explains the changes made to Iris for this release #. `@agriyakhetarpal`_ prevented the GitHub action for publishing releases to PyPI from running in forks. (:pull:`5220`) +#. `@trexfeathers`_ moved the benchmark runner conveniences from ``noxfile.py`` + to a dedicated ``benchmarks/bm_runner.py``. (:pull:`5215`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/tests/test_coding_standards.py b/lib/iris/tests/test_coding_standards.py index b52934c568..e3786f5cd5 100644 --- a/lib/iris/tests/test_coding_standards.py +++ b/lib/iris/tests/test_coding_standards.py @@ -14,6 +14,7 @@ import os from pathlib import Path import subprocess +from typing import List, Tuple import iris from iris.fileformats.netcdf import _thread_safe_nc @@ -52,8 +53,7 @@ def test_netcdf4_import(): files_including_import = [] for file_path in Path(IRIS_DIR).rglob("*.py"): - with file_path.open("r") as open_file: - file_text = open_file.read() + file_text = file_path.read_text() if any([i in file_text for i in import_strings]): files_including_import.append(file_path) @@ -66,6 +66,73 @@ def test_netcdf4_import(): assert set(files_including_import) == set(expected) +def test_python_versions(): + """ + This test is designed to fail whenever Iris' supported Python versions are + updated, insisting that versions are updated EVERYWHERE in-sync. + """ + latest_supported = "3.10" + all_supported = ["3.8", "3.9", latest_supported] + + root_dir = Path(__file__).parents[3] + workflows_dir = root_dir / ".github" / "workflows" + benchmarks_dir = root_dir / "benchmarks" + + # Places that are checked: + setup_cfg_file = root_dir / "setup.cfg" + requirements_dir = root_dir / "requirements" + nox_file = root_dir / "noxfile.py" + ci_wheels_file = workflows_dir / "ci-wheels.yml" + ci_tests_file = workflows_dir / "ci-tests.yml" + asv_config_file = benchmarks_dir / "asv.conf.json" + benchmark_runner_file = benchmarks_dir / "bm_runner.py" + + text_searches: List[Tuple[Path, str]] = [ + ( + setup_cfg_file, + "\n ".join( + [ + "Programming Language :: Python :: " + ver + for ver in all_supported + ] + ), + ), + ( + nox_file, + "_PY_VERSIONS_ALL = [" + + ", ".join([f'"{ver}"' for ver in all_supported]), + ), + ( + ci_wheels_file, + "python-version: [" + + ", ".join([f'"{ver}"' for ver in all_supported]), + ), + ( + ci_tests_file, + ( + f'python-version: ["{latest_supported}"]\n' + f'{" " * 8}session: ["doctest", "gallery", "linkcheck"]' + ), + ), + (asv_config_file, f"PY_VER={latest_supported}"), + (benchmark_runner_file, f'python_version = "{latest_supported}"'), + ] + + for ver in all_supported: + req_yaml = requirements_dir / f"py{ver.replace('.', '')}.yml" + text_searches.append((req_yaml, f"- python ={ver}")) + + text_searches.append( + ( + ci_tests_file, + f'python-version: "{ver}"\n{" " * 12}session: "tests"', + ) + ) + + for path, search in text_searches: + assert search in path.read_text() + + class TestLicenseHeaders(tests.IrisTest): @staticmethod def whatchanged_parse(whatchanged_output): diff --git a/noxfile.py b/noxfile.py index f3210cfc1b..d34155ecaf 100755 --- a/noxfile.py +++ b/noxfile.py @@ -5,13 +5,9 @@ """ -from datetime import datetime import hashlib import os from pathlib import Path -import re -from tempfile import NamedTemporaryFile -from typing import Literal import nox from nox.logger import logger @@ -20,7 +16,7 @@ nox.options.reuse_existing_virtualenvs = True #: Python versions we can run sessions under -_PY_VERSIONS_ALL = ["3.8"] +_PY_VERSIONS_ALL = ["3.8", "3.9", "3.10"] _PY_VERSION_LATEST = _PY_VERSIONS_ALL[-1] #: One specific python version for docs builds @@ -303,236 +299,3 @@ def wheel(session: nox.sessions.Session): "import iris; print(f'{iris.__version__=}')", external=True, ) - - -@nox.session -@nox.parametrize( - "run_type", - ["overnight", "branch", "cperf", "sperf", "custom"], - ids=["overnight", "branch", "cperf", "sperf", "custom"], -) -def benchmarks( - session: nox.sessions.Session, - run_type: Literal["overnight", "branch", "cperf", "sperf", "custom"], -): - """ - Perform Iris performance benchmarks (using Airspeed Velocity). - - All run types require a single Nox positional argument (e.g. - ``nox --session="foo" -- my_pos_arg``) - detailed in the parameters - section - and can optionally accept a series of further arguments that will - be added to session's ASV command. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - run_type: {"overnight", "branch", "cperf", "sperf", "custom"} - * ``overnight``: benchmarks all commits between the input **first - commit** to ``HEAD``, comparing each to its parent for performance - shifts. If a commit causes shifts, the output is saved to a file: - ``.asv/performance-shifts/``. Designed for checking the - previous 24 hours' commits, typically in a scheduled script. - * ``branch``: Performs the same operations as ``overnight``, but always - on two commits only - ``HEAD``, and ``HEAD``'s merge-base with the - input **base branch**. Output from this run is never saved to a file. - Designed for testing if the active branch's changes cause performance - shifts - anticipating what would be caught by ``overnight`` once - merged. - **For maximum accuracy, avoid using the machine that is running this - session. Run time could be >1 hour for the full benchmark suite.** - * ``cperf``: Run the on-demand CPerf suite of benchmarks (part of the - UK Met Office NG-VAT project) for the ``HEAD`` of ``upstream/main`` - only, and publish the results to the input **publish directory**, - within a unique subdirectory for this run. - * ``sperf``: As with CPerf, but for the SPerf suite. - * ``custom``: run ASV with the input **ASV sub-command**, without any - preset arguments - must all be supplied by the user. So just like - running ASV manually, with the convenience of re-using the session's - scripted setup steps. - - Examples - -------- - * ``nox --session="benchmarks(overnight)" -- a1b23d4`` - * ``nox --session="benchmarks(branch)" -- upstream/main`` - * ``nox --session="benchmarks(branch)" -- upstream/mesh-data-model`` - * ``nox --session="benchmarks(branch)" -- upstream/main --bench=regridding`` - * ``nox --session="benchmarks(cperf)" -- my_publish_dir - * ``nox --session="benchmarks(custom)" -- continuous a1b23d4 HEAD --quick`` - - """ - # The threshold beyond which shifts are 'notable'. See `asv compare`` docs - # for more. - COMPARE_FACTOR = 1.2 - - session.install("asv", "nox") - - data_gen_var = "DATA_GEN_PYTHON" - if data_gen_var in os.environ: - print("Using existing data generation environment.") - else: - print("Setting up the data generation environment...") - # Get Nox to build an environment for the `tests` session, but don't - # run the session. Will re-use a cached environment if appropriate. - session.run_always( - "nox", - "--session=tests", - "--install-only", - f"--python={_PY_VERSION_LATEST}", - ) - # Find the environment built above, set it to be the data generation - # environment. - data_gen_python = next( - Path(".nox").rglob(f"tests*/bin/python{_PY_VERSION_LATEST}") - ).resolve() - session.env[data_gen_var] = data_gen_python - - mule_dir = data_gen_python.parents[1] / "resources" / "mule" - if not mule_dir.is_dir(): - print("Installing Mule into data generation environment...") - session.run_always( - "git", - "clone", - "https://github.com/metomi/mule.git", - str(mule_dir), - external=True, - ) - session.run_always( - str(data_gen_python), - "-m", - "pip", - "install", - str(mule_dir / "mule"), - external=True, - ) - - print("Running ASV...") - session.cd("benchmarks") - # Skip over setup questions for a new machine. - session.run("asv", "machine", "--yes") - - # All run types require one Nox posarg. - run_type_arg = { - "overnight": "first commit", - "branch": "base branch", - "cperf": "publish directory", - "sperf": "publish directory", - "custom": "ASV sub-command", - } - if run_type not in run_type_arg.keys(): - message = f"Unsupported run-type: {run_type}" - raise NotImplementedError(message) - if not session.posargs: - message = ( - f"Missing mandatory first Nox session posarg: " - f"{run_type_arg[run_type]}" - ) - raise ValueError(message) - first_arg = session.posargs[0] - # Optional extra arguments to be passed down to ASV. - asv_args = session.posargs[1:] - - def asv_compare(*commits): - """Run through a list of commits comparing each one to the next.""" - commits = [commit[:8] for commit in commits] - shifts_dir = Path(".asv") / "performance-shifts" - for i in range(len(commits) - 1): - before = commits[i] - after = commits[i + 1] - asv_command_ = f"asv compare {before} {after} --factor={COMPARE_FACTOR} --split" - session.run(*asv_command_.split(" ")) - - if run_type == "overnight": - # Record performance shifts. - # Run the command again but limited to only showing performance - # shifts. - shifts = session.run( - *asv_command_.split(" "), "--only-changed", silent=True - ) - if shifts: - # Write the shifts report to a file. - # Dir is used by .github/workflows/benchmarks.yml, - # but not cached - intended to be discarded after run. - shifts_dir.mkdir(exist_ok=True, parents=True) - shifts_path = (shifts_dir / after).with_suffix(".txt") - with shifts_path.open("w") as shifts_file: - shifts_file.write(shifts) - - # Common ASV arguments for all run_types except `custom`. - asv_harness = ( - "asv run {posargs} --attribute rounds=4 --interleave-rounds --strict " - "--show-stderr" - ) - - if run_type == "overnight": - first_commit = first_arg - commit_range = f"{first_commit}^^.." - asv_command = asv_harness.format(posargs=commit_range) - session.run(*asv_command.split(" "), *asv_args) - - # git rev-list --first-parent is the command ASV uses. - git_command = f"git rev-list --first-parent {commit_range}" - commit_string = session.run( - *git_command.split(" "), silent=True, external=True - ) - commit_list = commit_string.rstrip().split("\n") - asv_compare(*reversed(commit_list)) - - elif run_type == "branch": - base_branch = first_arg - git_command = f"git merge-base HEAD {base_branch}" - merge_base = session.run( - *git_command.split(" "), silent=True, external=True - )[:8] - - with NamedTemporaryFile("w") as hashfile: - hashfile.writelines([merge_base, "\n", "HEAD"]) - hashfile.flush() - commit_range = f"HASHFILE:{hashfile.name}" - asv_command = asv_harness.format(posargs=commit_range) - session.run(*asv_command.split(" "), *asv_args) - - asv_compare(merge_base, "HEAD") - - elif run_type in ("cperf", "sperf"): - publish_dir = Path(first_arg) - if not publish_dir.is_dir(): - message = ( - f"Input 'publish directory' is not a directory: {publish_dir}" - ) - raise NotADirectoryError(message) - publish_subdir = ( - publish_dir - / f"{run_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - ) - publish_subdir.mkdir() - - # Activate on demand benchmarks (C/SPerf are deactivated for 'standard' runs). - session.env["ON_DEMAND_BENCHMARKS"] = "True" - commit_range = "upstream/main^!" - - asv_command = ( - asv_harness.format(posargs=commit_range) + f" --bench={run_type}" - ) - # C/SPerf benchmarks are much bigger than the CI ones: - # Don't fail the whole run if memory blows on 1 benchmark. - asv_command = asv_command.replace(" --strict", "") - # Only do a single round. - asv_command = re.sub(r"rounds=\d", "rounds=1", asv_command) - session.run(*asv_command.split(" "), *asv_args) - - asv_command = f"asv publish {commit_range} --html-dir={publish_subdir}" - session.run(*asv_command.split(" ")) - - # Print completion message. - location = Path().cwd() / ".asv" - print( - f'New ASV results for "{run_type}".\n' - f'See "{publish_subdir}",' - f'\n or JSON files under "{location / "results"}".' - ) - - else: - asv_subcommand = first_arg - assert run_type == "custom" - session.run("asv", asv_subcommand, *asv_args)