From 1344cddbebb0c3e06ec581820615ad8be35e0c03 Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Tue, 24 Sep 2024 14:44:34 +0100 Subject: [PATCH] Skip uninstallable tags in QPY backwards compatibility tests (#13202) * Skip uninstallable tags in QPY backwards compatibility tests When a tag has been made, but the package has not yet landed on PyPI, the QPY job fails in the environment-building step. This is not actually a failure of the QPY backwards-compatibility guarantees, and it isn't the job of the QPY tests to detect a bad tag anyway. * Query PyPI to find versions to test The logic we actually want for versions to test is "find the versions that _should_ be installable", rather than "try it and see"; the latter is susceptible to silently suppressing errors. This new form now queries PyPI to find what versions of Qiskit are available in binary distributions for this platform, and filters based on that. * Include release candidates in testing --- .azure/lint_docs_qpy-linux.yml | 2 +- test/qpy_compat/compare_versions.py | 40 ------------ test/qpy_compat/get_versions.py | 99 +++++++++++++++++++++++++++++ test/qpy_compat/process_version.sh | 76 +++++++++++----------- test/qpy_compat/run_tests.sh | 18 ++++-- 5 files changed, 150 insertions(+), 85 deletions(-) delete mode 100755 test/qpy_compat/compare_versions.py create mode 100644 test/qpy_compat/get_versions.py diff --git a/.azure/lint_docs_qpy-linux.yml b/.azure/lint_docs_qpy-linux.yml index 99d2b7dc41c9..c08ea702ede5 100644 --- a/.azure/lint_docs_qpy-linux.yml +++ b/.azure/lint_docs_qpy-linux.yml @@ -61,6 +61,6 @@ jobs: pushd test/qpy_compat ./run_tests.sh popd - mkdir qpy_files || : + mkdir -p qpy_files mv test/qpy_compat/qpy_* qpy_files/. displayName: 'Run QPY backwards compat tests' diff --git a/test/qpy_compat/compare_versions.py b/test/qpy_compat/compare_versions.py deleted file mode 100755 index 5deec1b9ca5b..000000000000 --- a/test/qpy_compat/compare_versions.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# This code is part of Qiskit. -# -# (C) Copyright IBM 2023. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -"""Compare Qiskit versions to determine if we should run qpy compat tests.""" - -import argparse -import sys - -from qiskit.qpy.interface import VERSION_PATTERN_REGEX - - -def main(): - """Main function.""" - parser = argparse.ArgumentParser(prog="compare_version", description="Compare version strings") - parser.add_argument( - "source_version", help="Source version of Qiskit that is generating the payload" - ) - parser.add_argument("test_version", help="Version under test that will load the QPY payload") - args = parser.parse_args() - source_match = VERSION_PATTERN_REGEX.search(args.source_version) - target_match = VERSION_PATTERN_REGEX.search(args.test_version) - source_version = tuple(int(x) for x in source_match.group("release").split(".")) - target_version = tuple(int(x) for x in target_match.group("release").split(".")) - if source_version > target_version: - sys.exit(1) - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/test/qpy_compat/get_versions.py b/test/qpy_compat/get_versions.py new file mode 100644 index 000000000000..a89e3b8508de --- /dev/null +++ b/test/qpy_compat/get_versions.py @@ -0,0 +1,99 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2024. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""List the versions of Qiskit and Terra that should be tested for QPY compatibility.""" + +import json +import sys +import urllib.request + +import packaging.version +import packaging.tags + +import qiskit + + +def tags_from_wheel_name(wheel: str): + """Extract the wheel tag from its filename.""" + assert wheel.lower().endswith(".whl") + # For more information, see: + # - https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + # - https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + # + # In particular, take note that a wheel's filename can include "compressed tag sets" (and our + # Linux wheels generally do this), which is why this function returns an iterable of tags. + _prefix, interpreters, abis, platforms = wheel[:-4].rsplit("-", 3) + yield from ( + packaging.tags.Tag(interpreter, abi, platform) + for interpreter in interpreters.split(".") + for abi in abis.split(".") + for platform in platforms.split(".") + ) + + +def available_versions(): + """Get all the versions of Qiskit that support exporting QPY, and are installable with the + active version of Python on this platform.""" + our_version = packaging.version.parse(qiskit.__version__) + supported_tags = set(packaging.tags.sys_tags()) + + def available_versions_for_package(package, min_version=None, max_version=None): + with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/json") as fd: + data = json.load(fd) + min_version = min_version and packaging.version.parse(min_version) + max_version = max_version and packaging.version.parse(max_version) + for other_version, payload in data["releases"].items(): + other_version = packaging.version.parse(other_version) + if min_version is not None and other_version < min_version: + continue + if max_version is not None and other_version >= max_version: + continue + if other_version > our_version: + continue + if other_version.pre is not None and other_version.pre[0] in ("a", "b"): + # We skip alpha and beta prereleases, but we currently want to test for + # compatibility with release candidates. + continue + # Note: this ignores versions that are uninstallable because we're using a Python + # version that's too new, which can be a problem for the oldest Terras, especially from + # before we built for abi3. We're not counting sdists, since if we didn't release a + # compatible wheel for the current Python version, there's no guarantee it'll install. + if not any( + tag in supported_tags + for release in payload + if release["packagetype"] == "bdist_wheel" + for tag in tags_from_wheel_name(release["filename"]) + ): + print( + f"skipping '{other_version}', which has no installable binary artifacts", + file=sys.stderr, + ) + continue + yield other_version + + yield from ( + ("qiskit-terra", version) + for version in available_versions_for_package("qiskit-terra", "0.18.0", "1.0.0") + ) + yield from ( + ("qiskit", version) for version in available_versions_for_package("qiskit", "1.0.0") + ) + + +def main(): + """main""" + for package, version in available_versions(): + print(package, version) + + +if __name__ == "__main__": + main() diff --git a/test/qpy_compat/process_version.sh b/test/qpy_compat/process_version.sh index 3fc34557b69e..c56c44f554bb 100755 --- a/test/qpy_compat/process_version.sh +++ b/test/qpy_compat/process_version.sh @@ -14,50 +14,50 @@ set -e set -x -# version is the source version, this is the release with which to generate -# qpy files with to load with the version under test -version=$1 -parts=( ${version//./ } ) -# qiskit_version is the version under test, We're testing that we can correctly -# read the qpy files generated with source version with this version. -qiskit_version=`./qiskit_venv/bin/python -c "import qiskit;print(qiskit.__version__)"` -qiskit_parts=( ${qiskit_version//./ } ) +function usage { + echo "usage: ${BASH_SOURCE[0]} -p /path/to/qiskit/python " 1>&2 + exit 1 +} - -# If source version is less than 0.18 QPY didn't exist yet so exit fast -if [[ ${parts[0]} -eq 0 && ${parts[1]} -lt 18 ]] ; then - exit 0 +python="python" +while getopts "p:" opt; do + case "$opt" in + p) + python="$OPTARG" + ;; + *) + usage + ;; + esac +done +shift "$((OPTIND-1))" +if [[ $# != 2 ]]; then + usage fi -# Exclude any non-rc pre-releases as they don't have stable API guarantees -if [[ $version == *"b"* || $version == *"a"* ]] ; then - exit 0 -fi +# `package` is the name of the Python distribution to install (qiskit or qiskit-terra). `version` is +# the source version: the release with which to generate qpy files with to load with the version +# under test. +package="$1" +version="$2" -# If the source version is newer than the version under test exit fast because -# there is no QPY compatibility for loading a qpy file generated from a newer -# release with an older release of Qiskit. -if ! ./qiskit_venv/bin/python ./compare_versions.py "$version" "$qiskit_version" ; then - exit 0 -fi +our_dir="$(realpath -- "$(dirname -- "${BASH_SOURCE[0]}")")" +cache_dir="$(pwd -P)/qpy_$version" +venv_dir="$(pwd -P)/${version}" -if [[ ! -d qpy_$version ]] ; then - echo "Building venv for qiskit-terra $version" - python -m venv $version - if [[ ${parts[0]} -eq 0 ]] ; then - ./$version/bin/pip install -c qpy_test_constraints.txt "qiskit-terra==$version" - else - ./$version/bin/pip install -c qpy_test_constraints.txt "qiskit==$version" - fi - mkdir qpy_$version - pushd qpy_$version - echo "Generating qpy files with qiskit-terra $version" - ../$version/bin/python ../test_qpy.py generate --version=$version +if [[ ! -d $cache_dir ]] ; then + echo "Building venv for $package==$version" + "$python" -m venv "$venv_dir" + "$venv_dir/bin/pip" install -c "${our_dir}/qpy_test_constraints.txt" "${package}==${version}" + mkdir "$cache_dir" + pushd "$cache_dir" + echo "Generating QPY files with $package==$version" + "$venv_dir/bin/python" "${our_dir}/test_qpy.py" generate --version="$version" else echo "Using cached QPY files for $version" - pushd qpy_$version + pushd "${cache_dir}" fi -echo "Loading qpy files from $version with dev qiskit-terra" -../qiskit_venv/bin/python ../test_qpy.py load --version=$version +echo "Loading qpy files from $version with dev Qiskit" +"$python" "${our_dir}/test_qpy.py" load --version="$version" popd -rm -rf ./$version +rm -rf "$venv_dir" diff --git a/test/qpy_compat/run_tests.sh b/test/qpy_compat/run_tests.sh index 3810773ec0f3..437585bac581 100755 --- a/test/qpy_compat/run_tests.sh +++ b/test/qpy_compat/run_tests.sh @@ -12,6 +12,7 @@ # copyright notice, and modified files need to carry a notice indicating # that they have been altered from the originals. set -e +set -o pipefail set -x # Set fixed hash seed to ensure set orders are identical between saving and @@ -19,12 +20,17 @@ set -x export PYTHONHASHSEED=$(python -S -c "import random; print(random.randint(1, 4294967295))") echo "PYTHONHASHSEED=$PYTHONHASHSEED" -python -m venv qiskit_venv -qiskit_venv/bin/pip install -c ../../constraints.txt ../.. +our_dir="$(realpath -- "$(dirname -- "${BASH_SOURCE[0]}")")" -parallel bash ./process_version.sh ::: `git tag --sort=-creatordate` +qiskit_venv="$(pwd -P)/qiskit_venv" +qiskit_python="$qiskit_venv/bin/python" +python -m venv "$qiskit_venv" +# `packaging` is needed for the `get_versions.py` script. +"$qiskit_venv/bin/pip" install -c "$our_dir/../../constraints.txt" "$our_dir/../.." packaging + +"$qiskit_python" "$our_dir/get_versions.py" | parallel --colsep=" " bash "$our_dir/process_version.sh" -p "$qiskit_python" # Test dev compatibility -dev_version=`qiskit_venv/bin/python -c 'import qiskit;print(qiskit.__version__)'` -qiskit_venv/bin/python test_qpy.py generate --version=$dev_version -qiskit_venv/bin/python test_qpy.py load --version=$dev_version +dev_version="$("$qiskit_python" -c 'import qiskit; print(qiskit.__version__)')" +"$qiskit_python" "$our_dir/test_qpy.py" generate --version="$dev_version" +"$qiskit_python" "$our_dir/test_qpy.py" load --version="$dev_version"