From 07e4ce2c80472ac618a0585e57758348d94492bb Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 29 Mar 2023 13:52:23 -0400 Subject: [PATCH] Add support for Python 3.11 (#1685) --- .buildkite/it/pipeline.yml | 4 ++-- .buildkite/it/run.sh | 3 +-- .ci/variables.json | 7 ++++--- .github/workflows/ci.yml | 6 +++--- .pylintrc | 4 ++-- .readthedocs.yml => .readthedocs.yaml | 7 ++++++- Makefile | 4 +++- esrally/driver/driver.py | 17 +++++++---------- esrally/mechanic/supplier.py | 2 +- esrally/utils/net.py | 20 ++++++++++---------- noxfile.py | 4 ++-- pyproject.toml | 7 ++++--- tests/driver/driver_test.py | 2 -- tests/track/loader_test.py | 3 +-- 14 files changed, 46 insertions(+), 44 deletions(-) rename .readthedocs.yml => .readthedocs.yaml (62%) diff --git a/.buildkite/it/pipeline.yml b/.buildkite/it/pipeline.yml index 5cfa48749..eb0aae8a9 100644 --- a/.buildkite/it/pipeline.yml +++ b/.buildkite/it/pipeline.yml @@ -5,5 +5,5 @@ agents: steps: - label: "Run 3.8 integration tests :test_tube:" command: bash .buildkite/it/run.sh 3.8 - - label: "Run 3.10 integration tests :test_tube:" - command: bash .buildkite/it/run.sh 3.10 + - label: "Run 3.11 integration tests :test_tube:" + command: bash .buildkite/it/run.sh 3.11 diff --git a/.buildkite/it/run.sh b/.buildkite/it/run.sh index dafa6d9c2..4c02dce31 100644 --- a/.buildkite/it/run.sh +++ b/.buildkite/it/run.sh @@ -49,8 +49,7 @@ retry 5 sudo apt-get update retry 5 sudo apt-get install -y \ "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-dev" "python${PYTHON_VERSION}-venv" \ git make jq docker \ - openjdk-17-jdk-headless openjdk-11-jdk-headless \ - zlib1g zlib1g-dev libssl-dev libbz2-dev libsqlite3-dev + openjdk-17-jdk-headless openjdk-11-jdk-headless export JAVA11_HOME=/usr/lib/jvm/java-11-openjdk-amd64 export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64 diff --git a/.ci/variables.json b/.ci/variables.json index 4bdd64342..ecede5bb4 100644 --- a/.ci/variables.json +++ b/.ci/variables.json @@ -6,9 +6,10 @@ "WHEEL": "0.37.1" }, "python_versions": { - "PY38": "3.8.13", - "PY39": "3.9.13", - "PY310": "3.10.4", + "PY38": "3.8.16", + "PY39": "3.9.16", + "PY310": "3.10.10", + "PY310": "3.11.2", "MIN_PY_VER": "3.8.13" } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f11b7158..41a93f64b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: pyproject.toml - name: "Install dependencies" @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] os: - macos-latest - ubuntu-latest @@ -50,7 +50,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: pyproject.toml - uses: actions/setup-java@v3 diff --git a/.pylintrc b/.pylintrc index f5e760696..a23ec39a2 100644 --- a/.pylintrc +++ b/.pylintrc @@ -531,5 +531,5 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 62% rename from .readthedocs.yml rename to .readthedocs.yaml index 58d984b64..e371301f1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yaml @@ -1,7 +1,11 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: "3.8" install: - method: pip path: . @@ -10,3 +14,4 @@ python: sphinx: fail_on_warning: true + configuration: docs/conf.py diff --git a/Makefile b/Makefile index f47d29e51..c78bc753b 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ PIP_WRAPPER := $(PY_BIN) -m pip export PY38 := $(shell jq -r '.python_versions.PY38' .ci/variables.json) export PY39 := $(shell jq -r '.python_versions.PY39' .ci/variables.json) export PY310 := $(shell jq -r '.python_versions.PY310' .ci/variables.json) +export PY311 := $(shell jq -r '.python_versions.PY311' .ci/variables.json) export HATCH_VERSION := $(shell jq -r '.prerequisite_versions.HATCH' .ci/variables.json) export HATCHLING_VERSION := $(shell jq -r '.prerequisite_versions.HATCHLING' .ci/variables.json) export PIP_VERSION := $(shell jq -r '.prerequisite_versions.PIP' .ci/variables.json) @@ -40,6 +41,7 @@ prereq: pyenv install --skip-existing $(PY38) pyenv install --skip-existing $(PY39) pyenv install --skip-existing $(PY310) + pyenv install --skip-existing $(PY311) pyenv local $(PY38) @# Ensure all Python versions are registered for this project @ jq -r '.python_versions | [.[] | tostring] | join("\n")' .ci/variables.json > .python-version @@ -100,7 +102,7 @@ test: check-venv # checks min and max python versions it: check-venv python-caches-clean . $(VENV_ACTIVATE_FILE); nox -s test-3.8 - . $(VENV_ACTIVATE_FILE); nox -s test-3.10 + . $(VENV_ACTIVATE_FILE); nox -s test-3.11 check-all: lint test it diff --git a/esrally/driver/driver.py b/esrally/driver/driver.py index a10d626e5..b0ab4de06 100644 --- a/esrally/driver/driver.py +++ b/esrally/driver/driver.py @@ -1713,16 +1713,13 @@ def __init__(self, cfg, track, task_allocations, sampler, cancel, complete, abor self.logger = logging.getLogger(__name__) def __call__(self, *args, **kwargs): - # only possible in Python 3.7+ (has introduced get_running_loop) - # try: - # loop = asyncio.get_running_loop() - # except RuntimeError: - # loop = asyncio.new_event_loop() - # asyncio.set_event_loop(loop) - loop = asyncio.new_event_loop() - loop.set_debug(self.debug_event_loop) - loop.set_exception_handler(self._logging_exception_handler) - asyncio.set_event_loop(loop) + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + loop.set_debug(self.debug_event_loop) + loop.set_exception_handler(self._logging_exception_handler) + asyncio.set_event_loop(loop) try: loop.run_until_complete(self.run()) finally: diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index b2087748f..344c44453 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -701,7 +701,7 @@ def _try_init(self, may_skip_init=False): elif os.path.isdir(self.src_dir) and may_skip_init: self.logger.info("Skipping repository initialization for %s.", self.name) else: - exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name) + raise exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name) def _update(self, revision): if self.has_remote() and revision == "latest": diff --git a/esrally/utils/net.py b/esrally/utils/net.py index 1a086aa7e..b39600bb4 100644 --- a/esrally/utils/net.py +++ b/esrally/utils/net.py @@ -262,16 +262,16 @@ def download(url, local_path, expected_size_in_bytes=None, progress_indicator=No if os.path.isfile(tmp_data_set_path): os.remove(tmp_data_set_path) raise - else: - download_size = os.path.getsize(tmp_data_set_path) - if expected_size_in_bytes is not None and download_size != expected_size_in_bytes: - if os.path.isfile(tmp_data_set_path): - os.remove(tmp_data_set_path) - raise exceptions.DataError( - "Download of [%s] is corrupt. Downloaded [%d] bytes but [%d] bytes are expected. Please retry." - % (local_path, download_size, expected_size_in_bytes) - ) - os.rename(tmp_data_set_path, local_path) + + download_size = os.path.getsize(tmp_data_set_path) + if expected_size_in_bytes is not None and download_size != expected_size_in_bytes: + if os.path.isfile(tmp_data_set_path): + os.remove(tmp_data_set_path) + raise exceptions.DataError( + "Download of [%s] is corrupt. Downloaded [%d] bytes but [%d] bytes are expected. Please retry." + % (local_path, download_size, expected_size_in_bytes) + ) + os.rename(tmp_data_set_path, local_path) def retrieve_content_as_string(url): diff --git a/noxfile.py b/noxfile.py index 65a769f77..4dbc5d985 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,13 +1,13 @@ import nox -@nox.session(python=["3.8", "3.9", "3.10"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11"]) def test(session: nox.Session) -> None: session.install(".[develop]") session.run("pytest") -@nox.session(python=["3.8", "3.9", "3.10"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11"]) def it(session: nox.Session) -> None: session.install(".[develop]") session.run("pytest", "-s", "it") diff --git a/pyproject.toml b/pyproject.toml index 63fabb849..182897368 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers=[ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", ] ################################################################################################ @@ -48,7 +49,7 @@ dependencies = [ "urllib3==1.26.9", "docker==6.0.0", # License: BSD - "psutil==5.8.0", + "psutil==5.9.4", # License: MIT "py-cpuinfo==7.0.0", # License: MIT @@ -67,7 +68,7 @@ dependencies = [ # License: MPL 2.0 "certifi", # License: MIT - "yappi==1.3.3", + "yappi==1.4.0", # License: BSD "ijson==2.6.1", # License: Apache 2.0 @@ -105,7 +106,7 @@ develop = [ "wheel==0.37.1", "github3.py==3.2.0", "pre-commit==2.20.0", - "pylint==2.14.5", + "pylint==2.17.0", "trustme==0.9.0", "GitPython==3.1.30", ] diff --git a/tests/driver/driver_test.py b/tests/driver/driver_test.py index b317cecb6..acabe2494 100644 --- a/tests/driver/driver_test.py +++ b/tests/driver/driver_test.py @@ -236,8 +236,6 @@ def test_client_reaches_join_point_which_completes_parent(self): assert d.current_step == 0 assert len(d.workers_completed_current_step) == 0 - # this requires at least Python 3.6 - # target.on_task_finished.assert_called_once() assert target.on_task_finished.call_count == 1 assert target.drive_at.call_count == 4 diff --git a/tests/track/loader_test.py b/tests/track/loader_test.py index 2041facbe..1d79e49a0 100644 --- a/tests/track/loader_test.py +++ b/tests/track/loader_test.py @@ -896,9 +896,8 @@ def dummy_read_glob(c): base_path = "~/.rally/benchmarks/tracks/default/geonames" template_file_name = "track.json" tmpl_src = loader.TemplateSource(base_path, template_file_name) - # pylint: disable=trailing-whitespace expected_response = textwrap.dedent( - """ + """ {% import "rally.helpers" as rally with context %} { "version": 2,