Add support for Python 3.11 (#1685)

.buildkite/it/pipeline.yml

@@ -5,5 +5,5 @@ agents:
 steps:
   - label: "Run 3.8 integration tests :test_tube:"
     command: bash .buildkite/it/run.sh 3.8
-  - label: "Run 3.10 integration tests :test_tube:"
-    command: bash .buildkite/it/run.sh 3.10
+  - label: "Run 3.11 integration tests :test_tube:"
+    command: bash .buildkite/it/run.sh 3.11

.buildkite/it/run.sh

@@ -49,8 +49,7 @@ retry 5 sudo apt-get update
 retry 5 sudo apt-get install -y \
     "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-dev" "python${PYTHON_VERSION}-venv" \
     git make jq docker \
-    openjdk-17-jdk-headless openjdk-11-jdk-headless \
-    zlib1g zlib1g-dev libssl-dev libbz2-dev libsqlite3-dev
+    openjdk-17-jdk-headless openjdk-11-jdk-headless
 export JAVA11_HOME=/usr/lib/jvm/java-11-openjdk-amd64
 export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64
 

.ci/variables.json

@@ -6,9 +6,10 @@
     "WHEEL": "0.37.1"
   },
   "python_versions": {
-    "PY38": "3.8.13",
-    "PY39": "3.9.13",
-    "PY310": "3.10.4",
+    "PY38": "3.8.16",
+    "PY39": "3.9.16",
+    "PY310": "3.10.10",
+    "PY310": "3.11.2",
     "MIN_PY_VER": "3.8.13"
   }
 }

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: pip
           cache-dependency-path: pyproject.toml
       - name: "Install dependencies"
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         os:
           - macos-latest
           - ubuntu-latest
@@ -50,7 +50,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: pip
           cache-dependency-path: pyproject.toml
       - uses: actions/setup-java@v3

.pylintrc

@@ -531,5 +531,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

.readthedocs.yml → .readthedocs.yaml

@@ -1,7 +1,11 @@
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
 python:
-  version: "3.8"
   install:
     - method: pip
       path: .
@@ -10,3 +14,4 @@ python:
 
 sphinx:
   fail_on_warning: true
+  configuration: docs/conf.py

Makefile

@@ -24,6 +24,7 @@ PIP_WRAPPER := $(PY_BIN) -m pip
 export PY38 := $(shell jq -r '.python_versions.PY38' .ci/variables.json)
 export PY39 := $(shell jq -r '.python_versions.PY39' .ci/variables.json)
 export PY310 := $(shell jq -r '.python_versions.PY310' .ci/variables.json)
+export PY311 := $(shell jq -r '.python_versions.PY311' .ci/variables.json)
 export HATCH_VERSION := $(shell jq -r '.prerequisite_versions.HATCH' .ci/variables.json)
 export HATCHLING_VERSION := $(shell jq -r '.prerequisite_versions.HATCHLING' .ci/variables.json)
 export PIP_VERSION := $(shell jq -r '.prerequisite_versions.PIP' .ci/variables.json)
@@ -40,6 +41,7 @@ prereq:
 	pyenv install --skip-existing $(PY38)
 	pyenv install --skip-existing $(PY39)
 	pyenv install --skip-existing $(PY310)
+	pyenv install --skip-existing $(PY311)
 	pyenv local $(PY38)
 	@# Ensure all Python versions are registered for this project
 	@ jq -r '.python_versions | [.[] | tostring] | join("\n")' .ci/variables.json > .python-version
@@ -100,7 +102,7 @@ test: check-venv
 # checks min and max python versions
 it: check-venv python-caches-clean
 	. $(VENV_ACTIVATE_FILE); nox -s test-3.8
-	. $(VENV_ACTIVATE_FILE); nox -s test-3.10
+	. $(VENV_ACTIVATE_FILE); nox -s test-3.11
 
 check-all: lint test it
 

esrally/driver/driver.py

@@ -1713,16 +1713,13 @@ def __init__(self, cfg, track, task_allocations, sampler, cancel, complete, abor
         self.logger = logging.getLogger(__name__)
 
     def __call__(self, *args, **kwargs):
-        # only possible in Python 3.7+ (has introduced get_running_loop)
-        # try:
-        #     loop = asyncio.get_running_loop()
-        # except RuntimeError:
-        #     loop = asyncio.new_event_loop()
-        #     asyncio.set_event_loop(loop)
-        loop = asyncio.new_event_loop()
-        loop.set_debug(self.debug_event_loop)
-        loop.set_exception_handler(self._logging_exception_handler)
-        asyncio.set_event_loop(loop)
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            loop.set_debug(self.debug_event_loop)
+            loop.set_exception_handler(self._logging_exception_handler)
+            asyncio.set_event_loop(loop)
         try:
             loop.run_until_complete(self.run())
         finally:

esrally/mechanic/supplier.py

@@ -701,7 +701,7 @@ def _try_init(self, may_skip_init=False):
             elif os.path.isdir(self.src_dir) and may_skip_init:
                 self.logger.info("Skipping repository initialization for %s.", self.name)
             else:
-                exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name)
+                raise exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name)
 
     def _update(self, revision):
         if self.has_remote() and revision == "latest":

esrally/utils/net.py

@@ -262,16 +262,16 @@ def download(url, local_path, expected_size_in_bytes=None, progress_indicator=No
         if os.path.isfile(tmp_data_set_path):
             os.remove(tmp_data_set_path)
         raise
-    else:
-        download_size = os.path.getsize(tmp_data_set_path)
-        if expected_size_in_bytes is not None and download_size != expected_size_in_bytes:
-            if os.path.isfile(tmp_data_set_path):
-                os.remove(tmp_data_set_path)
-            raise exceptions.DataError(
-                "Download of [%s] is corrupt. Downloaded [%d] bytes but [%d] bytes are expected. Please retry."
-                % (local_path, download_size, expected_size_in_bytes)
-            )
-        os.rename(tmp_data_set_path, local_path)
+
+    download_size = os.path.getsize(tmp_data_set_path)
+    if expected_size_in_bytes is not None and download_size != expected_size_in_bytes:
+        if os.path.isfile(tmp_data_set_path):
+            os.remove(tmp_data_set_path)
+        raise exceptions.DataError(
+            "Download of [%s] is corrupt. Downloaded [%d] bytes but [%d] bytes are expected. Please retry."
+            % (local_path, download_size, expected_size_in_bytes)
+        )
+    os.rename(tmp_data_set_path, local_path)
 
 
 def retrieve_content_as_string(url):

noxfile.py

@@ -1,13 +1,13 @@
 import nox
 
 
-@nox.session(python=["3.8", "3.9", "3.10"])
+@nox.session(python=["3.8", "3.9", "3.10", "3.11"])
 def test(session: nox.Session) -> None:
     session.install(".[develop]")
     session.run("pytest")
 
 
-@nox.session(python=["3.8", "3.9", "3.10"])
+@nox.session(python=["3.8", "3.9", "3.10", "3.11"])
 def it(session: nox.Session) -> None:
     session.install(".[develop]")
     session.run("pytest", "-s", "it")

pyproject.toml

@@ -28,6 +28,7 @@ classifiers=[
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3 :: Only",
 ]
 ################################################################################################
@@ -48,7 +49,7 @@ dependencies = [
     "urllib3==1.26.9",
     "docker==6.0.0",
     # License: BSD
-    "psutil==5.8.0",
+    "psutil==5.9.4",
     # License: MIT
     "py-cpuinfo==7.0.0",
     # License: MIT
@@ -67,7 +68,7 @@ dependencies = [
     # License: MPL 2.0
     "certifi",
     # License: MIT
-    "yappi==1.3.3",
+    "yappi==1.4.0",
     # License: BSD
     "ijson==2.6.1",
     # License: Apache 2.0
@@ -105,7 +106,7 @@ develop = [
     "wheel==0.37.1",
     "github3.py==3.2.0",
     "pre-commit==2.20.0",
-    "pylint==2.14.5",
+    "pylint==2.17.0",
     "trustme==0.9.0",
     "GitPython==3.1.30",
 ]

tests/driver/driver_test.py

@@ -236,8 +236,6 @@ def test_client_reaches_join_point_which_completes_parent(self):
         assert d.current_step == 0
         assert len(d.workers_completed_current_step) == 0
 
-        # this requires at least Python 3.6
-        # target.on_task_finished.assert_called_once()
         assert target.on_task_finished.call_count == 1
         assert target.drive_at.call_count == 4
 

tests/track/loader_test.py

@@ -896,9 +896,8 @@ def dummy_read_glob(c):
         base_path = "~/.rally/benchmarks/tracks/default/geonames"
         template_file_name = "track.json"
         tmpl_src = loader.TemplateSource(base_path, template_file_name)
-        # pylint: disable=trailing-whitespace
         expected_response = textwrap.dedent(
-            """                                
+            """
             {% import "rally.helpers" as rally with context %}
             {
               "version": 2,