Bump Python 3 to 3.12 (#18212)

* bump py version to 3.12 * update dbm dev container * update builders and build deps * fix python string * Build lxml dependencies for linux x64 (#18467) * fix rmq version * fix gunicorn tests * Fix lock file generation * comment out if clause to force dep resolution * bump python to 3.12.6 * fix python on Windows sha * remove DD_PYTHON2 unbound variable * fix hatch errors * remove setup.py files * fix CI * revert datadog_checks_test_helper --------- Co-authored-by: Alex Lopez <[email protected]> Co-authored-by: Ilia Kurenkov <[email protected]> Co-authored-by: Alex Lopez <[email protected]>
DataDog · Sep 17, 2024 · 69cffd9 · 69cffd9
1 parent fcfc8f1
commit 69cffd9
Show file tree

Hide file tree

Showing 602 changed files with 696 additions and 459 deletions.
diff --git a/.builders/images/linux-aarch64/Dockerfile b/.builders/images/linux-aarch64/Dockerfile
@@ -53,11 +53,11 @@ RUN yum install -y perl-IPC-Cmd && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.11.8
+ENV PYTHON3_VERSION=3.12.6
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="d3019a613b9e8761d260d9ebe3bd4df63976de30464e5c0189566e1ae3f61889" \
+ SHA256="85a4c1be906d20e5c5a69f2466b00da769c221d6a684acfd3a514dbf5bf10a66" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh \
  --prefix=/opt/python/${PYTHON_VERSION} \

diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile
@@ -51,11 +51,11 @@ RUN yum install -y perl-IPC-Cmd && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.11.8
+ENV PYTHON3_VERSION=3.12.6
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="d3019a613b9e8761d260d9ebe3bd4df63976de30464e5c0189566e1ae3f61889" \
+ SHA256="85a4c1be906d20e5c5a69f2466b00da769c221d6a684acfd3a514dbf5bf10a66" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh --prefix=/opt/python/${PYTHON_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
 ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}"
@@ -81,6 +81,36 @@ RUN \
  RELATIVE_PATH="krb5-{{version}}/src" \
  bash install-from-source.sh --without-keyutils --without-system-verto --without-libedit --disable-static
 
+# libxml & libxslt for lxml
+RUN \
+ DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.12/libxml2-{{version}}.tar.xz" \
+ VERSION="2.12.6" \
+ SHA256="889c593a881a3db5fdd96cc9318c87df34eb648edfc458272ad46fd607353fbb" \
+ RELATIVE_PATH="libxml2-{{version}}" \
+ bash install-from-source.sh \
+ --without-iconv \
+ --without-python \
+ --without-icu \
+ --without-debug \
+ --without-mem-debug \
+ --without-run-debug \
+ --without-legacy \
+ --without-catalog \
+ --without-docbook \
+ --disable-static
+
+RUN \
+ DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+ VERSION="1.1.39" \
+ SHA256="2a20ad621148339b0759c4d4e96719362dee64c9a096dbba625ba053846349f0" \
+ RELATIVE_PATH="libxslt-{{version}}" \
+ bash install-from-source.sh \
+ --without-python \
+ --without-crypto \
+ --without-profiler \
+ --without-debugger \
+ --disable-static
+
 # libpq and pg_config as needed by psycopg2
 RUN \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \

diff --git a/.builders/images/windows-x86_64/Dockerfile b/.builders/images/windows-x86_64/Dockerfile
@@ -73,17 +73,17 @@ RUN Get-RemoteFile `
     Approve-File -Path $($Env:USERPROFILE + '\.cargo\bin\rustc.exe') -Hash $Env:RUSTC_HASH
 
 # Install Python 3
-ENV PYTHON_VERSION="3.11.7"
+ENV PYTHON_VERSION="3.12.6"
 RUN Get-RemoteFile `
       -Uri https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe `
       -Path python-$Env:PYTHON_VERSION-amd64.exe `
-      -Hash 'c117c6444494bbe4cc937e8a5a61899d53f7f5c5bc573c5d130304e457d54024'; `
+      -Hash '5914748e6580e70bedeb7c537a0832b3071de9e09a2e4e7e3d28060616045e0a'; `
     Start-Process -Wait python-$Env:PYTHON_VERSION-amd64.exe -ArgumentList '/quiet', 'InstallAllUsers=1'; `
     Remove-Item python-$Env:PYTHON_VERSION-amd64.exe; `
-    & 'C:\Program Files\Python311\python.exe' -m pip install --no-warn-script-location --upgrade pip; `
-    & 'C:\Program Files\Python311\python.exe' -m pip install --no-warn-script-location virtualenv; `
-    & 'C:\Program Files\Python311\python.exe' -m virtualenv 'C:\py3'; `
-    Add-ToPath -Append 'C:\Program Files\Python311'
+    & 'C:\Program Files\Python312\python.exe' -m pip install --no-warn-script-location --upgrade pip; `
+    & 'C:\Program Files\Python312\python.exe' -m pip install --no-warn-script-location virtualenv; `
+    & 'C:\Program Files\Python312\python.exe' -m virtualenv 'C:\py3'; `
+    Add-ToPath -Append 'C:\Program Files\Python312'
 
 # Install Python 2
 ENV PYTHON_VERSION="2.7.18"

diff --git a/.builders/lock.py b/.builders/lock.py
@@ -36,6 +36,11 @@ def default_python_version() -> str:
     return match.group(1)
 
 
+@cache
+def target_python_for_major(python_major: str):
+    return '2.7' if python_major == '2' else default_python_version()
+
+
 def is_compatible_wheel(
     target_name: str,
     target_python_major: str,
@@ -44,7 +49,7 @@ def is_compatible_wheel(
     platform: str,
 ) -> bool:
     if interpreter.startswith('cp'):
-        target_python = '2.7' if target_python_major == '2' else default_python_version()
+        target_python = target_python_for_major(target_python_major)
         expected_tag = f'cp{target_python_major}' if abi == 'abi3' else f'cp{target_python}'.replace('.', '')
         if expected_tag not in interpreter:
             return False
@@ -59,8 +64,17 @@ def is_compatible_wheel(
     return True
 
 
-def generate_lock_file(requirements_file: Path, lock_file: Path) -> None:
-    target, _, python_version = lock_file.stem.rpartition('_')
+def generate_lock_file(
+    requirements_file: Path,
+    lock_file_folder: Path,
+    target: str,
+    python_version: str,
+) -> None:
+    python_target = target_python_for_major(python_version)
+    # The lockfiles contain the major.minor Python version
+    # so that the Agent can transition safely
+    lock_file = lock_file_folder / f'{target}_{python_target}.txt'
+
     python_major = python_version[-1]
 
     dependencies: dict[str, str] = {}
@@ -135,7 +149,10 @@ def main():
         for python_version in target.iterdir():
             if python_version.name.startswith('py'):
                 generate_lock_file(
-                    python_version / 'frozen.txt', LOCK_FILE_DIR / f'{target.name}_{python_version.name}.txt'
+                    python_version / 'frozen.txt',
+                    LOCK_FILE_DIR,
+                    target.name,
+                    python_version.name.strip('py'),
                 )
 
         if (image_digest_file := target / 'image_digest').is_file():

diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,6 +1,6 @@
 # This dockerfile is used to build the devcontainer environment.
 # more info about vscode devcontainer: https://code.visualstudio.com/docs/devcontainers/containers
-FROM mcr.microsoft.com/devcontainers/python:1-3.11-bullseye
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y liblz4-dev libunwind-dev ca-certificates curl gnupg
 # Docker and docker-compose installation
 RUN install -m 0755 -d /etc/apt/keyrings

diff --git a/.devcontainer/dbm/Dockerfile b/.devcontainer/dbm/Dockerfile
@@ -1,6 +1,6 @@
 # This dockerfile is used to build the devcontainer environment.
 # more info about vscode devcontainer: https://code.visualstudio.com/docs/devcontainers/containers
-FROM mcr.microsoft.com/devcontainers/python:1-3.11-bullseye
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y liblz4-dev libunwind-dev ca-certificates curl gnupg
 # Docker and docker-compose installation
 RUN install -m 0755 -d /etc/apt/keyrings

diff --git a/.devcontainer/dbm/devcontainer.json b/.devcontainer/dbm/devcontainer.json
@@ -8,7 +8,7 @@
 	"privileged": true,
 	"features": {
 		"ghcr.io/devcontainers/features/python:1": {
-			"version": "3.11"
+			"version": "3.12"
 		}
 	},
 	"customizations": {

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -16,7 +16,7 @@
 		// When updating it, modify both the base Dockerfile and the devcontainer.json reference.
 		// Ref: https://github.com/devcontainers/features/blob/562305d37b97d47331d96306ffc2a0a3cce55e64/src/python/install.sh#L10
 		"ghcr.io/devcontainers/features/python:1": {
-			"version": "3.11"
+			"version": "3.12"
 		}
 	},
 

diff --git a/.github/workflows/build-ddev.yml b/.github/workflows/build-ddev.yml
@@ -24,7 +24,7 @@ defaults:
 
 env:
   APP_NAME: ddev
-  PYTHON_VERSION: "3.11"
+  PYTHON_VERSION: "3.12"
   PYOXIDIZER_VERSION: "0.24.0"
 
 jobs:

diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml
@@ -29,7 +29,7 @@ defaults:
 
 env:
   PYTHONUNBUFFERED: "1"
-  PYTHON_VERSION: "3.11"
+  PYTHON_VERSION: "3.12"
   DIRECT_DEPENDENCY_FILE: agent_requirements.in
   # https://reproducible-builds.org/specs/source-date-epoch/
   SOURCE_DATE_EPOCH: "1580601600"
@@ -109,7 +109,7 @@ jobs:
       if: matrix.job.image == 'linux-aarch64'
       run: |
         mkdir -p ~/miniconda3
-        wget https://repo.anaconda.com/miniconda/Miniconda3-py311_24.1.2-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
+        wget https://repo.anaconda.com/miniconda/Miniconda3-py312_24.5.0-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
         bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
         rm -rf ~/miniconda3/miniconda.sh
         ~/miniconda3/bin/conda init bash
@@ -201,7 +201,7 @@ jobs:
     env:
       TARGET_NAME: macos-x86_64
       OUT_DIR: output/macos-x86_64
-      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python"
+      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.12/bin/python"
 
     steps:
     - name: Set up environment
@@ -214,7 +214,7 @@ jobs:
     - name: Set up Python
       env:
         # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
-        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
+        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.12.6/python-3.12.6-macos11.pkg"
       run: |-
         curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
         sudo installer -pkg python3.pkg -target /

diff --git a/.github/workflows/cache-shared-deps.yml b/.github/workflows/cache-shared-deps.yml
@@ -16,7 +16,7 @@ jobs:
         os: [ubuntu-22.04, windows-2022]
 
     env:
-      PYTHON_VERSION: "3.11"
+      PYTHON_VERSION: "3.12"
 
     steps:
     - uses: actions/checkout@v4

diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
@@ -15,7 +15,7 @@ defaults:
     shell: bash
 
 env:
-  PYTHON_VERSION: "3.11"
+  PYTHON_VERSION: "3.12"
   MATRIX_SCRIPT: "ddev/src/ddev/utils/scripts/ci_matrix.py"
 
 jobs:

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,7 +32,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
         cache: 'pip'
 
     - name: Upgrade Python packaging tools

diff --git a/.github/workflows/pr-quick-check.yml b/.github/workflows/pr-quick-check.yml
@@ -12,7 +12,7 @@ defaults:
     shell: bash
 
 env:
-  PYTHON_VERSION: "3.11"
+  PYTHON_VERSION: "3.12"
   CHECK_SCRIPT: "ddev/src/ddev/utils/scripts/check_pr.py"
 
 jobs:

diff --git a/.github/workflows/release-base.yml b/.github/workflows/release-base.yml
@@ -15,7 +15,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
 
     - name: Upgrade Python packaging tools
       run: pip install --disable-pip-version-check --upgrade pip setuptools wheel

diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml
@@ -15,7 +15,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
 
     - name: Upgrade Python packaging tools
       run: pip install --disable-pip-version-check --upgrade pip setuptools wheel

diff --git a/.github/workflows/release-hash-check.yml b/.github/workflows/release-hash-check.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
 
     - id: files
       run: |

diff --git a/.github/workflows/run-validations.yml b/.github/workflows/run-validations.yml
@@ -117,7 +117,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     env:
-      PYTHON_VERSION: "3.11"
+      PYTHON_VERSION: "3.12"
       TARGET: ${{ github.event_name == 'pull_request' && 'changed' || '' }}
 
     steps:

diff --git a/.github/workflows/test-target.yml b/.github/workflows/test-target.yml
@@ -94,8 +94,8 @@ jobs:
 
     env:
       FORCE_COLOR: "1"
-      PYTHON_VERSION: "${{ inputs.python-version || '3.11' }}"
-      PYTHON_FILTER: "${{ (inputs.test-py2 && !inputs.test-py3) && '2.7' || (!inputs.test-py2 && inputs.test-py3) && (inputs.python-version || '3.11') || '' }}"
+      PYTHON_VERSION: "${{ inputs.python-version || '3.12' }}"
+      PYTHON_FILTER: "${{ (inputs.test-py2 && !inputs.test-py3) && '2.7' || (!inputs.test-py2 && inputs.test-py3) && (inputs.python-version || '3.12') || '' }}"
       SKIP_ENV_NAME: "${{ (inputs.test-py2 && !inputs.test-py3) && 'py3.*' || (!inputs.test-py2 && inputs.test-py3) && 'py2.*' || '' }}"
       # Windows E2E requires Windows containers
       DDEV_E2E_AGENT: "${{ inputs.platform == 'windows' && (inputs.agent-image-windows || 'datadog/agent-dev:master-py3-win-servercore') || inputs.agent-image }}"

diff --git a/.github/workflows/update-agent-changelog.yml b/.github/workflows/update-agent-changelog.yml
@@ -18,7 +18,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: 3.12
     - name: Install ddev from local folder
       run: |-
         pip install -e ./datadog_checks_dev[cli]

diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
@@ -18,7 +18,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: 3.12
     - name: Install ddev from local folder
       run: |-
         pip install -e ./datadog_checks_dev[cli]

diff --git a/.gitlab/tagger/Dockerfile b/.gitlab/tagger/Dockerfile
@@ -1,5 +1,5 @@
 # Use a separate image to build ddev from source so the final image doesn't need git
-FROM python:3.11 AS ddev-python
+FROM python:3.12 AS ddev-python
 COPY . /home
 RUN python -m pip install build
 RUN python -m build -s ./home/ddev

diff --git a/active_directory/changelog.d/18207.added b/active_directory/changelog.d/18207.added
@@ -0,0 +1 @@
+Bump the python version from 3.11 to 3.12
diff --git a/active_directory/hatch.toml b/active_directory/hatch.toml
@@ -10,4 +10,4 @@ platforms = [
 ]
 
 [[envs.default.matrix]]
-python = ["3.11"]
+python = ["3.12"]
diff --git a/active_directory/pyproject.toml b/active_directory/pyproject.toml
@@ -25,7 +25,7 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
     "License :: OSI Approved :: BSD License",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
     "Private :: Do Not Upload",
 ]

diff --git a/activemq/changelog.d/18207.added b/activemq/changelog.d/18207.added
@@ -0,0 +1 @@
+Bump the python version from 3.11 to 3.12
diff --git a/activemq/hatch.toml b/activemq/hatch.toml
@@ -1,12 +1,12 @@
 [env.collectors.datadog-checks]
 
 [[envs.default.matrix]]
-python = ["3.11"]
+python = ["3.12"]
 version = ["2.15.0"]
 compose-file = ["artemis.yaml"]
 
 [[envs.default.matrix]]
-python = ["3.11"]
+python = ["3.12"]
 version = ["5.15.9"]
 compose-file = ["activemq.yaml"]
 

diff --git a/activemq/pyproject.toml b/activemq/pyproject.toml
@@ -25,7 +25,7 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
     "License :: OSI Approved :: BSD License",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
     "Private :: Do Not Upload",
 ]

diff --git a/activemq_xml/changelog.d/18207.added b/activemq_xml/changelog.d/18207.added
@@ -0,0 +1 @@
+Bump the python version from 3.11 to 3.12
diff --git a/activemq_xml/hatch.toml b/activemq_xml/hatch.toml
@@ -1,7 +1,7 @@
 [env.collectors.datadog-checks]
 
 [[envs.default.matrix]]
-python = ["3.11"]
+python = ["3.12"]
 version = ["5.11.1"]
 
 [envs.default.overrides]

diff --git a/activemq_xml/pyproject.toml b/activemq_xml/pyproject.toml
@@ -25,7 +25,7 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
     "License :: OSI Approved :: BSD License",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
     "Private :: Do Not Upload",
 ]

diff --git a/aerospike/changelog.d/18207.added b/aerospike/changelog.d/18207.added
@@ -0,0 +1 @@
+Bump the python version from 3.11 to 3.12