From 4ccd7943d17bcb2d5946261b581a34af456b1c64 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Tue, 3 Sep 2024 08:28:48 -0700 Subject: [PATCH] bug-1912360: shade elasticsearch libs to make way for new versions (#6687) --- .devcontainer/Dockerfile | 14 +++++++++-- Makefile | 2 ++ bin/es_cli.py | 4 ++-- bin/verify_reqs.sh | 4 ++++ docker/Dockerfile | 10 ++++++-- docker/set_up_legacy_es_requirements.sh | 24 +++++++++++++++++++ legacy-es-requirements.in | 4 ++++ legacy-es-requirements.txt | 23 ++++++++++++++++++ requirements.in | 12 ++++++++-- requirements.txt | 16 +++---------- socorro/external/es/connection_context.py | 2 +- socorro/external/es/crashstorage.py | 6 ++--- socorro/external/es/query.py | 2 +- socorro/external/es/supersearch.py | 4 ++-- socorro/tests/conftest.py | 2 +- .../crashstats/supersearch/libsupersearch.py | 4 ++-- 16 files changed, 102 insertions(+), 31 deletions(-) create mode 100755 docker/set_up_legacy_es_requirements.sh create mode 100644 legacy-es-requirements.in create mode 100644 legacy-es-requirements.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8ac6e25d58..4627bd42cf 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -31,10 +31,20 @@ RUN /tmp/set_up_stackwalker.sh && \ COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/ RUN cd /webapp-frontend-deps/ && npm install +# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the +# new location (aka shaded), so that they can be installed at the same time as newer versions +COPY --chown=app:app legacy-es-requirements.txt /app/ +RUN pip install -U 'pip==24.0' && \ + pip install --no-cache-dir --no-deps -r legacy-es-requirements.txt && \ + cd /usr/local/lib/python3.11/site-packages/ && \ + mv elasticsearch elasticsearch_1_9_0 && \ + mv elasticsearch_dsl elasticsearch_dsl_0_0_11 && \ + cd elasticsearch_dsl_0_0_11/ && \ + sed 's/^from elasticsearch/from elasticsearch_1_9_0/' -i connections.py search.py serializer.py + # Install Python dependencies COPY requirements.txt /app/ -RUN pip install -U 'pip>=20' && \ - pip install --no-cache-dir --no-deps -r requirements.txt && \ +RUN pip install --no-cache-dir --no-deps -r requirements.txt && \ pip install --no-cache-dir ipython && \ pip check --disable-pip-version-check diff --git a/Makefile b/Makefile index 664f32c61c..c8316388e2 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,8 @@ testshell: my.env .docker-build ## | Open a shell in the test environment. .PHONY: rebuildreqs rebuildreqs: .env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. ${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras + ${DC} run --rm --no-deps app shell pip-compile --generate-hashes \ + --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in .PHONY: updatereqs updatereqs: .env .docker-build ## | Update deps in requirements.txt file. diff --git a/bin/es_cli.py b/bin/es_cli.py index cb6db22111..55ba5a2986 100755 --- a/bin/es_cli.py +++ b/bin/es_cli.py @@ -13,8 +13,8 @@ import json import click -from elasticsearch_dsl import Search -from elasticsearch.client import IndicesClient +from elasticsearch_dsl_0_0_11 import Search +from elasticsearch_1_9_0.client import IndicesClient from socorro import settings from socorro.external.es.base import generate_list_of_indexes diff --git a/bin/verify_reqs.sh b/bin/verify_reqs.sh index 9231d985d4..9444a83407 100755 --- a/bin/verify_reqs.sh +++ b/bin/verify_reqs.sh @@ -14,3 +14,7 @@ set -euo pipefail cp requirements.txt requirements.txt.orig pip-compile --quiet --generate-hashes --strip-extras diff requirements.txt requirements.txt.orig + +cp legacy-es-requirements.txt legacy-es-requirements.txt.orig +pip-compile --quiet --generate-hashes --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in +diff legacy-es-requirements.txt legacy-es-requirements.txt.orig diff --git a/docker/Dockerfile b/docker/Dockerfile index eb2c1b4bc8..3fa2fd2c6e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,9 +30,15 @@ RUN /tmp/set_up_stackwalker.sh && \ COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/ RUN cd /webapp-frontend-deps/ && npm install +# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the +# new location (aka shaded), so that they can be installed at the same time as newer versions +COPY --chown=app:app legacy-es-requirements.txt /app/ +COPY docker/set_up_legacy_es_requirements.sh /tmp/set_up_legacy_es_requirements.sh +RUN /tmp/set_up_legacy_es_requirements.sh && \ + rm /tmp/set_up_legacy_es_requirements.sh + COPY --chown=app:app requirements.txt /app/ -RUN pip install -U 'pip==24.0' && \ - pip install --no-cache-dir --no-deps -r requirements.txt && \ +RUN pip install --no-cache-dir --no-deps -r requirements.txt && \ pip check --disable-pip-version-check ENV PYTHONUNBUFFERED=1 \ diff --git a/docker/set_up_legacy_es_requirements.sh b/docker/set_up_legacy_es_requirements.sh new file mode 100755 index 0000000000..b4db3ed5fd --- /dev/null +++ b/docker/set_up_legacy_es_requirements.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Usage: docker/set_up_legacy_es_requirements.sh +# +# Installs legacy elasticsearch requirements to locations shaded by version. +# +# These dependencies are installed separately, relocated, and patched to reference the +# new location (aka shaded), so that they can be installed at the same time as newer versions + +# install shaded packages without dependencies +pip install --no-cache-dir --no-deps -r legacy-es-requirements.txt + +# move packages to shaded locations +cd /usr/local/lib/python3.11/site-packages/ +mv elasticsearch elasticsearch_1_9_0 +mv elasticsearch_dsl elasticsearch_dsl_0_0_11 + +# patch shaded elasticsearch-dsl to use shaded elasticearch +cd elasticsearch_dsl_0_0_11/ +sed 's/^from elasticsearch/from elasticsearch_1_9_0/' -i connections.py search.py serializer.py diff --git a/legacy-es-requirements.in b/legacy-es-requirements.in new file mode 100644 index 0000000000..6eb3379235 --- /dev/null +++ b/legacy-es-requirements.in @@ -0,0 +1,4 @@ +# NOTE(willkg): Need to keep elasticsearch and elasticsearch-dsl at these versions +# because they work with the cluster we're using +elasticsearch==1.9.0 +elasticsearch-dsl==0.0.11 diff --git a/legacy-es-requirements.txt b/legacy-es-requirements.txt new file mode 100644 index 0000000000..afc6e100e9 --- /dev/null +++ b/legacy-es-requirements.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in +# +elasticsearch==1.9.0 \ + --hash=sha256:cbee23c29f9947b1b78be5c0f741c1f8b504d3c53163b1355ea08f69cbd0fc0c \ + --hash=sha256:eb0d4b5b20a4ccf9e49bd20498b6215d5e960d8a63a0eae44cd4431e3f303a24 + # via + # -r legacy-es-requirements.in + # elasticsearch-dsl +elasticsearch-dsl==0.0.11 \ + --hash=sha256:59a76c4142478a1952bba6f9a9ca4fc7b029afb619e8ffcf0d135ce37ea692da \ + --hash=sha256:663fb62ad39200c7d903e973aa0aa693578613264d83796455cbf4cd172bd878 + # via -r legacy-es-requirements.in + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes and the requirement is not +# satisfied by a package already installed. Consider using the --allow-unsafe flag. +# python-dateutil +# six +# urllib3 diff --git a/requirements.in b/requirements.in index 021b6c7590..677555d35f 100644 --- a/requirements.in +++ b/requirements.in @@ -66,5 +66,13 @@ django==4.2.15 # NOTE(willkg): Need to keep elasticsearch and elasticsearch-dsl at these versions # because they work with the cluster we're using -elasticsearch==1.9.0 -elasticsearch-dsl==0.0.11 +# via docker/set_up_legacy_es_requirements.sh and legacy-es-requirements.txt +# NOTE(relud): these dependencies are installed separately via legacy-es-requirements.txt and +# docker/set_up_legacy_es_requirements.sh, relocated, and patched to reference the new location (aka +# shaded), so that they can be installed at the same time as newer versions +# elasticsearch==1.9.0 +# elasticsearch-dsl==0.0.11 +# NOTE(relud): specify transitive dependencies that are not shaded +python-dateutil # via elasticsearch-dsl==0.0.11 +six # via elasticsearch-dsl==0.0.11 +urllib3>=1.8, <2.0 # via elasticsearch==1.9.0 diff --git a/requirements.txt b/requirements.txt index 36006845d4..156b106fbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -298,16 +298,6 @@ docutils==0.20.1 \ # via # sphinx # sphinx-rtd-theme -elasticsearch==1.9.0 \ - --hash=sha256:cbee23c29f9947b1b78be5c0f741c1f8b504d3c53163b1355ea08f69cbd0fc0c \ - --hash=sha256:eb0d4b5b20a4ccf9e49bd20498b6215d5e960d8a63a0eae44cd4431e3f303a24 - # via - # -r requirements.in - # elasticsearch-dsl -elasticsearch-dsl==0.0.11 \ - --hash=sha256:59a76c4142478a1952bba6f9a9ca4fc7b029afb619e8ffcf0d135ce37ea692da \ - --hash=sha256:663fb62ad39200c7d903e973aa0aa693578613264d83796455cbf4cd172bd878 - # via -r requirements.in enforce-typing==1.0.0.post1 \ --hash=sha256:90347a61d08e7f7578d9714b4f0fd8abd9b6bc48c8ac8d46d7f290d413afabb7 \ --hash=sha256:d3184dfdbfd7f9520c884986561751a6106c57cdd65d730470645d2d40c47e18 @@ -1002,7 +992,7 @@ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # elasticsearch-dsl + # -r requirements.in # freezegun python-decouple==3.8 \ --hash=sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f \ @@ -1224,7 +1214,7 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # elasticsearch-dsl + # -r requirements.in # isodate # oauth2client # python-dateutil @@ -1292,7 +1282,7 @@ urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # elasticsearch + # -r requirements.in # requests # sentry-sdk urlwait==1.0 \ diff --git a/socorro/external/es/connection_context.py b/socorro/external/es/connection_context.py index 8b69c57e7f..aed29789dd 100644 --- a/socorro/external/es/connection_context.py +++ b/socorro/external/es/connection_context.py @@ -4,7 +4,7 @@ import contextlib -import elasticsearch +import elasticsearch_1_9_0 as elasticsearch class ConnectionContext: diff --git a/socorro/external/es/crashstorage.py b/socorro/external/es/crashstorage.py index bae04eeed2..87af8a5f7a 100644 --- a/socorro/external/es/crashstorage.py +++ b/socorro/external/es/crashstorage.py @@ -8,9 +8,9 @@ import re import time -import elasticsearch -from elasticsearch.exceptions import NotFoundError -from elasticsearch_dsl import Search +import elasticsearch_1_9_0 as elasticsearch +from elasticsearch_1_9_0.exceptions import NotFoundError +from elasticsearch_dsl_0_0_11 import Search import glom import markus diff --git a/socorro/external/es/query.py b/socorro/external/es/query.py index 8c86b085e5..c01eeb0cb8 100644 --- a/socorro/external/es/query.py +++ b/socorro/external/es/query.py @@ -3,7 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import datetime -import elasticsearch +import elasticsearch_1_9_0 as elasticsearch import json import re diff --git a/socorro/external/es/supersearch.py b/socorro/external/es/supersearch.py index f59056db16..a57e6218e9 100644 --- a/socorro/external/es/supersearch.py +++ b/socorro/external/es/supersearch.py @@ -7,8 +7,8 @@ import datetime import re -from elasticsearch.exceptions import NotFoundError, RequestError -from elasticsearch_dsl import A, F, Q, Search +from elasticsearch_1_9_0.exceptions import NotFoundError, RequestError +from elasticsearch_dsl_0_0_11 import A, F, Q, Search from socorro.external.es.base import generate_list_of_indexes from socorro.external.es.super_search_fields import get_search_key diff --git a/socorro/tests/conftest.py b/socorro/tests/conftest.py index 515ab7adeb..1ea2d40cdc 100644 --- a/socorro/tests/conftest.py +++ b/socorro/tests/conftest.py @@ -13,7 +13,7 @@ import uuid import sys -from elasticsearch_dsl import Search +from elasticsearch_dsl_0_0_11 import Search from google.api_core.exceptions import AlreadyExists, NotFound from google.auth.credentials import AnonymousCredentials from google.cloud import storage diff --git a/webapp/crashstats/supersearch/libsupersearch.py b/webapp/crashstats/supersearch/libsupersearch.py index 955ae6c066..9d09a26881 100644 --- a/webapp/crashstats/supersearch/libsupersearch.py +++ b/webapp/crashstats/supersearch/libsupersearch.py @@ -5,8 +5,8 @@ from dataclasses import dataclass import datetime -import elasticsearch -from elasticsearch_dsl import Search +import elasticsearch_1_9_0 as elasticsearch +from elasticsearch_dsl_0_0_11 import Search from socorro import settings as socorro_settings from socorro.external.es.super_search_fields import FIELDS