bug-1921849: support elasticsearch 8 (#6741)

* support elasticsearch 8

Co-authored-by: krzepka <[email protected]>

* address review

* update error handling for dropping fields

* address review

* fix topcrashers

* address review

---------

Co-authored-by: krzepka <[email protected]>

bin/setup_services.sh

@@ -13,6 +13,9 @@
 
 set -euo pipefail
 
+# wait for dev services to start up
+./bin/waitfor_services.sh
+
 # Drop and re-create the breakpad database with tables, stored procedures,
 # types, indexes, and keys; also bulk-loads static data for some lookup tables
 /app/bin/setup_postgres.sh

bin/test.sh

@@ -27,13 +27,7 @@ PYTHON="$(which python)"
 
 echo ">>> wait for services to be ready"
 
-waitfor --verbose --conn-only "${DATABASE_URL}"
-waitfor --verbose "${LEGACY_ELASTICSEARCH_URL}"
-waitfor --verbose "http://${PUBSUB_EMULATOR_HOST}"
-waitfor --verbose "${STORAGE_EMULATOR_HOST}/storage/v1/b"
-waitfor --verbose --codes={200,404} "${SENTRY_DSN}"
-# wait for this last because it's slow to start
-waitfor --verbose --timeout=30 "${ELASTICSEARCH_URL}"
+./bin/waitfor_services.sh
 
 echo ">>> build queue things and db things"
 
@@ -53,16 +47,14 @@ popd
 echo ">>> run tests"
 
 # Run socorro tests
-ELASTICSEARCH_MODE=LEGACY_ONLY "${PYTEST}"
-# override ELASTICSEARCH_URL to use legacy elasticsearch so we can test PREFER_NEW without
-# implementing es8 support. Override will be removed when socorrro/external/es expects es8
-ELASTICSEARCH_MODE=PREFER_NEW ELASTICSEARCH_URL="${LEGACY_ELASTICSEARCH_URL}" "${PYTEST}"
+# ignore tests because elasticsearch 8 tests cannot be run with LEGACY_ONLY
+ELASTICSEARCH_MODE=LEGACY_ONLY "${PYTEST}" --ignore=socorro/tests/external/es/
+# ignore tests because elasticsearch 8 supersearch is not implemented yet
+ELASTICSEARCH_MODE=PREFER_NEW "${PYTEST}"
 
 # Collect static and then run pytest in the webapp
 pushd webapp
 ${PYTHON} manage.py collectstatic --noinput
 ELASTICSEARCH_MODE=LEGACY_ONLY "${PYTEST}"
-# override ELASTICSEARCH_URL to use legacy elasticsearch so we can test PREFER_NEW without
-# implementing es8 support. Override will be removed when socorrro/external/es expects es8
-ELASTICSEARCH_MODE=PREFER_NEW ELASTICSEARCH_URL="${LEGACY_ELASTICSEARCH_URL}" "${PYTEST}"
+ELASTICSEARCH_MODE=PREFER_NEW "${PYTEST}"
 popd

bin/waitfor_services.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Usage: bin/waitfor_services.sh
+#
+# Waits for dev services to start up.
+#
+# Note: This should be called from inside a container.
+
+set -euo pipefail
+
+waitfor --verbose --conn-only "${DATABASE_URL}"
+waitfor --verbose "${LEGACY_ELASTICSEARCH_URL}"
+waitfor --verbose "http://${PUBSUB_EMULATOR_HOST}"
+waitfor --verbose "${STORAGE_EMULATOR_HOST}/storage/v1/b"
+waitfor --verbose --codes={200,404} "${SENTRY_DSN}"
+# wait for this last because it's slow to start
+waitfor --verbose --timeout=30 "${ELASTICSEARCH_URL}"

docker-compose.yml

@@ -206,7 +206,7 @@ services:
     ports:
       - "9200:9200"
 
-  # https://www.elastic.co/guide/en/elasticsearch/reference/8.14/docker.html
+  # https://www.elastic.co/guide/en/elasticsearch/reference/8.16/docker.html
   elasticsearch:
     build:
       context: docker/images/elasticsearch

requirements.in

@@ -64,8 +64,10 @@ django-waffle==2.3.0
 # NOTE(willkg): We stick with LTS releases and the next one is 5.2 (2025).
 django==4.2.16
 
-# NOTE(willkg): Need to keep elasticsearch and elasticsearch-dsl at these versions
-# because they work with the cluster we're using
+# NOTE(relud): Need to keep elasticsearch and elasticsearch-dsl at versions compatible with the
+# cluster we're using
+elasticsearch==8.15.1
+elasticsearch-dsl==8.15.3
 # via docker/set_up_legacy_es_requirements.sh and legacy-es-requirements.txt
 # NOTE(relud): these dependencies are installed separately via legacy-es-requirements.txt and
 # docker/set_up_legacy_es_requirements.sh, relocated, and patched to reference the new location (aka

requirements.txt

@@ -48,6 +48,7 @@ certifi==2024.7.4 \
     --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
     --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
     # via
+    #   elastic-transport
     #   requests
     #   sentry-sdk
 cffi==1.16.0 \
@@ -311,6 +312,20 @@ docutils==0.20.1 \
     # via
     #   sphinx
     #   sphinx-rtd-theme
+elastic-transport==8.15.0 \
+    --hash=sha256:85d62558f9baafb0868c801233a59b235e61d7b4804c28c2fadaa866b6766233 \
+    --hash=sha256:d7080d1dada2b4eee69e7574f9c17a76b42f2895eff428e562f94b0360e158c0
+    # via elasticsearch
+elasticsearch==8.15.1 \
+    --hash=sha256:02a0476e98768a30d7926335fc0d305c04fdb928eea1354c6e6040d8c2814569 \
+    --hash=sha256:40c0d312f8adf8bdc81795bc16a0b546ddf544cb1f90e829a244e4780c4dbfd8
+    # via
+    #   -r requirements.in
+    #   elasticsearch-dsl
+elasticsearch-dsl==8.15.3 \
+    --hash=sha256:4005fa1abef5c0adc5266896c3177d0e98ff5ac49603c36dda9058d616f79257 \
+    --hash=sha256:9541a669e5ea996a447289ba3aefb3540dd9983ca0670c0f90ad4a711753422d
+    # via -r requirements.in
 enforce-typing==1.0.0.post1 \
     --hash=sha256:90347a61d08e7f7578d9714b4f0fd8abd9b6bc48c8ac8d46d7f290d413afabb7 \
     --hash=sha256:d3184dfdbfd7f9520c884986561751a6106c57cdd65d730470645d2d40c47e18
@@ -1032,6 +1047,7 @@ python-dateutil==2.9.0.post0 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
     # via
     #   -r requirements.in
+    #   elasticsearch-dsl
     #   freezegun
 python-decouple==3.8 \
     --hash=sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f \
@@ -1320,12 +1336,14 @@ typing-extensions==4.9.0 \
     --hash=sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd
     # via
     #   dj-database-url
+    #   elasticsearch-dsl
     #   opentelemetry-sdk
 urllib3==1.26.20 \
     --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \
     --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32
     # via
     #   -r requirements.in
+    #   elastic-transport
     #   requests
     #   sentry-sdk
 urlwait==1.0 \

socorro/external/es/connection_context.py

@@ -2,9 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import contextlib
+from contextlib import contextmanager
 
-import elasticsearch_1_9_0 as elasticsearch
+from elasticsearch import Elasticsearch, RequestError
 
 
 class ConnectionContext:
@@ -37,10 +37,9 @@ def connection(self, name=None, timeout=None):
         if timeout is None:
             timeout = self.timeout
 
-        return elasticsearch.Elasticsearch(
+        return Elasticsearch(
             hosts=self.url,
-            timeout=timeout,
-            connection_class=elasticsearch.connection.RequestsHttpConnection,
+            request_timeout=timeout,
             verify_certs=True,
         )
 
@@ -51,9 +50,9 @@ def indices_client(self, name=None):
         http://elasticsearch-py.readthedocs.org/en/latest/api.html#indices
 
         """
-        return elasticsearch.client.IndicesClient(self.connection())
+        return self.connection().indices
 
-    @contextlib.contextmanager
+    @contextmanager
     def __call__(self, name=None, timeout=None):
         conn = self.connection(name, timeout)
         yield conn
@@ -72,10 +71,9 @@ def create_index(self, index_name, index_settings):
             client.create(index=index_name, body=index_settings)
             return True
 
-        except elasticsearch.exceptions.RequestError as exc:
+        except RequestError as exc:
             # If this index already exists, swallow the error.
-            # NOTE! This is NOT how the error looks like in ES 2.x
-            if "IndexAlreadyExistsException" not in str(exc):
+            if "resource_already_exists_exception" not in str(exc):
                 raise
             return False
 
@@ -85,23 +83,20 @@ def get_indices(self):
         :returns: list of str
 
         """
-        indices_client = self.indices_client()
-        status = indices_client.status()
-        indices = status["indices"].keys()
-        return indices
+        return self.indices_client().get_alias().keys()
 
     def delete_index(self, index_name):
         """Delete an index."""
-        self.indices_client().delete(index_name)
+        self.indices_client().delete(index=index_name)
 
-    def get_mapping(self, index_name, doc_type):
-        """Return the mapping for the specified index and doc_type."""
+    def get_mapping(self, index_name):
+        """Return the mapping for the specified index."""
         resp = self.indices_client().get_mapping(index=index_name)
-        return resp[index_name]["mappings"][doc_type]["properties"]
+        return resp[index_name]["mappings"]["properties"]
 
     def refresh(self, index_name=None):
         self.indices_client().refresh(index=index_name or "_all")
 
     def health_check(self):
         with self() as conn:
-            conn.cluster.health(wait_for_status="yellow", request_timeout=5)
+            conn.options(request_timeout=5).cluster.health(wait_for_status="yellow")