From 976725dcb425da129f96372d2d3a9683e5a30f33 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Thu, 3 Oct 2024 10:12:17 -0700 Subject: [PATCH] bug-1921849: fix deprecated type: multi_field from super_search_fields (#6736) --- socorro/external/es/crashstorage.py | 7 -- socorro/external/es/super_search_fields.py | 68 +++++++------------ .../external/es/test_super_search_fields.py | 18 ++--- 3 files changed, 31 insertions(+), 62 deletions(-) diff --git a/socorro/external/es/crashstorage.py b/socorro/external/es/crashstorage.py index 87af8a5f7a..66695e8d03 100644 --- a/socorro/external/es/crashstorage.py +++ b/socorro/external/es/crashstorage.py @@ -236,13 +236,6 @@ def build_document(src, crash_document, fields, all_keys): # Fix values so they index correctly storage_type = field.get("type", field["storage_mapping"].get("type")) - if ( - storage_type == "multi_field" - and glom.glom(field, "storage_mapping.fields.full.type", default="") - == "string" - ): - storage_type = "string" - if storage_type == "string": analyzer = field.get("analyzer", field["storage_mapping"].get("analyzer")) if analyzer == "keyword": diff --git a/socorro/external/es/super_search_fields.py b/socorro/external/es/super_search_fields.py index c5999de886..df44a25767 100644 --- a/socorro/external/es/super_search_fields.py +++ b/socorro/external/es/super_search_fields.py @@ -206,8 +206,8 @@ def is_doc_values_friendly(storage_value): if not storage_type: return False - # object, and multi_field storages don't work with doc_values=True - if storage_type in ("object", "multi_field"): + # object storages don't work with doc_values=True + if storage_type == "object": return False # analyzed string storages don't work with doc_values=True @@ -796,10 +796,9 @@ def apply_schema_properties(fields, schema): "query_type": "string", "storage_mapping": { "fields": { - "Android_Model": {"type": "string"}, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "type": "string", }, }, "android_packagename": keyword_field(name="android_packagename"), @@ -855,14 +854,11 @@ def apply_schema_properties(fields, schema): "query_type": "string", "storage_mapping": { "fields": { - "AsyncShutdownTimeout": { - "analyzer": "standard", - "index": "analyzed", - "type": "string", - }, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "analyzer": "standard", + "index": "analyzed", + "type": "string", }, }, "available_page_file": { @@ -1038,14 +1034,11 @@ def apply_schema_properties(fields, schema): "query_type": "string", "storage_mapping": { "fields": { - "cpu_info": { - "analyzer": "standard", - "index": "analyzed", - "type": "string", - }, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "analyzer": "standard", + "index": "analyzed", + "type": "string", }, }, "cpu_microcode_version": { @@ -1120,13 +1113,10 @@ def apply_schema_properties(fields, schema): "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "dumper_error": { - "type": "string", - "index": "analyzed", - "analyzer": "standard", - }, }, - "type": "multi_field", + "type": "string", + "index": "analyzed", + "analyzer": "standard", }, }, "em_check_compatibility": { @@ -1713,14 +1703,11 @@ def apply_schema_properties(fields, schema): "name": "platform", "namespace": "processed_crash", "query_type": "enum", - # FIXME(willkg): storage_mapping should either be not_analyzed or analyzed as a - # keyword "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "os_name": {"type": "string"}, }, - "type": "multi_field", + "type": "string", }, }, "platform_pretty_version": { @@ -1767,11 +1754,10 @@ def apply_schema_properties(fields, schema): "query_type": "enum", "storage_mapping": { "fields": { - "plugin_filename": {"index": "analyzed", "type": "string"}, - "PluginFilename": {"index": "analyzed", "type": "string"}, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "index": "analyzed", + "type": "string", }, }, "plugin_name": { @@ -1786,11 +1772,10 @@ def apply_schema_properties(fields, schema): "query_type": "enum", "storage_mapping": { "fields": { - "PluginName": {"index": "analyzed", "type": "string"}, - "plugin_name": {"index": "analyzed", "type": "string"}, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "index": "analyzed", + "type": "string", }, }, "plugin_version": { @@ -1805,11 +1790,10 @@ def apply_schema_properties(fields, schema): "query_type": "enum", "storage_mapping": { "fields": { - "PluginVersion": {"index": "analyzed", "type": "string"}, - "plugin_version": {"index": "analyzed", "type": "string"}, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "index": "analyzed", + "type": "string", }, }, "possible_bit_flips_max_confidence": integer_field( @@ -1863,9 +1847,9 @@ def apply_schema_properties(fields, schema): "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "product": {"index": "analyzed", "type": "string"}, }, - "type": "multi_field", + "index": "analyzed", + "type": "string", }, }, "productid": { @@ -1925,9 +1909,9 @@ def apply_schema_properties(fields, schema): "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "reason": {"analyzer": "standard", "type": "string"}, }, - "type": "multi_field", + "analyzer": "standard", + "type": "string", }, }, "release_channel": { @@ -1993,9 +1977,8 @@ def apply_schema_properties(fields, schema): "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "signature": {"type": "string"}, }, - "type": "multi_field", + "type": "string", }, }, "stackwalk_version": keyword_field( @@ -2139,9 +2122,8 @@ def apply_schema_properties(fields, schema): "storage_mapping": { "fields": { "full": {"index": "not_analyzed", "type": "string"}, - "user_comments": {"type": "string"}, }, - "type": "multi_field", + "type": "string", }, }, "useragent_locale": { diff --git a/socorro/tests/external/es/test_super_search_fields.py b/socorro/tests/external/es/test_super_search_fields.py index f5d2581f4b..797c3f13cc 100644 --- a/socorro/tests/external/es/test_super_search_fields.py +++ b/socorro/tests/external/es/test_super_search_fields.py @@ -252,24 +252,18 @@ def test_add_doc_values(): storage_mapping = { "fields": { - "AsyncShutdownTimeout": { - "analyzer": "standard", - "index": "analyzed", - "type": "string", - }, "full": {"index": "not_analyzed", "type": "string"}, }, - "type": "multi_field", + "analyzer": "standard", + "index": "analyzed", + "type": "string", } add_doc_values(storage_mapping) assert storage_mapping == { "fields": { - "AsyncShutdownTimeout": { - "analyzer": "standard", - "index": "analyzed", - "type": "string", - }, "full": {"index": "not_analyzed", "type": "string", "doc_values": True}, }, - "type": "multi_field", + "analyzer": "standard", + "index": "analyzed", + "type": "string", }