From 3b63faa976bf408f9de9edcf7581470c92a33e40 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Fri, 29 Nov 2024 12:54:51 +1100 Subject: [PATCH 1/3] [SD-517] Upgrade data_pipelines to 2.0 --- composer.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 75d430f8..5768af01 100644 --- a/composer.json +++ b/composer.json @@ -36,10 +36,9 @@ "drupal/m4032404": "^1.0-alpha4", "drupal/maxlength": "^2.1", "drupal/entity_hierarchy": "4.x", - "drupal/elasticsearch_connector": "^7.0", + "drupal/elasticsearch_connector": "^8.0", "drupal/search_api": "^1.36", - "drupal/data_pipelines": "1.0.0-alpha22", - "drupal/data_pipelines_elasticsearch": "^1.0@alpha", + "drupal/data_pipelines": "^2.0", "drupal/key": "^1.17", "drupal/media_entity_audio": "^3.1", "drupal/metatag": "^2.0", @@ -412,11 +411,6 @@ "KeyProviderInterface::getKeyValue() doesn't always return a string - https://www.drupal.org/project/key/issues/3356052#comment-15030602": "https://git.drupalcode.org/project/key/-/merge_requests/11/diffs.patch", "Error: Call to a member function getKeyValue on null - https://www.drupal.org/project/key/issues/3385523": "https://www.drupal.org/files/issues/2023-09-06/3385523-4.patch" }, - "drupal/data_pipelines": { - "CSV data header row whitespace needs to be cleaned up - https://drupal.org/project/data_pipelines/issues/3391214": "https://www.drupal.org/files/issues/2023-10-03/trim-whitespace-from-csv-header-names-3391214-2.patch", - "set a default value based on certain criteria - https://www.drupal.org/project/data_pipelines/issues/3426248#comment-15507258": "https://www.drupal.org/files/issues/2024-03-21/data_pipelines-3426248-6.patch.patch", - "Complete the fix for bulk indexing on Elasticsearch endpoint - https://www.drupal.org/project/data_pipelines/issues/3467021#comment-15720597": "https://www.drupal.org/files/issues/2024-08-09/Change-bulk-deleting-on-Elasticsearch-endpoint-3467021.patch" - }, "drupal/paragraphs": { "Add default paragraph count setting - https://www.drupal.org/project/paragraphs/issues/3089423#comment-14517270": "https://www.drupal.org/files/issues/2022-05-17/paragraphs-default-quantity-3089423-18.patch", "Nested paragraphs title gets cut off - https://www.drupal.org/project/paragraphs/issues/3246140#comment-14698713": "https://www.drupal.org/files/issues/2022-09-20/3246140-11-fix_paragraph_title_cutoff.patch", From 7bdd2d020e0495f1681919ad5cd9014895020349 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Mon, 9 Dec 2024 11:45:19 +1100 Subject: [PATCH 2/3] remove the incompatible class --- .../Factory/TideSearchIndexFactory.php | 183 ------------------ .../src/TideSearchServiceProvider.php | 21 -- 2 files changed, 204 deletions(-) delete mode 100644 modules/tide_search/src/ElasticSearch/Parameters/Factory/TideSearchIndexFactory.php delete mode 100644 modules/tide_search/src/TideSearchServiceProvider.php diff --git a/modules/tide_search/src/ElasticSearch/Parameters/Factory/TideSearchIndexFactory.php b/modules/tide_search/src/ElasticSearch/Parameters/Factory/TideSearchIndexFactory.php deleted file mode 100644 index 95108bb2..00000000 --- a/modules/tide_search/src/ElasticSearch/Parameters/Factory/TideSearchIndexFactory.php +++ /dev/null @@ -1,183 +0,0 @@ -server->get('SEARCH_HASH') ?: '') . '-'; - $aliasName = $aliasPrefix . str_replace('_', '-', $filteredIndexName) . '-alias'; - - $indexConfig = [ - 'index' => $indexName, - 'body' => [ - 'aliases' => [ - $aliasName => [ - 'is_hidden' => FALSE, - ], - ], - 'settings' => [ - 'number_of_shards' => $index->getOption('number_of_shards', 5), - 'number_of_replicas' => $index->getOption('number_of_replicas', 1), - 'analysis' => [ - "filter" => [ - "front_ngram" => [ - "type" => "edge_ngram", - "min_gram" => "1", - "max_gram" => "12", - ], - ], - "analyzer" => [ - "i_prefix" => [ - "filter" => [ - "cjk_width", - "lowercase", - "asciifolding", - "front_ngram", - ], - "tokenizer" => "standard", - ], - "q_prefix" => [ - "filter" => [ - "cjk_width", - "lowercase", - "asciifolding", - ], - "tokenizer" => "standard", - ], - ], - ], - ], - 'mappings' => [ - "properties" => [ - "title" => [ - "type" => "text", - "fields" => [ - "keyword" => [ - "type" => "keyword", - "ignore_above" => 256, - ], - "prefix" => [ - "type" => "text", - "index_options" => "docs", - "analyzer" => "i_prefix", - "search_analyzer" => "q_prefix", - ], - ], - ], - "summary_processed" => [ - "type" => "text", - "fields" => [ - "keyword" => [ - "type" => "keyword", - "ignore_above" => 256, - ], - "prefix" => [ - "type" => "text", - "index_options" => "docs", - "analyzer" => "i_prefix", - "search_analyzer" => "q_prefix", - ], - ], - ], - ], - ], - ], - ]; - - // Allow other modules to alter index config before we create it. - $dispatcher = \Drupal::service('event_dispatcher'); - $prepareIndexEvent = new PrepareIndexEvent($indexConfig, $indexName); - $event = $dispatcher->dispatch($prepareIndexEvent, PrepareIndexEvent::PREPARE_INDEX); - $indexConfig = $event->getIndexConfig(); - return $indexConfig; - } - - /** - * Overrides the elasticsearch_connector bulk delete params. - * - * @param \Drupal\search_api\IndexInterface $index - * The Search API Index. - * @param array $ids - * The ids of the entities to delete. - * - * @return array - * The query parameters. - */ - public static function bulkDelete(IndexInterface $index, array $ids) { - $params = IndexFactory::index($index, TRUE); - - // This convoluted path to the host domain is due to - // https://www.drupal.org/project/search_api/issues/2976339 not populating - // `search_api.server.server_id` config with the correct values. - $cluster_id = Server::load($index->getServerId())->getBackend()->getCluster(); - $host = parse_url(Cluster::load($cluster_id)->url, PHP_URL_HOST); - $hash = strstr($host, '.', TRUE); - - if (isset($hash) && strlen($hash) === self::HASH_LENGTH) { - $params['index'] = $hash . '--' . $params['index']; - } - - foreach ($ids as $id) { - $params['body'][] = [ - 'delete' => [ - '_index' => $params['index'], - '_id' => $id, - ], - ]; - } - - return $params; - } - - /** - * Build parameters required to create an index mapping. - * - * @todo We need also: - * $params['index'] - (Required) - * ['type'] - The name of the document type - * ['timeout'] - (time) Explicit operation timeout. - * - * @param \Drupal\search_api\IndexInterface $index - * Index object. - * - * @return array - * Parameters required to create an index mapping. - */ - public static function mapping(IndexInterface $index) { - $mapping = parent::mapping($index); - $filtered = []; - foreach ($mapping['body']['properties'] as $property_key => $property) { - if (isset($property['boost'])) { - unset($property['boost']); - } - $filtered[$property_key] = $property; - }; - $mapping['body']['properties'] = $filtered; - - return $mapping; - } - -} diff --git a/modules/tide_search/src/TideSearchServiceProvider.php b/modules/tide_search/src/TideSearchServiceProvider.php deleted file mode 100644 index 7924273f..00000000 --- a/modules/tide_search/src/TideSearchServiceProvider.php +++ /dev/null @@ -1,21 +0,0 @@ -getDefinition('elasticsearch_connector.index_factory'); - $definition->setClass('Drupal\tide_search\ElasticSearch\Parameters\Factory\TideSearchIndexFactory'); - } - -} From 3ba089681cccb26ff59f52fc79ff04a057f7c2bd Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Thu, 12 Dec 2024 00:33:05 +1100 Subject: [PATCH 3/3] Make some necessary changes to meet the requirements of elasticsearch_connector 8.x --- .../search_api.server.elasticsearch_bay.yml | 23 +++++ .../ElasticSearchEventSubscribers.php | 99 +++++++++++++++++++ .../TextFieldWithKeywordAndPrefix.php | 21 ++++ modules/tide_search/tide_search.install | 19 ++++ modules/tide_search/tide_search.services.yml | 5 + 5 files changed, 167 insertions(+) create mode 100644 modules/tide_search/config/optional/search_api.server.elasticsearch_bay.yml create mode 100644 modules/tide_search/src/EventSubscriber/ElasticSearchEventSubscribers.php create mode 100644 modules/tide_search/src/Plugin/search_api/data_type/TextFieldWithKeywordAndPrefix.php create mode 100644 modules/tide_search/tide_search.services.yml diff --git a/modules/tide_search/config/optional/search_api.server.elasticsearch_bay.yml b/modules/tide_search/config/optional/search_api.server.elasticsearch_bay.yml new file mode 100644 index 00000000..2b8fb0c1 --- /dev/null +++ b/modules/tide_search/config/optional/search_api.server.elasticsearch_bay.yml @@ -0,0 +1,23 @@ +uuid: ff7ac093-067c-4a15-98f4-2ab4dfda7b9c +langcode: en +status: true +dependencies: + module: + - elasticsearch_connector +_core: + default_config_hash: 4ulEtCC_6q-TNQFpsg_DJsn4keGx1hpp6ZlcMtKggzo +id: elasticsearch_bay +name: 'Elasticsearch on Bay' +description: '' +backend: elasticsearch +backend_config: + connector: standard + connector_config: + url: 'http://elasticsearch:9200' + enable_debug_logging: true + advanced: + fuzziness: auto + prefix: elasticsearch_index_drupal_ + suffix: '' + synonyms: + - '' diff --git a/modules/tide_search/src/EventSubscriber/ElasticSearchEventSubscribers.php b/modules/tide_search/src/EventSubscriber/ElasticSearchEventSubscribers.php new file mode 100644 index 00000000..51cc3fac --- /dev/null +++ b/modules/tide_search/src/EventSubscriber/ElasticSearchEventSubscribers.php @@ -0,0 +1,99 @@ + 'onAlterSettingsEvent', + FieldMappingEvent::class => 'onFieldMappingEvent', + SupportsDataTypeEvent::class => 'onSupportsDataTypeEvent', + ]; + } + + /** + * Event handler for SupportsDataTypeEvent. + */ + public function onSupportsDataTypeEvent(SupportsDataTypeEvent $event): void { + if ($event->getType() === 'tide_search_text_field_with_keyword_and_prefix') { + $event->setIsSupported(TRUE); + } + } + + /** + * Event handler for FieldMappingEvent. + */ + public function onFieldMappingEvent(FieldMappingEvent $event): void { + $param = $event->getParam(); + $field = $event->getField(); + $type = $field->getType(); + $param = match ($type) { + 'tide_search_text_field_with_keyword_and_prefix' => [ + "type" => "text", + "fields" => [ + "keyword" => [ + "type" => "keyword", + "ignore_above" => 256, + ], + "prefix" => [ + "type" => "text", + "index_options" => "docs", + "analyzer" => "i_prefix", + "search_analyzer" => "q_prefix", + ], + ], + ], + default => [], + }; + $event->setParam($param); + } + + /** + * Event handler for AlterSettingsEvent. + */ + public function onAlterSettingsEvent(AlterSettingsEvent $event): void { + $settings = $event->getSettings(); + $settings['analysis'] = [ + "filter" => [ + "front_ngram" => [ + "type" => "edge_ngram", + "min_gram" => "1", + "max_gram" => "12", + ], + ], + "analyzer" => [ + "i_prefix" => [ + "filter" => [ + "cjk_width", + "lowercase", + "asciifolding", + "front_ngram", + ], + "tokenizer" => "standard", + ], + "q_prefix" => [ + "filter" => [ + "cjk_width", + "lowercase", + "asciifolding", + ], + "tokenizer" => "standard", + ], + ], + ]; + $event->setSettings($settings); + } + +} diff --git a/modules/tide_search/src/Plugin/search_api/data_type/TextFieldWithKeywordAndPrefix.php b/modules/tide_search/src/Plugin/search_api/data_type/TextFieldWithKeywordAndPrefix.php new file mode 100644 index 00000000..fa076a92 --- /dev/null +++ b/modules/tide_search/src/Plugin/search_api/data_type/TextFieldWithKeywordAndPrefix.php @@ -0,0 +1,21 @@ + 10005]; + $dependencies['tide_search'][10005] = ['elasticsearch_connector' => 9802]; return $dependencies; } @@ -117,3 +119,20 @@ function tide_search_update_10004() { _tide_core_field_content_category_default_value('tide_search_listing', 'Search listing'); _tide_core_content_category_form_display('tide_search_listing'); } + +/** + * Update for elasticsearch php package version 8. + */ +function tide_search_update_10005() { + $update_helper = \Drupal::service('tide_core.entity_update_helper'); + $update_helper->updateFromOptional('search_api_server', 'elasticsearch_bay'); + $index = Index::load('node'); + $field_settings = $index->get('field_settings'); + if (!empty($field_settings['title'])) { + $field_settings['title']['type'] = 'tide_search_text_field_with_keyword_and_prefix'; + } + if (!empty($field_settings['summary_processed'])) { + $field_settings['summary_processed']['type'] = 'tide_search_text_field_with_keyword_and_prefix'; + } + $index->set('field_settings', $field_settings)->save(); +} diff --git a/modules/tide_search/tide_search.services.yml b/modules/tide_search/tide_search.services.yml new file mode 100644 index 00000000..350027cc --- /dev/null +++ b/modules/tide_search/tide_search.services.yml @@ -0,0 +1,5 @@ +services: + tide_search.event_subscriber: + class: Drupal\tide_search\EventSubscriber\ElasticSearchEventSubscribers + tags: + - { name: event_subscriber }